Compare commits

..

1 Commits

Author SHA1 Message Date
yongkangc
082daa7fc1 fix(rocksdb): filter account/storage history to match MDBX semantics 2026-01-22 12:58:59 +00:00
352 changed files with 6867 additions and 17568 deletions

2
.github/CODEOWNERS vendored
View File

@@ -37,7 +37,7 @@ crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse

View File

@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
done
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/scripts/hive/no_sim_build.diff
git apply ../.github/assets/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -60,6 +60,7 @@ jobs:
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

54
.github/workflows/docker-git.yml vendored Normal file
View File

@@ -0,0 +1,54 @@
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
# images from this workflow will be tagged with the git sha of the branch used
# and will NOT tag it as `latest`.
name: docker-git
on:
workflow_dispatch: {}
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
GIT_SHA: ${{ github.sha }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the git-sha-tagged reth image'
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
- name: 'Build and push the git-sha-tagged op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

65
.github/workflows/docker-nightly.yml vendored Normal file
View File

@@ -0,0 +1,65 @@
# Publishes the nightly Docker image.
name: docker-nightly
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:
- uses: actions/checkout@v6
- name: Remove bloatware
uses: laverdet/remove-bloatware@v1.0.0
with:
docker: true
lang: rust
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,9 +1,4 @@
# Publishes Docker images.
#
# Triggers:
# - Push tag v*: builds release (RC or latest)
# - Schedule: builds nightly + profiling
# - Manual: builds git-sha or nightly
# Publishes the Docker image.
name: docker
@@ -11,94 +6,84 @@ on:
push:
tags:
- v*
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
build_type:
description: "Build type"
required: true
type: choice
options:
- git-sha
- nightly
default: git-sha
dry_run:
description: "Skip pushing images (dry run)"
required: false
type: boolean
default: false
env:
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: Build Docker images
build-rc:
if: contains(github.ref, '-rc')
name: build and push as release candidate
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
steps:
- uses: actions/checkout@v6
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v3
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get git info for vergen
id: git
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Determine build parameters
id: params
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
REGISTRY="ghcr.io/${{ github.repository_owner }}"
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
if [[ "${{ github.event_name }}" == "push" ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
uses: depot/bake-action@v1
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
build:
if: ${{ !contains(github.ref, '-rc') }}
name: build and push as latest
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -44,24 +44,3 @@ jobs:
--exclude 'op-reth' \
--exclude 'reth' \
-E 'binary(e2e_testsuite)'
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run RocksDB e2e tests
run: |
cargo nextest run \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -58,11 +58,11 @@ jobs:
uses: actions/cache@v5
with:
path: ./hive_assets
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
- name: Build hive assets
if: steps.cache-hive.outputs.cache-hit != 'true'
run: .github/scripts/hive/build_simulators.sh
run: .github/assets/hive/build_simulators.sh
- name: Load cached Docker images
if: steps.cache-hive.outputs.cache-hit == 'true'
@@ -213,7 +213,7 @@ jobs:
path: /tmp
- name: Load Docker images
run: .github/scripts/hive/load_images.sh
run: .github/assets/hive/load_images.sh
- name: Move hive binary
run: |
@@ -241,11 +241,11 @@ jobs:
FILTER="/"
fi
echo "filter: $FILTER"
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
- name: Print simulator output
if: ${{ failure() }}

View File

@@ -22,7 +22,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
name: test / ${{ matrix.network }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
env:
@@ -30,17 +30,13 @@ jobs:
strategy:
matrix:
network: ["ethereum", "optimism"]
storage: ["stable", "edge"]
exclude:
- network: optimism
storage: edge
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
run: .github/scripts/install_geth.sh
run: .github/assets/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
@@ -50,7 +46,7 @@ jobs:
name: Run tests
run: |
cargo nextest run \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--locked --features "asm-keccak ${{ matrix.network }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"
- if: matrix.network == 'optimism'

View File

@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/scripts/label_pr.js')
const label_pr = require('./.github/assets/label_pr.js')
await label_pr({github, context})

View File

@@ -76,7 +76,7 @@ jobs:
- name: Run Wasm checks
run: |
sudo apt update && sudo apt install gcc-multilib
.github/scripts/check_wasm.sh
.github/assets/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
cache-on-failure: true
- uses: dcarbone/install-jq-action@v3
- name: Run RISC-V checks
run: .github/scripts/check_rv32imac.sh
run: .github/assets/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})

View File

@@ -43,7 +43,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
file: .github/scripts/hive/Dockerfile
file: .github/assets/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |

606
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -481,52 +481,52 @@ revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.1"
revm-inspectors = "0.34.0"
# eth
alloy-dyn-abi = "1.5.4"
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
# op
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -588,7 +588,7 @@ tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
byteorder = "1"
fixed-cache = { version = "0.1.7", features = ["stats"] }
mini-moka = "0.10"
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
@@ -671,7 +671,7 @@ tracing-opentelemetry = "0.32"
# misc-testing
arbitrary = "1.3"
assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
@@ -733,7 +733,7 @@ snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "=0.3.5"
tracing-logfmt = "0.3.3"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
@@ -790,10 +790,39 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

15
Dockerfile.cross Normal file
View File

@@ -0,0 +1,15 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth"]

View File

@@ -1,84 +0,0 @@
# syntax=docker/dockerfile:1
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
# Usage:
# reth: --build-arg BINARY=reth
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
FROM rust:1 AS builder
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get install -y libclang-dev pkg-config
# Install sccache for compilation caching
RUN cargo install sccache --locked
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Binary to build (reth or op-reth)
ARG BINARY=reth
# Manifest path for the binary
ARG MANIFEST_PATH=bin/reth
# Build profile, release by default
ARG BUILD_PROFILE=release
ENV BUILD_PROFILE=$BUILD_PROFILE
# Extra Cargo flags
ARG RUSTFLAGS=""
ENV RUSTFLAGS="$RUSTFLAGS"
# Extra Cargo features
ARG FEATURES=""
ENV FEATURES=$FEATURES
# Git info for vergen (since .git is excluded from Docker context)
ARG VERGEN_GIT_SHA=""
ARG VERGEN_GIT_DESCRIBE=""
ARG VERGEN_GIT_DIRTY="false"
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build application
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
RUN sccache --show-stats || true
# Copy binary to a known location (ARG not resolved in COPY)
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
cp /app/target/release/$BINARY /app/binary
FROM ubuntu:24.04 AS runtime
WORKDIR /app
# Binary name for entrypoint
ARG BINARY=reth
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy binary from build stage and create canonical symlink for entrypoint
COPY --from=builder /app/binary /usr/local/bin/
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
chmod +x /usr/local/bin/$BINARY
# Copy licenses
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth-binary"]

15
DockerfileOp.cross Normal file
View File

@@ -0,0 +1,15 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/op-reth"]

134
Makefile
View File

@@ -35,6 +35,9 @@ EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
# The docker image name
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
##@ Help
.PHONY: help
@@ -239,6 +242,137 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
--features "$(FEATURES)" \
--profile "$(PROFILE)"
##@ Docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-git-sha
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-latest
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
docker buildx build --file ./Dockerfile.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Optimism docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-git-sha
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-latest
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call op_docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly-profiling
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
$(call docker_build_push,nightly-profiling,nightly-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly-profiling
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
docker buildx build --file ./DockerfileOp.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Other
.PHONY: clean

View File

@@ -274,10 +274,10 @@ impl Args {
/// Get the default RPC URL for a given chain
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
match chain.id() {
8453 => "https://base.reth.rs/rpc", // base
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
}
}

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-primitives.workspace = true
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -22,6 +22,29 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
@@ -214,3 +237,50 @@ const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -3,9 +3,7 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use crate::authenticated_transport::AuthenticatedTransportConnect;
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -132,24 +130,13 @@ impl<S: TransactionSource> TransactionCollector<S> {
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns a `CollectionResult` with transactions, gas info, and next block.
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
self.collect_gas(start_block, self.target_gas).await
}
/// Collect transactions up to a specific gas target.
///
/// This is used both for initial collection and for retry top-ups.
pub async fn collect_gas(
&self,
start_block: u64,
gas_target: u64,
) -> eyre::Result<CollectionResult> {
let mut transactions: Vec<RawTransaction> = Vec::new();
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < gas_target {
while total_gas < self.target_gas {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
@@ -162,12 +149,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
continue;
}
if total_gas + tx.gas_used <= gas_target {
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
total_gas += tx.gas_used;
transactions.push(tx);
}
if total_gas >= gas_target {
if total_gas >= self.target_gas {
break;
}
}
@@ -175,7 +162,7 @@ impl<S: TransactionSource> TransactionCollector<S> {
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = gas_target.saturating_sub(total_gas);
let remaining_gas = self.target_gas.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
@@ -183,12 +170,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
info!(
total_txs = transactions.len(),
gas_sent = total_gas,
total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
Ok((transactions, total_gas, current_block))
}
}
@@ -215,26 +202,13 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
target_gas: u64,
/// Block number to start fetching transactions from (required).
///
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
/// The command collects transactions from historical blocks starting at this number
/// to pack into large blocks.
///
/// How to determine this value:
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
/// - If gas limit ramping has already been performed: use the block number that was the chain
/// tip BEFORE ramping began (you must track this yourself)
///
/// Using a block after ramping started will cause transaction collection to fail
/// because those blocks contain synthetic transactions that cannot be replayed.
/// Starting block number to fetch transactions from.
/// If not specified, starts from the engine's latest block.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: u64,
from_block: Option<u64>,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
@@ -263,80 +237,6 @@ struct BuiltPayload {
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
/// The actual gas used in the built block.
gas_used: u64,
}
/// Result of collecting transactions from blocks.
#[derive(Debug)]
pub struct CollectionResult {
/// Collected transactions with their gas info.
pub transactions: Vec<RawTransaction>,
/// Total gas sent (sum of historical `gas_used` for all collected txs).
pub gas_sent: u64,
/// Next block number to continue collecting from.
pub next_block: u64,
}
/// Constants for retry logic.
const MAX_BUILD_RETRIES: u32 = 5;
/// Maximum retries for fetching a transaction batch.
const MAX_FETCH_RETRIES: u32 = 5;
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
const MIN_TARGET_SLACK: u64 = 1_000_000;
/// Maximum gas to request in retries (10x target as safety cap).
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
/// Fetches a batch of transactions with retry logic.
///
/// Returns `None` if all retries are exhausted.
async fn fetch_batch_with_retry<S: TransactionSource>(
collector: &TransactionCollector<S>,
block: u64,
) -> Option<CollectionResult> {
for attempt in 1..=MAX_FETCH_RETRIES {
match collector.collect(block).await {
Ok(result) => return Some(result),
Err(e) => {
if attempt == MAX_FETCH_RETRIES {
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
return None;
}
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
None
}
/// Outcome of a build attempt check.
enum RetryOutcome {
/// Payload is close enough to target gas.
Success,
/// Max retries reached, accept what we have.
MaxRetries,
/// Need more transactions with the specified gas amount.
NeedMore(u64),
}
/// Buffer for receiving transaction batches from the fetcher.
///
/// This abstracts over the channel to allow the main loop to request
/// batches on demand, including for retries.
struct TxBuffer {
receiver: mpsc::Receiver<CollectionResult>,
}
impl TxBuffer {
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
Self { receiver }
}
/// Take the next available batch from the fetcher.
async fn take_batch(&mut self) -> Option<CollectionResult> {
self.receiver.recv().await
}
}
impl Command {
@@ -384,7 +284,7 @@ impl Command {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block;
let start_block = self.from_block.unwrap_or(parent_number);
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
@@ -397,20 +297,19 @@ impl Command {
)
.await?;
} else {
// Single payload - collect transactions and build with retry
// Single payload - collect transactions and build
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let result = collector.collect(start_block).await?;
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
if result.transactions.is_empty() {
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential_with_retry(
self.execute_sequential(
&auth_provider,
&testing_provider,
&collector,
result,
transactions,
parent_hash,
parent_timestamp,
)
@@ -421,34 +320,32 @@ impl Command {
Ok(())
}
/// Sequential execution path with retry logic for underfilled payloads.
async fn execute_sequential_with_retry<S: TransactionSource>(
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
initial_result: CollectionResult,
transactions: Vec<Bytes>,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
let mut current_result = initial_result;
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let built = self
.build_with_retry(
testing_provider,
collector,
&mut current_result,
i,
parent_hash,
parent_timestamp,
)
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
@@ -459,62 +356,7 @@ impl Command {
Ok(())
}
/// Build a payload with retry logic when `gas_used` is below target.
///
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
/// are needed to hit the target gas.
async fn build_with_retry<S: TransactionSource>(
&self,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let additional =
collector.collect_gas(result.next_block, additional_gas).await?;
result.transactions.extend(additional.transactions);
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
result.next_block = additional.next_block;
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
}
/// Pipelined execution - fetches transactions in background, builds with retry.
///
/// The fetcher continuously produces transaction batches. The main loop consumes them,
/// builds payloads with retry logic (requesting more transactions if underfilled),
/// and executes each payload before moving to the next.
/// Pipelined execution - fetches transactions and builds payloads in background.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
@@ -523,220 +365,167 @@ impl Command {
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches - fetcher sends CollectionResult
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return None;
return;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
if batch.transactions.is_empty() {
info!(block = current_block, "Reached chain tip, stopping fetcher");
break;
}
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
info!(
tx_count = batch.transactions.len(),
gas_sent = batch.gas_sent,
blocks = format!("{}..{}", current_block, batch.next_block),
"Fetched transaction batch"
);
current_block = batch.next_block;
if tx_sender.send(batch).await.is_err() {
break;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
Some(current_block)
});
// Transaction buffer: holds transactions from batches + any extras from retries
let mut tx_buffer = TxBuffer::new(tx_receiver);
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
// Get initial batch of transactions for this payload
let mut result = tx_buffer
.take_batch()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
let is_last = i == self.count - 1;
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
// Build with retry - may need to request more transactions
let built = self
.build_with_retry_buffered(
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
);
self.build_payload(
testing_provider,
&mut tx_buffer,
&mut result,
&transactions,
i,
parent_hash,
parent_timestamp,
)
.await?;
.await?
};
self.save_payload(&built)?;
self.save_payload(&current_payload)?;
let current_block_hash = built.block_hash;
let current_timestamp = built.timestamp;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
// Execute payload
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_buffer);
drop(tx_receiver);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a payload with retry logic, using the buffered transaction source.
async fn build_with_retry_buffered(
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
&self,
testing_provider: &RootProvider<AnyNetwork>,
tx_buffer: &mut TxBuffer,
result: &mut CollectionResult,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let mut collected_gas = 0u64;
while collected_gas < additional_gas {
if let Some(batch) = tx_buffer.take_batch().await {
collected_gas += batch.gas_sent;
result.transactions.extend(batch.transactions);
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
result.next_block = batch.next_block;
} else {
warn!("Transaction fetcher exhausted, proceeding with available transactions");
break;
}
}
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
}
/// Determines the outcome of a build attempt.
fn check_retry_outcome(
&self,
built: &BuiltPayload,
index: u64,
attempt: u32,
gas_sent: u64,
) -> RetryOutcome {
let gas_used = built.gas_used;
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
attempts = attempt,
"Payload built successfully"
);
return RetryOutcome::Success;
}
if attempt == MAX_BUILD_RETRIES {
warn!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
gas_sent,
"Underfilled after max retries, accepting payload"
);
return RetryOutcome::MaxRetries;
}
if gas_used == 0 {
warn!(
payload = index + 1,
"Zero gas used in payload, requesting fixed chunk of additional transactions"
);
return RetryOutcome::NeedMore(self.target_gas);
}
let gas_sent_needed_total =
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
if additional == 0 {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
"No additional transactions needed based on ratio"
);
return RetryOutcome::Success;
}
let ratio = gas_used as f64 / gas_sent as f64;
info!(
payload = index + 1,
gas_used,
gas_sent,
ratio = format!("{:.4}", ratio),
additional_gas = additional,
"Underfilled, collecting more transactions for retry"
);
RetryOutcome::NeedMore(additional)
}
/// Build a single payload via `testing_buildBlockV1`.
/// Static version for use in spawned tasks.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
@@ -774,9 +563,8 @@ impl Command {
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
let gas_used = inner.gas_used;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
}
/// Save a payload to disk.

View File

@@ -1,56 +1,6 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::io::{BufReader, Read};
/// Read input from either a file path or stdin.
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
Ok(match path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly.
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
match jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.to_string())),
}
}
None => Ok(None),
}
}
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
@@ -244,50 +194,3 @@ pub(crate) async fn get_payload_with_sidecar(
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -15,9 +15,7 @@ pub use generate_big_block::{
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -76,18 +74,6 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -103,7 +89,6 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -15,23 +15,28 @@ use crate::{
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_provider::Provider;
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -100,11 +105,7 @@ impl Command {
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url).await?;
let sub = self.setup_persistence_subscription().await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
@@ -244,22 +245,293 @@ impl Command {
results.into_iter().unzip();
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
write_benchmark_results(path, &gas_output_results, combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
let gas_output = TotalGasOutput::new(gas_output_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,
"Total Ggas/s: {:.4}",
gas_output.total_gigagas_per_second()
);
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -6,7 +6,7 @@ use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{fs, path::Path, time::Duration};
use std::{path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
@@ -158,58 +158,29 @@ pub(crate) struct TotalGasRow {
pub(crate) struct TotalGasOutput {
/// The total gas used in the benchmark.
pub(crate) total_gas_used: u64,
/// The total wall-clock duration of the benchmark (includes wait times).
/// The total duration of the benchmark.
pub(crate) total_duration: Duration,
/// The total execution-only duration (excludes wait times).
pub(crate) execution_duration: Duration,
/// The total gas used per second.
pub(crate) total_gas_per_second: f64,
/// The number of blocks processed.
pub(crate) blocks_processed: u64,
}
impl TotalGasOutput {
/// Create a new [`TotalGasOutput`] from gas rows only.
///
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
/// `execution_duration` will equal `total_duration`.
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
// the duration is obtained from the last row
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
Ok(Self {
total_gas_used,
total_duration,
execution_duration: total_duration,
blocks_processed,
})
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
}
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
///
/// - `rows`: Used for total gas and wall-clock duration
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
pub(crate) fn with_combined_results(
rows: Vec<TotalGasRow>,
combined_results: &[CombinedResult],
) -> eyre::Result<Self> {
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
// Sum execution-only time from combined results
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
}
/// Return the total gigagas per second based on wall-clock time.
/// Return the total gigagas per second.
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
}
/// Return the execution-only gigagas per second (excludes wait times).
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
self.total_gas_per_second / GIGAGAS as f64
}
}
@@ -221,10 +192,8 @@ impl TotalGasOutput {
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: &[CombinedResult],
combined_results: Vec<CombinedResult>,
) -> eyre::Result<()> {
fs::create_dir_all(output_dir)?;
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;

View File

@@ -1,304 +0,0 @@
//! Persistence waiting utilities for benchmarks.
//!
//! Provides waiting behavior to control benchmark pacing:
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
//! - **Persistence-based waits**: Wait for blocks to be persisted using
//! `reth_subscribePersistedBlock` subscription
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_transport_ws::WsConnect;
use eyre::Context;
use futures::StreamExt;
use std::time::Duration;
use tracing::{debug, info};
/// Default `WebSocket` RPC port for reth.
const DEFAULT_WS_RPC_PORT: u16 = 8546;
use url::Url;
/// Default timeout for waiting on persistence.
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `ws_rpc_url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since we may only have the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
pub(crate) fn derive_ws_rpc_url(
ws_rpc_url: Option<&str>,
engine_rpc_url: &str,
) -> eyre::Result<Url> {
if let Some(ws_url) = ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url
.set_port(Some(DEFAULT_WS_RPC_PORT))
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
pub(crate) struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
) -> eyre::Result<PersistenceSubscription> {
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
pub(crate) struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
pub(crate) const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Instant;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -2,27 +2,10 @@
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
output::{
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
bench::output::GasRampPayloadFile,
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
@@ -31,16 +14,11 @@ use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use humantime::parse_duration;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use std::{
path::PathBuf,
time::{Duration, Instant},
};
use std::path::PathBuf;
use tracing::{debug, info};
use url::Url;
/// `reth bench replay-payloads` command
///
@@ -73,42 +51,6 @@ pub struct Command {
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
/// Optional output directory for benchmark results (CSV files).
#[arg(long, value_name = "OUTPUT")]
output: Option<PathBuf>,
/// How long to wait after a forkchoice update before sending the next payload.
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// Optional `WebSocket` RPC URL for persistence subscription.
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
}
/// A loaded payload ready for execution.
@@ -136,33 +78,6 @@ impl Command {
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Log mode configuration
if let Some(duration) = self.wait_time {
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
};
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
@@ -229,11 +144,6 @@ impl Command {
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
if let Some(w) = &mut waiter {
w.on_block(payload.block_number).await?;
}
parent_hash = payload.file.block_hash;
}
@@ -241,112 +151,22 @@ impl Command {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
for (i, payload) in payloads.iter().enumerate() {
let envelope = &payload.envelope;
let block_hash = payload.block_hash;
let execution_payload = &envelope.envelope_inner.execution_payload;
let inner_payload = &execution_payload.payload_inner.payload_inner;
let gas_used = inner_payload.gas_used;
let gas_limit = inner_payload.gas_limit;
let block_number = inner_payload.block_number;
let transaction_count =
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
debug!(
info!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %block_hash,
block_hash = %payload.block_hash,
"Executing payload (newPayload + FCU)"
);
let start = Instant::now();
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = auth_provider
.new_payload_v4(
execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
total_latency,
};
let current_duration = total_benchmark_duration.elapsed();
info!(%combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(?status, ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
}
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
if let Some(ref path) = self.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
info!(count = payloads.len(), "All payloads replayed successfully");
Ok(())
}
@@ -464,4 +284,49 @@ impl Command {
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -1,219 +0,0 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -1,347 +0,0 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use super::helpers::{load_jwt_secret, read_input};
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = read_input(self.path.as_deref())?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -1,11 +1,10 @@
use super::helpers::{load_jwt_secret, read_input};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
/// block.
@@ -52,13 +51,38 @@ enum Mode {
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.clone())),
}
}
None => Ok(None),
}
}
/// Execute the generate payload command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
// Load block
let block_json = read_input(self.path.as_deref())?;
let block_json = self.read_input()?;
// Load JWT secret
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let jwt_secret = self.load_jwt_secret()?;
// Parse the block
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?

View File

@@ -260,9 +260,7 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
while !status.is_valid() {
if status.is_invalid() {
error!(?status, ?params, "Invalid {method}",);
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
format!("Invalid {method}: {status:?}"),
))))
panic!("Invalid {method}: {status:?}");
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -206,33 +206,11 @@ impl DeferredTrieData {
Default::default(), // prefix_sets are per-block, not cumulative
);
// Only trigger COW clone if there's actually data to add.
#[cfg(feature = "rayon")]
{
rayon::join(
|| {
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
},
|| {
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
},
);
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
}
#[cfg(not(feature = "rayon"))]
{
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
}
overlay
}
@@ -265,8 +243,53 @@ impl DeferredTrieData {
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// Iterates ancestors oldest -> newest, then extends with current block's data,
/// so later state takes precedence.
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
@@ -284,17 +307,8 @@ impl DeferredTrieData {
}
// Extend with current block's sorted data last (takes precedence)
#[cfg(feature = "rayon")]
rayon::join(
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
);
#[cfg(not(feature = "rayon"))]
{
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
}
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
overlay
}

View File

@@ -17,10 +17,7 @@ use reth_primitives_traits::{
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
TrieInputSorted,
};
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
@@ -951,36 +948,22 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let trie_data_handle = first.trie_data_handle();
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
);
for exec in rest {
let trie_data_handle = exec.trie_data_handle();
chain.append_block(
exec.recovered_block().clone(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
);
}
chain

View File

@@ -25,7 +25,6 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
/// Re-export for convenience
pub use reth_ethereum_forks::*;
pub use alloy_evm::EvmLimitParams;
pub use api::EthChainSpec;
pub use info::ChainInfo;
#[cfg(any(test, feature = "test-utils"))]

View File

@@ -50,7 +50,6 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true

View File

@@ -21,7 +21,6 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -83,33 +82,6 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -140,9 +112,6 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -221,9 +190,6 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -61,21 +61,19 @@ impl Command {
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
if let Some(search) = search.strip_prefix("0x") {
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
return hex::decode(search).unwrap()
}
}
None => Vec::new(),
};
search.as_bytes().to_vec()
})
.unwrap_or_default();
Ok(ListFilter {
ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -84,7 +82,7 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
})
}
}
}
@@ -101,8 +99,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
// We may be using the tui for a long time
tx.disable_long_read_transaction_safety();
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {
@@ -117,7 +115,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter()?;
let list_filter = self.args.list_filter();
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor, &data_dir)?;
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

View File

@@ -69,11 +69,6 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -120,7 +115,6 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -173,14 +167,6 @@ impl Command {
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -92,10 +92,10 @@ impl Command {
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
let stats = tx
.inner()
.inner
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
@@ -136,9 +136,9 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);
let freelist = tx.inner().env().freelist()?;
let freelist = tx.inner.env().freelist()?;
let pagesize =
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();
@@ -205,16 +205,6 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
let mut row = Row::new();
row.add_cell(Cell::new("WAL"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(wal_size as f64)))
.add_cell(Cell::new(""));
table.add_row(row);
}
table

View File

@@ -26,14 +26,6 @@ pub struct ImportCommand<C: ChainSpecParser> {
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
/// Fail immediately when an invalid block is encountered.
///
/// By default, the import will stop at the last valid block if an invalid block is
/// encountered during execution or validation, leaving the database at the last valid
/// block state. When this flag is set, the import will instead fail with an error.
#[arg(long, verbatim_doc_comment)]
fail_on_invalid_block: bool,
/// The path(s) to block file(s) for import.
///
/// The online stages (headers and bodies) are replaced by a file import, after which the
@@ -60,11 +52,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
let import_config = ImportConfig {
no_state: self.no_state,
chunk_len: self.chunk_len,
fail_on_invalid_block: self.fail_on_invalid_block,
};
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
let executor = components.evm_config().clone();
let consensus = Arc::new(components.consensus().clone());
@@ -93,20 +81,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
total_decoded_blocks += result.total_decoded_blocks;
total_decoded_txns += result.total_decoded_txns;
// Check if we stopped due to an invalid block
if result.stopped_on_invalid_block {
info!(target: "reth::cli",
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
result.last_valid_block.unwrap_or(0),
result.bad_block.unwrap_or(0),
path.display(),
result.total_imported_blocks,
result.total_imported_txns);
// Stop importing further files and exit successfully
break;
}
if !result.is_successful() {
if !result.is_complete() {
return Err(eyre::eyre!(
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
path.display(),
@@ -123,7 +98,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
}
info!(target: "reth::cli",
"Import complete. Total: {}/{} blocks, {}/{} transactions",
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
Ok(())
@@ -164,20 +139,4 @@ mod tests {
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
}
#[test]
fn parse_import_command_with_fail_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
assert!(args.fail_on_invalid_block);
assert_eq!(args.paths.len(), 1);
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
}
#[test]
fn parse_import_command_default_stops_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "chain.rlp"]);
assert!(!args.fail_on_invalid_block);
}
}

View File

@@ -22,11 +22,11 @@ use reth_provider::{
StageCheckpointReader,
};
use reth_prune::PruneModes;
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
use reth_static_file::StaticFileProducer;
use std::{path::Path, sync::Arc};
use tokio::sync::watch;
use tracing::{debug, error, info, warn};
use tracing::{debug, error, info};
/// Configuration for importing blocks from RLP files.
#[derive(Debug, Clone, Default)]
@@ -35,9 +35,6 @@ pub struct ImportConfig {
pub no_state: bool,
/// Chunk byte length to read from file.
pub chunk_len: Option<u64>,
/// If true, fail immediately when an invalid block is encountered.
/// By default (false), the import stops at the last valid block and exits successfully.
pub fail_on_invalid_block: bool,
}
/// Result of an import operation.
@@ -51,12 +48,6 @@ pub struct ImportResult {
pub total_imported_blocks: usize,
/// Total number of transactions imported into the database.
pub total_imported_txns: usize,
/// Whether the import was stopped due to an invalid block.
pub stopped_on_invalid_block: bool,
/// The block number that was invalid, if any.
pub bad_block: Option<u64>,
/// The last valid block number when stopped due to invalid block.
pub last_valid_block: Option<u64>,
}
impl ImportResult {
@@ -65,14 +56,6 @@ impl ImportResult {
self.total_decoded_blocks == self.total_imported_blocks &&
self.total_decoded_txns == self.total_imported_txns
}
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
///
/// In stop-on-invalid-block mode, a partial import is considered successful if we
/// stopped due to an invalid block (leaving the DB at the last valid block).
pub fn is_successful(&self) -> bool {
self.is_complete() || self.stopped_on_invalid_block
}
}
/// Imports blocks from an RLP-encoded file into the database.
@@ -120,11 +103,6 @@ where
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Track if we stopped due to an invalid block
let mut stopped_on_invalid_block = false;
let mut bad_block_number: Option<u64> = None;
let mut last_valid_block_number: Option<u64> = None;
while let Some(file_client) =
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
{
@@ -159,51 +137,12 @@ where
// Run pipeline
info!(target: "reth::import", "Starting sync pipeline");
if import_config.fail_on_invalid_block {
// Original behavior: fail on unwind
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
}
} else {
// Default behavior: Use run_loop() to handle unwinds gracefully
let result = tokio::select! {
res = pipeline.run_loop() => res,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
};
match result {
Ok(ControlFlow::Unwind { target, bad_block }) => {
// An invalid block was encountered; stop at last valid block
let bad = bad_block.block.number;
warn!(
target: "reth::import",
bad_block = bad,
last_valid_block = target,
"Invalid block encountered during import; stopping at last valid block"
);
stopped_on_invalid_block = true;
bad_block_number = Some(bad);
last_valid_block_number = Some(target);
break;
}
Ok(ControlFlow::Continue { block_number }) => {
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
}
Ok(ControlFlow::NoProgress { block_number }) => {
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
}
Err(e) => {
// Propagate other pipeline errors
return Err(e.into());
}
}
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
}
sealed_header = provider_factory
@@ -221,20 +160,9 @@ where
total_decoded_txns,
total_imported_blocks,
total_imported_txns,
stopped_on_invalid_block,
bad_block: bad_block_number,
last_valid_block: last_valid_block_number,
};
if result.stopped_on_invalid_block {
info!(target: "reth::import",
total_imported_blocks,
total_imported_txns,
bad_block = ?result.bad_block,
last_valid_block = ?result.last_valid_block,
"Import stopped at last valid block due to invalid block"
);
} else if !result.is_complete() {
if !result.is_complete() {
error!(target: "reth::import",
total_decoded_blocks,
total_imported_blocks,

View File

@@ -1,34 +0,0 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,7 +18,6 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -86,9 +85,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -103,7 +99,6 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -131,8 +126,6 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -232,16 +225,4 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -1,16 +1,8 @@
//! Command that runs pruning without any limits.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
chain::ChainSpecInfo,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
@@ -21,42 +13,14 @@ use tracing::info;
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
env: EnvironmentArgs<C>,
/// Prometheus metrics configuration.
#[command(flatten)]
metrics: MetricArgs,
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
/// Execute the `prune` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
self,
ctx: CliContext,
) -> eyre::Result<()> {
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
let env = self.env.init::<N>(AccessRights::RW)?;
let provider_factory = env.provider_factory;
let config = env.config.prune;
let data_dir = env.data_dir;
if let Some(listen_addr) = self.metrics.prometheus {
let config = MetricServerConfig::new(
listen_addr,
VersionInfo {
version: version_metadata().cargo_pkg_version.as_ref(),
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
git_sha: version_metadata().vergen_git_sha.as_ref(),
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;
}
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");

View File

@@ -15,8 +15,7 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -91,14 +90,11 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let provider_rw = tool.provider_factory.database_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -172,20 +168,8 @@ impl<C: ChainSpecParser> Command<C> {
)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -193,14 +177,7 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
tx.clear::<tables::TransactionHashNumbers>()?;
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -438,8 +438,6 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -453,7 +451,6 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -472,10 +469,6 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
@@ -487,7 +480,6 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = StaticFileMap::default();
@@ -500,7 +492,6 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {

View File

@@ -1,11 +1,14 @@
//! Collection of methods for block validation.
use alloy_consensus::{BlockHeader as _, EMPTY_OMMER_ROOT_HASH};
use alloy_consensus::{BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH};
use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams};
use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks};
use reth_consensus::ConsensusError;
use reth_consensus::{ConsensusError, TxGasLimitTooHighErr};
use reth_primitives_traits::{
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
constants::{
GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT,
},
transaction::TxHashRef,
Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader,
};
@@ -143,7 +146,7 @@ pub fn validate_block_pre_execution<B, ChainSpec>(
) -> Result<(), ConsensusError>
where
B: Block,
ChainSpec: EthChainSpec + EthereumHardforks,
ChainSpec: EthereumHardforks,
{
post_merge_hardfork_fields(block, chain_spec)?;
@@ -151,6 +154,19 @@ where
if let Err(error) = block.ensure_transaction_root_valid() {
return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
}
// EIP-7825 validation
if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) {
for tx in block.body().transactions() {
if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA {
return Err(TxGasLimitTooHighErr {
tx_hash: *tx.tx_hash(),
gas_limit: tx.gas_limit(),
max_allowed: MAX_TX_GAS_LIMIT_OSAKA,
}
.into());
}
}
}
Ok(())
}

View File

@@ -72,11 +72,3 @@ derive_more.workspace = true
[[test]]
name = "e2e_testsuite"
path = "tests/e2e-testsuite/main.rs"
[[test]]
name = "rocksdb"
path = "tests/rocksdb/main.rs"
required-features = ["edge"]
[features]
edge = ["reth-node-core/edge", "reth-provider/rocksdb", "reth-cli-commands/edge"]

View File

@@ -103,10 +103,7 @@ where
N: NodeBuilderHelper,
{
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_tree_config_modifier(move |base| {
// Apply caller's tree_config but preserve the small cache size from base
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
})
.with_tree_config_modifier(move |_| tree_config.clone())
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.with_connect_nodes(connect_nodes)
.build()

View File

@@ -112,13 +112,11 @@ where
..NetworkArgs::default()
};
// Apply tree config modifier if present, with test-appropriate defaults
let base_tree_config =
reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024);
// Apply tree config modifier if present
let tree_config = if let Some(modifier) = self.tree_config_modifier {
modifier(base_tree_config)
modifier(reth_node_api::TreeConfig::default())
} else {
base_tree_config
reth_node_api::TreeConfig::default()
};
let mut nodes = (0..self.num_nodes)

View File

@@ -125,10 +125,7 @@ pub async fn setup_engine_with_chain_import(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
)?;
// Initialize genesis if needed
@@ -331,7 +328,6 @@ mod tests {
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone())
.with_default_tables()
.build()
.unwrap(),
)
@@ -396,7 +392,6 @@ mod tests {
reth_provider::providers::StaticFileProvider::read_only(static_files_path, false)
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)
@@ -495,10 +490,7 @@ mod tests {
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
)
.expect("failed to create provider factory");

View File

@@ -38,18 +38,6 @@ impl TransactionTestContext {
signed.encoded_2718().into()
}
/// Creates a transfer with a specific nonce and signs it, returning bytes.
/// Uses high `max_fee_per_gas` (1000 gwei) to ensure tx acceptance regardless of basefee.
pub async fn transfer_tx_bytes_with_nonce(
chain_id: u64,
wallet: PrivateKeySigner,
nonce: u64,
) -> Bytes {
let tx = tx(chain_id, 21000, None, None, nonce, Some(1000e9 as u128));
let signed = Self::sign_tx(wallet, tx).await;
signed.encoded_2718().into()
}
/// Creates a deployment transaction and signs it, returning an envelope.
pub async fn deploy_tx(
chain_id: u64,

View File

@@ -1,591 +0,0 @@
//! E2E tests for `RocksDB` provider functionality.
#![cfg(all(feature = "edge", unix))]
use alloy_consensus::BlockHeader;
use alloy_primitives::B256;
use alloy_rpc_types_eth::{Transaction, TransactionReceipt};
use eyre::Result;
use jsonrpsee::core::client::ClientT;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_db::tables;
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
use reth_node_core::args::RocksDbArgs;
use reth_node_ethereum::EthereumNode;
use reth_payload_builder::EthPayloadBuilderAttributes;
use reth_provider::{RocksDBProviderFactory, StorageSettings};
use std::{sync::Arc, time::Duration};
const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60);
const ROCKSDB_POLL_INTERVAL: Duration = Duration::from_millis(50);
/// Polls RPC until the given `tx_hash` is visible as pending (not yet mined).
/// Prevents race conditions where `advance_block` is called before txs are in the pool.
/// Returns the pending transaction.
async fn wait_for_pending_tx<C: ClientT>(client: &C, tx_hash: B256) -> Transaction {
let start = std::time::Instant::now();
loop {
let tx: Option<Transaction> = client
.request("eth_getTransactionByHash", [tx_hash])
.await
.expect("RPC request failed");
if let Some(tx) = tx {
assert!(
tx.block_number.is_none(),
"Expected pending tx but tx_hash={tx_hash:?} is already mined in block {:?}",
tx.block_number
);
return tx;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} to appear in pending pool",
start.elapsed()
);
tokio::time::sleep(ROCKSDB_POLL_INTERVAL).await;
}
}
/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers`.
/// Returns the `tx_number` on success, or panics on timeout.
async fn poll_tx_in_rocksdb<P: RocksDBProviderFactory>(provider: &P, tx_hash: B256) -> u64 {
let start = std::time::Instant::now();
let mut interval = ROCKSDB_POLL_INTERVAL;
loop {
// Re-acquire handle each iteration to avoid stale snapshot reads
let rocksdb = provider.rocksdb_provider();
let tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(tx_hash).expect("RocksDB get failed");
if let Some(n) = tx_number {
return n;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} in RocksDB",
start.elapsed()
);
tokio::time::sleep(interval).await;
// Simple backoff: 50ms -> 100ms -> 200ms (capped)
interval = std::cmp::min(interval * 2, Duration::from_millis(200));
}
}
/// Returns the test chain spec for `RocksDB` tests.
fn test_chain_spec() -> Arc<ChainSpec> {
Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(
serde_json::from_str(include_str!("../../src/testsuite/assets/genesis.json"))
.expect("failed to parse genesis.json"),
)
.cancun_activated()
.build(),
)
}
/// Returns test payload attributes for the given timestamp.
fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
let attributes = alloy_rpc_types_engine::PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Verifies that `RocksDB` CLI defaults match `StorageSettings::base()`.
#[test]
fn test_rocksdb_defaults_match_storage_settings() {
let args = RocksDbArgs::default();
let settings = StorageSettings::base();
assert_eq!(
args.tx_hash, settings.transaction_hash_numbers_in_rocksdb,
"tx_hash default should match StorageSettings::base()"
);
assert_eq!(
args.storages_history, settings.storages_history_in_rocksdb,
"storages_history default should match StorageSettings::base()"
);
assert_eq!(
args.account_history, settings.account_history_in_rocksdb,
"account_history default should match StorageSettings::base()"
);
}
/// Smoke test: node boots with `RocksDB` routing enabled.
#[tokio::test]
async fn test_rocksdb_node_startup() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let (nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Verify RocksDB provider is functional (can query without error)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_hash = B256::from([0xab; 32]);
let result: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(result.is_none(), "Missing hash should return None");
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
Ok(())
}
/// Block mining works with `RocksDB` storage.
#[tokio::test]
async fn test_rocksdb_block_mining() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.build()
.await?;
assert_eq!(nodes.len(), 1);
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
// Mine 3 blocks with transactions
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
for i in 1..=3u64 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), i - 1)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
let block = payload.block();
assert_eq!(block.number(), i);
assert_ne!(block.hash(), B256::ZERO);
// Verify tx was actually included in the block
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should exist after mining");
assert_eq!(receipt.block_number, Some(i), "Tx should be in block {i}");
}
// Verify all blocks are stored
for i in 0..=3 {
let block_hash = nodes[0].block_hash(i);
assert_ne!(block_hash, B256::ZERO);
}
Ok(())
}
/// Tx hash lookup exercises `TransactionHashNumbers` table.
#[tokio::test]
async fn test_rocksdb_transaction_queries() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Inject and mine a transaction
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Query each transaction by hash
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1));
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should be found");
assert_eq!(receipt.block_number, Some(1));
assert!(receipt.status());
// Direct RocksDB assertion - poll with timeout since persistence is async
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have TxNumber 0");
// Verify missing hash returns None
let missing_hash = B256::from([0xde; 32]);
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(missing_tx_number.is_none());
let missing_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [missing_hash]).await?;
assert!(missing_tx.is_none(), "expected no transaction for missing hash");
let missing_receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [missing_hash]).await?;
assert!(missing_receipt.is_none(), "expected no receipt for missing hash");
Ok(())
}
/// Multiple transactions in the same block are correctly persisted to `RocksDB`.
#[tokio::test]
async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
// Create 3 txs from the same wallet with sequential nonces
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
let mut tx_hashes = Vec::new();
for nonce in 0..3 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
tx_hashes.push(tx_hash);
}
// Wait for all txs to appear in pending pool before mining
for tx_hash in &tx_hashes {
wait_for_pending_tx(&client, *tx_hash).await;
}
// Mine one block containing all 3 txs
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Verify block contains all 3 txs
let block: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("0x1", true)).await?;
let block = block.expect("Block 1 should exist");
assert_eq!(block.transactions.len(), 3, "Block should contain 3 txs");
// Verify each tx via RPC
for tx_hash in &tx_hashes {
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1), "All txs should be in block 1");
}
// Poll RocksDB for all tx hashes and collect tx_numbers
let mut tx_numbers = Vec::new();
for tx_hash in &tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify tx_numbers form the set {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be 0, 1, 2");
Ok(())
}
/// Transactions across multiple blocks have globally continuous `tx_numbers`.
#[tokio::test]
async fn test_rocksdb_txs_across_blocks() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Block 1: 2 transactions
let tx_hash_0 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await,
)
.await?;
let tx_hash_1 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await,
)
.await?;
// Wait for both txs to appear in pending pool
wait_for_pending_tx(&client, tx_hash_0).await;
wait_for_pending_tx(&client, tx_hash_1).await;
let payload1 = nodes[0].advance_block().await?;
assert_eq!(payload1.block().number(), 1);
// Block 2: 1 transaction
let tx_hash_2 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await,
)
.await?;
wait_for_pending_tx(&client, tx_hash_2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Verify block contents via RPC
let tx0: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_1]).await?;
let tx2: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_2]).await?;
assert_eq!(tx0.expect("tx0").block_number, Some(1));
assert_eq!(tx1.expect("tx1").block_number, Some(1));
assert_eq!(tx2.expect("tx2").block_number, Some(2));
// Poll RocksDB and verify global tx_number continuity
let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2];
let mut tx_numbers = Vec::new();
for tx_hash in &all_tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify they form a continuous sequence {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be globally continuous: 0, 1, 2");
// Re-query block 1 txs after block 2 is mined (regression guard)
let tx0_again: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
assert!(tx0_again.is_some(), "Block 1 tx should still be queryable after block 2");
Ok(())
}
/// Pending transactions should NOT appear in `RocksDB` until mined.
#[tokio::test]
async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
// Inject tx but do NOT mine
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Verify tx is in pending pool via RPC
let client = nodes[0].rpc_client().expect("RPC client");
wait_for_pending_tx(&client, tx_hash).await;
let pending_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert!(pending_tx.is_some(), "Pending tx should be visible via RPC");
assert!(pending_tx.unwrap().block_number.is_none(), "Pending tx should have no block_number");
// Assert tx is NOT in RocksDB before mining (single check - tx is confirmed pending)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let tx_number: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(tx_hash)?;
assert!(
tx_number.is_none(),
"Pending tx should NOT be in RocksDB before mining, but found tx_number={:?}",
tx_number
);
// Now mine the block
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Poll until tx appears in RocksDB
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have tx_number 0");
// Verify tx is now mined via RPC
let mined_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert_eq!(mined_tx.expect("mined tx").block_number, Some(1));
Ok(())
}
/// Reorg with `RocksDB`: verifies that unwind correctly reads changesets from
/// storage-aware locations (static files vs MDBX) rather than directly from MDBX.
///
/// This test exercises `unwind_trie_state_from` which previously failed with
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
#[tokio::test]
async fn test_rocksdb_reorg_unwind() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Use two separate wallets to avoid nonce conflicts during reorg
let wallets = wallet::Wallet::new(2).with_chain_id(chain_id).wallet_gen();
let signer1 = wallets[0].clone();
let signer2 = wallets[1].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Mine block 1 with a transaction from signer1
let raw_tx1 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await;
let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?;
wait_for_pending_tx(&client, tx_hash1).await;
let payload1 = nodes[0].advance_block().await?;
let block1_hash = payload1.block().hash();
assert_eq!(payload1.block().number(), 1);
// Poll until tx1 appears in RocksDB (ensures persistence happened)
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
// Mine block 2 with transaction from signer1 (nonce 1)
let raw_tx2 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await;
let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?;
wait_for_pending_tx(&client, tx_hash2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Poll until tx2 appears in RocksDB
let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
assert_eq!(tx_number2, 1, "Second tx should have tx_number 1");
// Mine block 3 with transaction from signer1 (nonce 2)
let raw_tx3 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await;
let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?;
wait_for_pending_tx(&client, tx_hash3).await;
let payload3 = nodes[0].advance_block().await?;
assert_eq!(payload3.block().number(), 3);
// Poll until tx3 appears in RocksDB
let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
assert_eq!(tx_number3, 2, "Third tx should have tx_number 2");
// Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict)
// Inject a tx from signer2 (nonce 0) before building the alternate block
let raw_alt_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await;
let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?;
wait_for_pending_tx(&client, alt_tx_hash).await;
// Build an alternate payload (this builds on top of the current head, i.e., block 3)
// But we want to reorg back to block 1, so we'll use the payload and then FCU to it
let alt_payload = nodes[0].new_payload().await?;
let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?;
// Trigger reorg: make the alternate chain canonical by sending FCU pointing to block 1's hash
// as finalized, which should trigger an unwind of blocks 2 and 3
// The alt block becomes the new head
nodes[0].update_forkchoice(block1_hash, alt_block_hash).await?;
// Give time for the reorg to complete
tokio::time::sleep(Duration::from_millis(500)).await;
// Verify we can still query transactions and the chain is consistent
// If unwind_trie_state_from failed, this would have errored during reorg
let latest: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("latest", false)).await?;
let latest = latest.expect("Latest block should exist");
// The alt block is at height 4 (on top of block 3)
assert!(latest.header.number >= 3, "Should be at height >= 3 after operation");
// tx1 from block 1 should still be there
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash1]).await?;
assert!(tx1.is_some(), "tx1 from block 1 should still be queryable");
assert_eq!(tx1.unwrap().block_number, Some(1));
// Mine another block to verify the chain can continue
let raw_tx_final =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await;
let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?;
wait_for_pending_tx(&client, tx_hash_final).await;
let final_payload = nodes[0].advance_block().await?;
assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg");
// Verify tx_final is included
let tx_final: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_final]).await?;
assert!(tx_final.is_some(), "final tx should be in latest block");
Ok(())
}

View File

@@ -32,7 +32,9 @@ futures-util.workspace = true
# misc
eyre.workspace = true
tracing.workspace = true
op-alloy-rpc-types-engine = { workspace = true, optional = true }
reth-optimism-chainspec = { workspace = true, optional = true }
[lints]
workspace = true
@@ -40,6 +42,7 @@ workspace = true
[features]
op = [
"dep:op-alloy-rpc-types-engine",
"dep:reth-optimism-chainspec",
"reth-payload-primitives/op",
"reth-primitives-traits/op",
]

View File

@@ -72,18 +72,13 @@ where
&self,
parent: &SealedHeader<ChainSpec::Header>,
) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
/// Dummy system transaction for dev mode.
/// OP Mainnet transaction at index 0 in block 124665056.
///
/// <https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1>
const TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056: [u8; 251] = alloy_primitives::hex!(
"7ef8f8a0683079df94aa5b9cf86687d739a60a9b4f0835e520ec4d664e2e415dca17a6df94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985"
);
op_alloy_rpc_types_engine::OpPayloadAttributes {
payload_attributes: self.build(parent),
// Add dummy system transaction
transactions: Some(vec![TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056.into()]),
transactions: Some(vec![
reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056
.into(),
]),
no_tx_pool: None,
gas_limit: None,
eip_1559_params: None,

View File

@@ -34,11 +34,6 @@ fn default_account_worker_count() -> usize {
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
/// computation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
/// Default number of reserved CPU cores for non-reth processes.
///
/// This will be deducted from the thread count of main reth global threadpool.
@@ -50,17 +45,7 @@ pub const DEFAULT_PREWARM_MAX_CONCURRENCY: usize = 16;
const DEFAULT_BLOCK_BUFFER_LIMIT: u32 = EPOCH_SLOTS as u32 * 2;
const DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH: u32 = 256;
const DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE: usize = 4;
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: usize = default_cross_block_cache_size();
const fn default_cross_block_cache_size() -> usize {
if cfg!(test) {
1024 * 1024 // 1 MB in tests
} else if cfg!(target_pointer_width = "32") {
usize::MAX // max possible on wasm32 / 32-bit
} else {
4 * 1024 * 1024 * 1024 // 4 GB on 64-bit
}
}
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: u64 = 4 * 1024 * 1024 * 1024;
/// Determines if the host has enough parallelism to run the payload processor.
///
@@ -110,10 +95,12 @@ pub struct TreeConfig {
disable_state_cache: bool,
/// Whether to disable parallel prewarming.
disable_prewarming: bool,
/// Whether to disable the parallel sparse trie state root algorithm.
disable_parallel_sparse_trie: bool,
/// Whether to enable state provider metrics.
state_provider_metrics: bool,
/// Cross-block cache size in bytes.
cross_block_cache_size: usize,
cross_block_cache_size: u64,
/// Whether the host has enough parallelism to run state root task.
has_enough_parallelism: bool,
/// Whether multiproof task should chunk proof targets.
@@ -148,12 +135,10 @@ pub struct TreeConfig {
storage_worker_count: usize,
/// Number of account proof worker threads.
account_worker_count: usize,
/// Whether to disable V2 storage proofs.
disable_proof_v2: bool,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
/// Whether to enable sparse trie as cache.
enable_sparse_trie_as_cache: bool,
}
impl Default for TreeConfig {
@@ -168,6 +153,7 @@ impl Default for TreeConfig {
always_compare_trie_updates: false,
disable_state_cache: false,
disable_prewarming: false,
disable_parallel_sparse_trie: false,
state_provider_metrics: false,
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE,
has_enough_parallelism: has_enough_parallelism(),
@@ -181,9 +167,8 @@ impl Default for TreeConfig {
allow_unwind_canonical_header: false,
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
disable_proof_v2: false,
enable_proof_v2: false,
disable_cache_metrics: false,
enable_sparse_trie_as_cache: false,
}
}
}
@@ -201,8 +186,9 @@ impl TreeConfig {
always_compare_trie_updates: bool,
disable_state_cache: bool,
disable_prewarming: bool,
disable_parallel_sparse_trie: bool,
state_provider_metrics: bool,
cross_block_cache_size: usize,
cross_block_cache_size: u64,
has_enough_parallelism: bool,
multiproof_chunking_enabled: bool,
multiproof_chunk_size: usize,
@@ -214,7 +200,7 @@ impl TreeConfig {
allow_unwind_canonical_header: bool,
storage_worker_count: usize,
account_worker_count: usize,
disable_proof_v2: bool,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
@@ -227,6 +213,7 @@ impl TreeConfig {
always_compare_trie_updates,
disable_state_cache,
disable_prewarming,
disable_parallel_sparse_trie,
state_provider_metrics,
cross_block_cache_size,
has_enough_parallelism,
@@ -240,9 +227,8 @@ impl TreeConfig {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
disable_proof_v2,
enable_proof_v2,
disable_cache_metrics,
enable_sparse_trie_as_cache: false,
}
}
@@ -281,18 +267,6 @@ impl TreeConfig {
self.multiproof_chunk_size
}
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if !self.disable_proof_v2 &&
self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
self.multiproof_chunk_size
}
}
/// Return the number of reserved CPU cores for non-reth processes
pub const fn reserved_cpu_cores(&self) -> usize {
self.reserved_cpu_cores
@@ -309,6 +283,11 @@ impl TreeConfig {
self.state_provider_metrics
}
/// Returns whether or not the parallel sparse trie is disabled.
pub const fn disable_parallel_sparse_trie(&self) -> bool {
self.disable_parallel_sparse_trie
}
/// Returns whether or not state cache is disabled.
pub const fn disable_state_cache(&self) -> bool {
self.disable_state_cache
@@ -326,7 +305,7 @@ impl TreeConfig {
}
/// Returns the cross-block cache size.
pub const fn cross_block_cache_size(&self) -> usize {
pub const fn cross_block_cache_size(&self) -> u64 {
self.cross_block_cache_size
}
@@ -429,7 +408,7 @@ impl TreeConfig {
}
/// Setter for cross block cache size.
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: usize) -> Self {
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: u64) -> Self {
self.cross_block_cache_size = cross_block_cache_size;
self
}
@@ -446,6 +425,15 @@ impl TreeConfig {
self
}
/// Setter for whether to disable the parallel sparse trie
pub const fn with_disable_parallel_sparse_trie(
mut self,
disable_parallel_sparse_trie: bool,
) -> Self {
self.disable_parallel_sparse_trie = disable_parallel_sparse_trie;
self
}
/// Setter for whether multiproof task should chunk proof targets.
pub const fn with_multiproof_chunking_enabled(
mut self,
@@ -523,14 +511,14 @@ impl TreeConfig {
self
}
/// Return whether V2 storage proofs are disabled.
pub const fn disable_proof_v2(&self) -> bool {
self.disable_proof_v2
/// Return whether V2 storage proofs are enabled.
pub const fn enable_proof_v2(&self) -> bool {
self.enable_proof_v2
}
/// Setter for whether to disable V2 storage proofs.
pub const fn with_disable_proof_v2(mut self, disable_proof_v2: bool) -> Self {
self.disable_proof_v2 = disable_proof_v2;
/// Setter for whether to enable V2 storage proofs.
pub const fn with_enable_proof_v2(mut self, enable_proof_v2: bool) -> Self {
self.enable_proof_v2 = enable_proof_v2;
self
}
@@ -544,15 +532,4 @@ impl TreeConfig {
self.disable_cache_metrics = disable_cache_metrics;
self
}
/// Returns whether sparse trie as cache is enabled.
pub const fn enable_sparse_trie_as_cache(&self) -> bool {
self.enable_sparse_trie_as_cache
}
/// Setter for whether to enable sparse trie as cache.
pub const fn with_enable_sparse_trie_as_cache(mut self, value: bool) -> Self {
self.enable_sparse_trie_as_cache = value;
self
}
}

View File

@@ -62,8 +62,7 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
> + DeserializeOwned
+ Serialize
{
@@ -107,14 +106,6 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -101,7 +101,7 @@ where
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
let (to_tree_tx, from_tree) = EngineApiTreeHandler::<N::Primitives, _, _, _, _>::spawn_new(
blockchain_db,
consensus,
payload_validator,

View File

@@ -53,7 +53,7 @@ revm-primitives.workspace = true
futures.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
fixed-cache.workspace = true
mini-moka = { workspace = true, features = ["sync"] }
moka = { workspace = true, features = ["sync"] }
smallvec.workspace = true

File diff suppressed because it is too large Load Diff

View File

@@ -22,9 +22,6 @@ pub(crate) struct EngineApiMetrics {
pub(crate) block_validation: BlockValidationMetrics,
/// Canonical chain and reorg related metrics
pub tree: TreeMetrics,
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
#[allow(dead_code)]
pub(crate) bal: BalMetrics,
}
impl EngineApiMetrics {
@@ -242,8 +239,6 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_error: Counter,
/// The total gas of valid new payload messages received.
pub(crate) new_payload_total_gas: Histogram,
/// The gas used for the last valid new payload.
pub(crate) new_payload_total_gas_last: Gauge,
/// The gas per second of valid new payload messages received.
pub(crate) new_payload_gas_per_second: Histogram,
/// The gas per second for the last new payload call.
@@ -256,8 +251,6 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) time_between_new_payloads: Histogram,
/// Time from previous payload start to current payload start (total interval).
pub(crate) new_payload_interval: Histogram,
/// Time diff between forkchoice updated call response and the next new payload call request.
pub(crate) forkchoice_updated_new_payload_time_diff: Histogram,
}
impl NewPayloadStatusMetrics {
@@ -265,7 +258,6 @@ impl NewPayloadStatusMetrics {
pub(crate) fn update_response_metrics(
&mut self,
start: Instant,
latest_forkchoice_updated_at: &mut Option<Instant>,
result: &Result<TreeOutcome<PayloadStatus>, InsertBlockFatalError>,
gas_used: u64,
) {
@@ -285,7 +277,6 @@ impl NewPayloadStatusMetrics {
PayloadStatusEnum::Valid => {
self.new_payload_valid.increment(1);
self.new_payload_total_gas.record(gas_used as f64);
self.new_payload_total_gas_last.set(gas_used as f64);
let gas_per_second = gas_used as f64 / elapsed.as_secs_f64();
self.new_payload_gas_per_second.record(gas_per_second);
self.new_payload_gas_per_second_last.set(gas_per_second);
@@ -299,40 +290,9 @@ impl NewPayloadStatusMetrics {
self.new_payload_messages.increment(1);
self.new_payload_latency.record(elapsed);
self.new_payload_last.set(elapsed);
if let Some(latest_forkchoice_updated_at) = latest_forkchoice_updated_at.take() {
self.forkchoice_updated_new_payload_time_diff
.record(start - latest_forkchoice_updated_at);
}
}
}
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
///
/// See also <https://github.com/ethereum/execution-metrics/issues/5>
#[allow(dead_code)]
#[derive(Metrics, Clone)]
#[metrics(scope = "execution.block_access_list")]
pub(crate) struct BalMetrics {
/// Size of the BAL in bytes for the current block.
pub(crate) size_bytes: Gauge,
/// Total number of blocks with valid BALs.
pub(crate) valid_total: Counter,
/// Total number of blocks with invalid BALs.
pub(crate) invalid_total: Counter,
/// Time taken to validate the BAL against actual execution.
pub(crate) validation_time_seconds: Histogram,
/// Number of account changes in the BAL.
pub(crate) account_changes: Gauge,
/// Number of storage changes in the BAL.
pub(crate) storage_changes: Gauge,
/// Number of balance changes in the BAL.
pub(crate) balance_changes: Gauge,
/// Number of nonce changes in the BAL.
pub(crate) nonce_changes: Gauge,
/// Number of code changes in the BAL.
pub(crate) code_changes: Gauge,
}
/// Metrics for non-execution related block validation.
#[derive(Metrics, Clone)]
#[metrics(scope = "sync.block_validation")]
@@ -341,8 +301,6 @@ pub(crate) struct BlockValidationMetrics {
pub(crate) state_root_storage_tries_updated_total: Counter,
/// Total number of times the parallel state root computation fell back to regular.
pub(crate) state_root_parallel_fallback_total: Counter,
/// Total number of times the state root task failed but the fallback succeeded.
pub(crate) state_root_task_fallback_success_total: Counter,
/// Latest state root duration, ie the time spent blocked waiting for the state root.
pub(crate) state_root_duration: Gauge,
/// Histogram for state root duration ie the time spent blocked waiting for the state root

View File

@@ -32,8 +32,7 @@ use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, Sealed
use reth_provider::{
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
TransactionVariant,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
@@ -85,12 +84,6 @@ pub mod state;
/// backfill this gap.
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
/// The minimum number of blocks to retain in the changeset cache after eviction.
///
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
/// even when the finalized block is not set (e.g., on L2s like Optimism).
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
/// A builder for creating state providers that can be used across threads.
#[derive(Clone, Debug)]
pub struct StateProviderBuilder<N: NodePrimitives, P> {
@@ -324,7 +317,6 @@ where
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
@@ -1384,42 +1376,21 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the eviction threshold.
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
// the finalized block if set.
let min_threshold =
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
let eviction_threshold =
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
// Use the minimum of finalized block and retention threshold to be conservative
finalized.number.min(min_threshold)
} else {
// When finalized is not set (e.g., on L2s), use the retention threshold
min_threshold
};
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
// Invalidate cached overlay since the anchor has changed
self.state.tree_state.invalidate_cached_overlay();
self.on_new_persisted_block()?;
// Re-prepare overlay for the current canonical head with the new anchor.
// Spawn a background task to trigger computation so it's ready when the next payload
// arrives.
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
rayon::spawn(move || {
let _ = overlay.get();
});
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
self.on_new_persisted_block()?;
Ok(())
}
@@ -1505,10 +1476,6 @@ where
self.on_maybe_tree_event(res.event.take())?;
}
if let Err(ref err) = output {
error!(target: "engine::tree", %err, ?state, "Error processing forkchoice update");
}
self.metrics.engine.forkchoice_updated.update_response_metrics(
start,
&mut self.metrics.engine.new_payload.latest_finish_at,
@@ -1531,12 +1498,10 @@ where
let gas_used = payload.gas_used();
let num_hash = payload.num_hash();
let mut output = self.on_new_payload(payload);
self.metrics.engine.new_payload.update_response_metrics(
start,
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
&output,
gas_used,
);
self.metrics
.engine
.new_payload
.update_response_metrics(start, &output, gas_used);
let maybe_event =
output.as_mut().ok().and_then(|out| out.event.take());

View File

@@ -0,0 +1,188 @@
//! Configured sparse trie enum for switching between serial and parallel implementations.
use alloy_primitives::B256;
use reth_trie::{BranchNodeMasks, Nibbles, ProofTrieNode, TrieNode};
use reth_trie_sparse::{
errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError,
SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates,
};
use reth_trie_sparse_parallel::ParallelSparseTrie;
use std::borrow::Cow;
/// Enum for switching between serial and parallel sparse trie implementations.
///
/// This type allows runtime selection between different sparse trie implementations,
/// providing flexibility in choosing the appropriate implementation based on workload
/// characteristics.
#[derive(Debug, Clone)]
pub(crate) enum ConfiguredSparseTrie {
/// Serial implementation of the sparse trie.
Serial(Box<SerialSparseTrie>),
/// Parallel implementation of the sparse trie.
Parallel(Box<ParallelSparseTrie>),
}
impl From<SerialSparseTrie> for ConfiguredSparseTrie {
fn from(trie: SerialSparseTrie) -> Self {
Self::Serial(Box::new(trie))
}
}
impl From<ParallelSparseTrie> for ConfiguredSparseTrie {
fn from(trie: ParallelSparseTrie) -> Self {
Self::Parallel(Box::new(trie))
}
}
impl Default for ConfiguredSparseTrie {
fn default() -> Self {
Self::Serial(Default::default())
}
}
impl SparseTrieInterface for ConfiguredSparseTrie {
fn with_root(
self,
root: TrieNode,
masks: Option<BranchNodeMasks>,
retain_updates: bool,
) -> SparseTrieResult<Self> {
match self {
Self::Serial(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Serial(Box::new(t)))
}
Self::Parallel(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Parallel(Box::new(t)))
}
}
}
fn with_updates(self, retain_updates: bool) -> Self {
match self {
Self::Serial(trie) => Self::Serial(Box::new(trie.with_updates(retain_updates))),
Self::Parallel(trie) => Self::Parallel(Box::new(trie.with_updates(retain_updates))),
}
}
fn reserve_nodes(&mut self, additional: usize) {
match self {
Self::Serial(trie) => trie.reserve_nodes(additional),
Self::Parallel(trie) => trie.reserve_nodes(additional),
}
}
fn reveal_node(
&mut self,
path: Nibbles,
node: TrieNode,
masks: Option<BranchNodeMasks>,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_node(path, node, masks),
Self::Parallel(trie) => trie.reveal_node(path, node, masks),
}
}
fn reveal_nodes(&mut self, nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_nodes(nodes),
Self::Parallel(trie) => trie.reveal_nodes(nodes),
}
}
fn update_leaf<P: TrieNodeProvider>(
&mut self,
full_path: Nibbles,
value: Vec<u8>,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.update_leaf(full_path, value, provider),
Self::Parallel(trie) => trie.update_leaf(full_path, value, provider),
}
}
fn remove_leaf<P: TrieNodeProvider>(
&mut self,
full_path: &Nibbles,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.remove_leaf(full_path, provider),
Self::Parallel(trie) => trie.remove_leaf(full_path, provider),
}
}
fn root(&mut self) -> B256 {
match self {
Self::Serial(trie) => trie.root(),
Self::Parallel(trie) => trie.root(),
}
}
fn update_subtrie_hashes(&mut self) {
match self {
Self::Serial(trie) => trie.update_subtrie_hashes(),
Self::Parallel(trie) => trie.update_subtrie_hashes(),
}
}
fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec<u8>> {
match self {
Self::Serial(trie) => trie.get_leaf_value(full_path),
Self::Parallel(trie) => trie.get_leaf_value(full_path),
}
}
fn find_leaf(
&self,
full_path: &Nibbles,
expected_value: Option<&Vec<u8>>,
) -> Result<LeafLookup, LeafLookupError> {
match self {
Self::Serial(trie) => trie.find_leaf(full_path, expected_value),
Self::Parallel(trie) => trie.find_leaf(full_path, expected_value),
}
}
fn take_updates(&mut self) -> SparseTrieUpdates {
match self {
Self::Serial(trie) => trie.take_updates(),
Self::Parallel(trie) => trie.take_updates(),
}
}
fn wipe(&mut self) {
match self {
Self::Serial(trie) => trie.wipe(),
Self::Parallel(trie) => trie.wipe(),
}
}
fn clear(&mut self) {
match self {
Self::Serial(trie) => trie.clear(),
Self::Parallel(trie) => trie.clear(),
}
}
fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> {
match self {
Self::Serial(trie) => trie.updates_ref(),
Self::Parallel(trie) => trie.updates_ref(),
}
}
fn shrink_nodes_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_nodes_to(size),
Self::Parallel(trie) => trie.shrink_nodes_to(size),
}
}
fn shrink_values_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_values_to(size),
Self::Parallel(trie) => trie.shrink_values_to(size),
}
}
}

View File

@@ -2,19 +2,22 @@
use super::precompile_cache::PrecompileCacheMap;
use crate::tree::{
cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache},
cached_state::{
CachedStateMetrics, CachedStateProvider, ExecutionCache as StateExecutionCache,
ExecutionCacheBuilder, SavedCache,
},
payload_processor::{
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
sparse_trie::StateRootComputeOutcome,
},
sparse_trie::{SparseTrieCacheTask, SparseTrieTask},
sparse_trie::SparseTrieTask,
StateProviderBuilder, TreeConfig,
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::block::StateChangeSource;
use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_primitives::B256;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
@@ -22,7 +25,6 @@ use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
use rayon::prelude::*;
use reth_evm::{
block::ExecutableTxParts,
execute::{ExecutableTxFor, WithTxEnv},
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
@@ -39,7 +41,10 @@ use reth_trie_parallel::{
proof_task::{ProofTaskCtx, ProofWorkerHandle},
root::ParallelStateRootError,
};
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse::{
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, SparseStateTrie, SparseTrie,
};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
collections::BTreeMap,
@@ -54,12 +59,15 @@ use std::{
use tracing::{debug, debug_span, instrument, warn, Span};
pub mod bal;
mod configured_sparse_trie;
pub mod executor;
pub mod multiproof;
pub mod prewarm;
pub mod receipt_root_task;
pub mod sparse_trie;
use configured_sparse_trie::ConfiguredSparseTrie;
/// Default parallelism thresholds to use with the [`ParallelSparseTrie`].
///
/// These values were determined by performing benchmarks using gradually increasing values to judge
@@ -93,7 +101,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
<I as ExecutableTxTuple>::Error,
<N as NodePrimitives>::Receipt,
>;
@@ -107,11 +115,11 @@ where
/// The executor used by to spawn tasks.
executor: WorkloadExecutor,
/// The most recent cache used for execution.
execution_cache: PayloadExecutionCache,
execution_cache: ExecutionCache,
/// Metrics for trie operations
trie_metrics: MultiProofTaskMetrics,
/// Cross-block cache size in bytes.
cross_block_cache_size: usize,
cross_block_cache_size: u64,
/// Whether transactions should not be executed on prewarming task.
disable_transaction_prewarming: bool,
/// Whether state cache should be disable
@@ -125,8 +133,12 @@ where
/// A cleared `SparseStateTrie`, kept around to be reused for the state root computation so
/// that allocations can be minimized.
sparse_state_trie: Arc<
parking_lot::Mutex<Option<ClearedSparseStateTrie<ParallelSparseTrie, ParallelSparseTrie>>>,
parking_lot::Mutex<
Option<ClearedSparseStateTrie<ConfiguredSparseTrie, ConfiguredSparseTrie>>,
>,
>,
/// Whether to disable the parallel sparse trie.
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
@@ -161,6 +173,7 @@ where
precompile_cache_disabled: config.precompile_cache_disabled(),
precompile_cache_map,
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
@@ -234,9 +247,6 @@ where
let (to_sparse_trie, sparse_trie_rx) = channel();
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = !config.disable_proof_v2();
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
// When BAL is present, use BAL prewarming and send BAL to multiproof
@@ -253,7 +263,6 @@ where
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
v2_proofs_enabled,
)
} else {
// Normal path: spawn with transaction prewarming
@@ -264,7 +273,6 @@ where
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
v2_proofs_enabled,
)
};
@@ -272,6 +280,7 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
@@ -280,45 +289,36 @@ where
v2_proofs_enabled,
);
if !config.enable_sparse_trie_as_cache() {
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof.clone(),
)
.with_v2_proofs_enabled(v2_proofs_enabled);
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
);
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _disable_metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
}
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
// wire the sparse trie to the state root response receiver
let (state_root_tx, state_root_rx) = channel();
// Spawn the sparse trie task using any stored trie and parallel trie configuration.
self.spawn_sparse_trie_task(
sparse_trie_rx,
proof_handle,
state_root_tx,
from_multi_proof,
config,
);
self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx);
PayloadHandle {
to_multi_proof: Some(to_multi_proof),
@@ -344,9 +344,8 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -362,8 +361,8 @@ where
&self,
transactions: I,
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
@@ -378,10 +377,7 @@ where
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
WithTxEnv { tx_env, tx: Arc::new(tx) }
});
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
// Only send Ok(_) variants to prewarming task.
if let Ok(tx) = &tx {
let _ = prewarm_tx.send(tx.clone());
@@ -416,7 +412,6 @@ where
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
@@ -425,7 +420,6 @@ where
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
v2_proofs_enabled: bool,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -448,7 +442,6 @@ where
terminate_execution: Arc::new(AtomicBool::new(false)),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
v2_proofs_enabled,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
@@ -487,29 +480,28 @@ where
cache
} else {
debug!("creating new execution cache on cache miss");
let start = Instant::now();
let cache = ExecutionCache::new(self.cross_block_cache_size);
let metrics = CachedStateMetrics::zeroed();
metrics.record_cache_creation(start.elapsed());
SavedCache::new(parent_hash, cache, metrics)
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
/// Spawns the [`SparseTrieTask`] for this payload processor.
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
fn spawn_sparse_trie_task(
fn spawn_sparse_trie_task<BPF>(
&self,
sparse_trie_rx: mpsc::Receiver<SparseTrieUpdate>,
proof_worker_handle: ProofWorkerHandle,
proof_worker_handle: BPF,
state_root_tx: mpsc::Sender<Result<StateRootComputeOutcome, ParallelStateRootError>>,
from_multi_proof: CrossbeamReceiver<MultiProofMessage>,
config: &TreeConfig,
) {
) where
BPF: TrieNodeProviderFactory + Clone + Send + Sync + 'static,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
{
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
let trie_metrics = self.trie_metrics.clone();
let span = Span::current();
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
self.executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -517,10 +509,14 @@ where
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
// if there's none to reuse.
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
let default_trie = RevealableSparseTrie::blind_from(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
);
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
ConfiguredSparseTrie::Serial(Default::default())
} else {
ConfiguredSparseTrie::Parallel(Box::new(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
))
});
ClearedSparseStateTrie::from_state_trie(
SparseStateTrie::new()
.with_accounts_trie(default_trie.clone())
@@ -529,24 +525,14 @@ where
)
});
let (result, trie) = if disable_sparse_trie_as_cache {
SparseTrieTask::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
)
.run()
} else {
SparseTrieCacheTask::new_with_cleared_trie(
from_multi_proof,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
)
.run()
};
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
);
let (result, trie) = task.run();
// Send state root computation result
let _ = state_root_tx.send(result);
@@ -586,27 +572,28 @@ where
parent_hash = %block_with_parent.parent,
"Cannot find cache for parent hash, skip updating cache with new state for inserted executed block",
);
return
return;
}
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics, _) = match cached.take() {
Some(existing) => existing.split(),
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
let (c, m, _) = existing.split();
(c, m)
}
None => (
ExecutionCache::new(self.cross_block_cache_size),
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
CachedStateMetrics::zeroed(),
false,
),
};
// Insert the block's bundle state into cache
let new_cache =
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
return
return;
}
new_cache.update_metrics();
@@ -670,7 +657,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
}
/// Returns a clone of the caches used by prewarming
pub(super) fn caches(&self) -> Option<ExecutionCache> {
pub(super) fn caches(&self) -> Option<StateExecutionCache> {
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
}
@@ -774,29 +761,29 @@ impl<R> Drop for CacheTaskHandle<R> {
/// ## Cache Safety
///
/// **CRITICAL**: Cache update operations require exclusive access. All concurrent cache users
/// (such as prewarming tasks) must be terminated before calling
/// [`PayloadExecutionCache::update_with_guard`], otherwise the cache may be corrupted or cleared.
/// (such as prewarming tasks) must be terminated before calling `update_with_guard`, otherwise
/// the cache may be corrupted or cleared.
///
/// ## Cache vs Prewarming Distinction
///
/// **[`PayloadExecutionCache`]**:
/// **`ExecutionCache`**:
/// - Stores parent block's execution state after completion
/// - Used to fetch parent data for next block's execution
/// - Must be exclusively accessed during save operations
///
/// **[`PrewarmCacheTask`]**:
/// **`PrewarmCacheTask`**:
/// - Speculatively loads accounts/storage that might be used in transaction execution
/// - Prepares data for state root proof computation
/// - Runs concurrently but must not interfere with cache saves
#[derive(Clone, Debug, Default)]
struct PayloadExecutionCache {
struct ExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl PayloadExecutionCache {
impl ExecutionCache {
/// Returns the cache for `parent_hash` if it's available for use.
///
/// A cache is considered available when:
@@ -832,15 +819,11 @@ impl PayloadExecutionCache {
"Existing cache found"
);
if available {
// If the has is available (no other threads are using it), but has a mismatching
// parent hash, we can just clear it and keep using without re-creating from
// scratch.
if !hash_matches {
c.clear();
}
return Some(c.clone())
} else if hash_matches {
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
@@ -913,9 +896,9 @@ where
#[cfg(test)]
mod tests {
use super::PayloadExecutionCache;
use super::ExecutionCache;
use crate::tree::{
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
cached_state::{CachedStateMetrics, ExecutionCacheBuilder, SavedCache},
payload_processor::{
evm_state_to_hashed_post_state, executor::WorkloadExecutor, PayloadProcessor,
},
@@ -945,13 +928,13 @@ mod tests {
use std::sync::Arc;
fn make_saved_cache(hash: B256) -> SavedCache {
let execution_cache = ExecutionCache::new(1_000);
let execution_cache = ExecutionCacheBuilder::default().build_caches(1_000);
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
}
#[test]
fn execution_cache_allows_single_checkout() {
let execution_cache = PayloadExecutionCache::default();
let execution_cache = ExecutionCache::default();
let hash = B256::from([1u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -970,7 +953,7 @@ mod tests {
#[test]
fn execution_cache_checkout_releases_on_drop() {
let execution_cache = PayloadExecutionCache::default();
let execution_cache = ExecutionCache::default();
let hash = B256::from([2u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -986,21 +969,19 @@ mod tests {
}
#[test]
fn execution_cache_mismatch_parent_clears_and_returns() {
let execution_cache = PayloadExecutionCache::default();
fn execution_cache_mismatch_parent_returns_none() {
let execution_cache = ExecutionCache::default();
let hash = B256::from([3u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
// When the parent hash doesn't match, the cache is cleared and returned for reuse
let different_hash = B256::from([4u8; 32]);
let cache = execution_cache.get_cache_for(different_hash);
assert!(cache.is_some(), "cache should be returned for reuse after clearing")
let miss = execution_cache.get_cache_for(B256::from([4u8; 32]));
assert!(miss.is_none(), "checkout should fail for different parent hash");
}
#[test]
fn execution_cache_update_after_release_succeeds() {
let execution_cache = PayloadExecutionCache::default();
let execution_cache = ExecutionCache::default();
let initial = B256::from([5u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(initial)));

View File

@@ -11,18 +11,14 @@ use reth_metrics::Metrics;
use reth_provider::AccountReader;
use reth_revm::state::EvmState;
use reth_trie::{
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
MultiProofTargets,
};
#[cfg(test)]
use reth_trie_parallel::stats::ParallelTrieTracker;
use reth_trie_parallel::{
proof::ParallelProof,
proof_task::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
@@ -67,12 +63,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
#[derive(Debug)]
#[derive(Default, Debug)]
pub struct SparseTrieUpdate {
/// The state update that was used to calculate the proof
pub(crate) state: HashedPostState,
/// The calculated multiproof
pub(crate) multiproof: ProofResult,
pub(crate) multiproof: DecodedMultiProof,
}
impl SparseTrieUpdate {
@@ -84,11 +80,7 @@ impl SparseTrieUpdate {
/// Construct update from multiproof.
#[cfg(test)]
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
let stats = ParallelTrieTracker::default().finish();
Ok(Self {
state: HashedPostState::default(),
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
})
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
}
/// Extend update with contents of the other.
@@ -102,7 +94,7 @@ impl SparseTrieUpdate {
#[derive(Debug)]
pub(super) enum MultiProofMessage {
/// Prefetch proof targets
PrefetchProofs(VersionedMultiProofTargets),
PrefetchProofs(MultiProofTargets),
/// New state update from transaction execution with its source
StateUpdate(Source, EvmState),
/// State update that can be applied to the sparse trie without any new proofs.
@@ -231,155 +223,12 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
hashed_state
}
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
/// regardless of which variant the latter is.
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
match src {
VersionedMultiProofTargets::Legacy(targets) => {
dest.extend_ref(targets);
}
VersionedMultiProofTargets::V2(targets) => {
// Add all account targets
for target in &targets.account_targets {
dest.entry(target.key()).or_default();
}
// Add all storage targets
for (hashed_address, slots) in &targets.storage_targets {
let slot_set = dest.entry(*hashed_address).or_default();
for slot in slots {
slot_set.insert(slot.key());
}
}
}
}
}
/// A set of multiproof targets which can be either in the legacy or V2 representations.
#[derive(Debug)]
pub(super) enum VersionedMultiProofTargets {
/// Legacy targets
Legacy(MultiProofTargets),
/// V2 targets
V2(MultiProofTargetsV2),
}
impl VersionedMultiProofTargets {
/// Returns true if there are no account or storage targets.
fn is_empty(&self) -> bool {
match self {
Self::Legacy(targets) => targets.is_empty(),
Self::V2(targets) => targets.is_empty(),
}
}
/// Returns the number of account targets in the multiproof target
fn account_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the number of storage targets in the multiproof target
fn storage_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
Self::V2(targets) => {
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Returns the number of accounts in the multiproof targets.
fn len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the total storage slot count across all accounts.
fn storage_count(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
}
}
/// Returns the number of items that will be considered during chunking.
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Retains the targets representing the difference with another `MultiProofTargets`.
/// Removes all targets that are already present in `other`.
fn retain_difference(&mut self, other: &MultiProofTargets) {
match self {
Self::Legacy(targets) => {
targets.retain_difference(other);
}
Self::V2(targets) => {
// Remove account targets that exist in other
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
// For each account in storage_targets, remove slots that exist in other
targets.storage_targets.retain(|hashed_address, slots| {
if let Some(other_slots) = other.get(hashed_address) {
slots.retain(|slot| !other_slots.contains(&slot.key()));
!slots.is_empty()
} else {
true
}
});
}
}
}
/// Extends this `VersionedMultiProofTargets` with the contents of another.
///
/// Panics if the variants do not match.
fn extend(&mut self, other: Self) {
match (self, other) {
(Self::Legacy(dest), Self::Legacy(src)) => {
dest.extend(src);
}
(Self::V2(dest), Self::V2(src)) => {
dest.account_targets.extend(src.account_targets);
for (addr, slots) in src.storage_targets {
dest.storage_targets.entry(addr).or_default().extend(slots);
}
}
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
}
}
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
match self {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
}
}
}
/// Input parameters for dispatching a multiproof calculation.
#[derive(Debug)]
struct MultiproofInput {
source: Option<Source>,
hashed_state_update: HashedPostState,
proof_targets: VersionedMultiProofTargets,
proof_targets: MultiProofTargets,
proof_sequence_number: u64,
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
@@ -414,6 +263,8 @@ pub struct MultiproofManager {
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Metrics
metrics: MultiProofTaskMetrics,
/// Whether to use V2 storage proofs
v2_proofs_enabled: bool,
}
impl MultiproofManager {
@@ -427,7 +278,9 @@ impl MultiproofManager {
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
Self { metrics, proof_worker_handle, proof_result_tx }
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
}
/// Dispatches a new multiproof calculation to worker pools.
@@ -472,48 +325,41 @@ impl MultiproofManager {
multi_added_removed_keys,
} = multiproof_input;
let account_targets = proof_targets.len();
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
trace!(
target: "engine::tree::payload_processor::multiproof",
proof_sequence_number,
?proof_targets,
account_targets = proof_targets.account_targets_len(),
storage_targets = proof_targets.storage_targets_len(),
account_targets,
storage_targets,
?source,
"Dispatching multiproof to workers"
);
let start = Instant::now();
// Workers will send ProofResultMessage directly to proof_result_rx
let proof_result_sender = ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
);
let input = match proof_targets {
VersionedMultiProofTargets::Legacy(proof_targets) => {
// Extend prefix sets with targets
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
&Default::default(),
&proof_targets,
);
AccountMultiproofInput::Legacy {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
proof_result_sender,
}
}
VersionedMultiProofTargets::V2(proof_targets) => {
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
}
};
// Extend prefix sets with targets
let frozen_prefix_sets =
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
// Dispatch account multiproof to worker pool with result sender
let input = AccountMultiproofInput {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
// Workers will send ProofResultMessage directly to proof_result_rx
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
),
v2_proofs_enabled: self.v2_proofs_enabled,
};
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
return;
@@ -715,9 +561,6 @@ pub(super) struct MultiProofTask {
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
/// will be produced by state updates.
v2_proofs_enabled: bool,
}
impl MultiProofTask {
@@ -749,16 +592,9 @@ impl MultiProofTask {
),
metrics,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
v2_proofs_enabled: false,
}
}
/// Enables V2 proof target generation on state updates.
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
self.v2_proofs_enabled = v2_proofs_enabled;
self
}
/// Handles request for proof prefetch.
///
/// Returns how many multiproof tasks were dispatched for the prefetch request.
@@ -766,40 +602,37 @@ impl MultiProofTask {
level = "debug",
target = "engine::tree::payload_processor::multiproof",
skip_all,
fields(accounts = targets.account_targets_len(), chunks = 0)
fields(accounts = targets.len(), chunks = 0)
)]
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
// Remove already fetched proof targets to avoid redundant work.
targets.retain_difference(&self.fetched_proof_targets);
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
self.fetched_proof_targets.extend_ref(&targets);
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
// Make sure all target accounts have an `AddedRemovedKeySet` in the
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
// we still want to optimistically fetch extension children for the leaf addition case.
// V2 multiproofs don't need this.
//
// Only clone the AddedRemovedKeys for accounts in the targets, not the entire accumulated
// set, to avoid O(n) cloning with many buffered blocks.
let multi_added_removed_keys =
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
Some(Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: legacy_targets
.keys()
.filter_map(|k| {
self.multi_added_removed_keys.storages.get(k).map(|v| (*k, v.clone()))
})
.collect(),
}))
} else {
None
};
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: targets
.keys()
.filter_map(|account| {
self.multi_added_removed_keys
.storages
.get(account)
.cloned()
.map(|keys| (*account, keys))
})
.collect(),
});
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
self.metrics
.prefetch_proof_targets_storages_histogram
.record(targets.storage_count() as f64);
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
let chunking_len = targets.chunking_length();
let available_account_workers =
@@ -813,7 +646,7 @@ impl MultiProofTask {
self.max_targets_for_chunking,
available_account_workers,
available_storage_workers,
VersionedMultiProofTargets::chunks,
MultiProofTargets::chunks,
|proof_targets| {
self.multiproof_manager.dispatch(MultiproofInput {
source: None,
@@ -821,7 +654,7 @@ impl MultiProofTask {
proof_targets,
proof_sequence_number: self.proof_sequencer.next_sequence(),
state_root_message_sender: self.tx.clone(),
multi_added_removed_keys: multi_added_removed_keys.clone(),
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
});
},
);
@@ -924,7 +757,6 @@ impl MultiProofTask {
self.multiproof_manager.proof_worker_handle.available_account_workers();
let available_storage_workers =
self.multiproof_manager.proof_worker_handle.available_storage_workers();
let num_chunks = dispatch_with_chunking(
not_fetched_state_update,
chunking_len,
@@ -938,9 +770,8 @@ impl MultiProofTask {
&hashed_state_update,
&self.fetched_proof_targets,
&multi_added_removed_keys,
self.v2_proofs_enabled,
);
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
spawned_proof_targets.extend_ref(&proof_targets);
self.multiproof_manager.dispatch(MultiproofInput {
source: Some(source),
@@ -1040,10 +871,7 @@ impl MultiProofTask {
batch_metrics.proofs_processed += 1;
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
SparseTrieUpdate { state, multiproof: Default::default() },
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1070,7 +898,8 @@ impl MultiProofTask {
}
let account_targets = merged_targets.len();
let storage_targets = merged_targets.storage_count();
let storage_targets =
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
trace!(
target: "engine::tree::payload_processor::multiproof",
@@ -1174,10 +1003,7 @@ impl MultiProofTask {
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
SparseTrieUpdate { state, multiproof: Default::default() },
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1280,7 +1106,7 @@ impl MultiProofTask {
let update = SparseTrieUpdate {
state: proof_result.state,
multiproof: proof_result_data,
multiproof: proof_result_data.proof,
};
if let Some(combined_update) =
@@ -1370,7 +1196,7 @@ struct MultiproofBatchCtx {
/// received.
updates_finished_time: Option<Instant>,
/// Reusable buffer for accumulating prefetch targets during batching.
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
accumulated_prefetch_targets: Vec<MultiProofTargets>,
}
impl MultiproofBatchCtx {
@@ -1416,77 +1242,40 @@ fn get_proof_targets(
state_update: &HashedPostState,
fetched_proof_targets: &MultiProofTargets,
multi_added_removed_keys: &MultiAddedRemovedKeys,
v2_enabled: bool,
) -> VersionedMultiProofTargets {
if v2_enabled {
let mut targets = MultiProofTargetsV2::default();
) -> MultiProofTargets {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for &hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(&hashed_address) {
targets.account_targets.push(hashed_address.into());
}
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
continue
}
let changed_slots = storage
.storage
.keys()
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
.map(|slot| Into::<proof_v2::Target>::into(*slot))
.collect::<Vec<_>>();
if !changed_slots.is_empty() {
targets.account_targets.push((*hashed_address).into());
targets.storage_targets.insert(*hashed_address, changed_slots);
}
}
VersionedMultiProofTargets::V2(targets)
} else {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
VersionedMultiProofTargets::Legacy(targets)
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
targets
}
/// Dispatches work items as a single unit or in chunks based on target size and worker
@@ -1527,15 +1316,14 @@ where
#[cfg(test)]
mod tests {
use crate::tree::cached_state::CachedStateProvider;
use super::*;
use crate::tree::cached_state::{CachedStateProvider, ExecutionCacheBuilder};
use alloy_eip7928::{AccountChanges, BalanceChange};
use alloy_primitives::Address;
use reth_provider::{
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
@@ -1562,7 +1350,6 @@ mod tests {
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + Clone
+ Send
@@ -1589,7 +1376,7 @@ mod tests {
{
let db_provider = factory.database_provider_ro().unwrap();
let state_provider: StateProviderBox = Box::new(LatestStateProvider::new(db_provider));
let cache = crate::tree::cached_state::ExecutionCache::new(1000);
let cache = ExecutionCacheBuilder::default().build_caches(1000);
CachedStateProvider::new(state_provider, cache, Default::default())
}
@@ -1694,24 +1481,12 @@ mod tests {
state
}
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
match targets {
VersionedMultiProofTargets::Legacy(targets) => targets,
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
}
}
#[test]
fn test_get_proof_targets_new_account_targets() {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should return all accounts as targets since nothing was fetched before
assert_eq!(targets.len(), state.accounts.len());
@@ -1725,12 +1500,7 @@ mod tests {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// verify storage slots are included for accounts with storage
for (addr, storage) in &state.storages {
@@ -1758,12 +1528,7 @@ mod tests {
// mark the account as already fetched
fetched.insert(*fetched_addr, HashSet::default());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should not include the already fetched account since it has no storage updates
assert!(!targets.contains_key(fetched_addr));
@@ -1783,12 +1548,7 @@ mod tests {
fetched_slots.insert(fetched_slot);
fetched.insert(*addr, fetched_slots);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// should not include the already fetched storage slot
let target_slots = &targets[addr];
@@ -1801,12 +1561,7 @@ mod tests {
let state = HashedPostState::default();
let fetched = MultiProofTargets::default();
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
assert!(targets.is_empty());
}
@@ -1833,12 +1588,7 @@ mod tests {
fetched_slots.insert(slot1);
fetched.insert(addr1, fetched_slots);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
assert!(targets.contains_key(&addr2));
assert!(!targets[&addr1].contains(&slot1));
@@ -1864,12 +1614,7 @@ mod tests {
assert!(!state.accounts.contains_key(&addr));
assert!(!fetched.contains_key(&addr));
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
// verify that we still get the storage slots for the unmodified account
assert!(targets.contains_key(&addr));
@@ -1911,12 +1656,7 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// slot1 should be included despite being fetched, because it's marked as removed
assert!(targets.contains_key(&addr));
@@ -1943,12 +1683,7 @@ mod tests {
storage.storage.insert(slot1, U256::from(100));
state.storages.insert(addr, storage);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// account should be included because storage is wiped and account wasn't fetched
assert!(targets.contains_key(&addr));
@@ -1991,12 +1726,7 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
// only slots in the state update can be included, so slot3 should not appear
assert!(!targets.contains_key(&addr));
@@ -2023,12 +1753,9 @@ mod tests {
targets3.insert(addr3, HashSet::default());
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
let proofs_requested =
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
@@ -2042,12 +1769,11 @@ mod tests {
assert_eq!(num_batched, 3);
assert_eq!(merged_targets.len(), 3);
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(legacy_targets.contains_key(&addr3));
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(merged_targets.contains_key(&addr3));
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
task.on_prefetch_proof(merged_targets)
} else {
panic!("Expected PrefetchProofs message");
};
@@ -2122,16 +1848,11 @@ mod tests {
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
targets3.clone(),
)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
let mut pending_msg: Option<MultiProofMessage> = None;
@@ -2157,10 +1878,9 @@ mod tests {
// Should have batched exactly 2 PrefetchProofs (not 3!)
assert_eq!(num_batched, 2, "Should batch only until different message type");
assert_eq!(merged_targets.len(), 2);
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
} else {
panic!("Expected PrefetchProofs message");
}
@@ -2185,8 +1905,7 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&addr3));
assert!(targets.contains_key(&addr3));
}
_ => panic!("PrefetchProofs3 was lost!"),
}
@@ -2232,13 +1951,9 @@ mod tests {
let source = StateChangeSource::Transaction(99);
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
prefetch2.clone(),
)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
let mut ctx = MultiproofBatchCtx::new(Instant::now());
let mut batch_metrics = MultiproofBatchMetrics::default();
@@ -2271,8 +1986,7 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&prefetch_addr2));
assert!(targets.contains_key(&prefetch_addr2));
}
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
}

View File

@@ -16,19 +16,20 @@ use crate::tree::{
payload_processor::{
bal::{total_slots, BALSlotIter},
executor::WorkloadExecutor,
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
PayloadExecutionCache,
multiproof::MultiProofMessage,
ExecutionCache as PayloadExecutionCache,
},
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
ExecutionEnv, StateProviderBuilder,
};
use alloy_consensus::transaction::TxHashRef;
use alloy_eip7928::BlockAccessList;
use alloy_eips::Typed2718;
use alloy_evm::Database;
use alloy_primitives::{keccak256, map::B256Set, B256};
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use crossbeam_channel::Sender as CrossbeamSender;
use metrics::{Counter, Gauge, Histogram};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
@@ -65,6 +66,19 @@ struct IndexedTransaction<Tx> {
tx: Tx,
}
/// Maximum standard Ethereum transaction type value.
///
/// Standard transaction types are:
/// - Type 0: Legacy transactions (original Ethereum)
/// - Type 1: EIP-2930 (access list transactions)
/// - Type 2: EIP-1559 (dynamic fee transactions)
/// - Type 3: EIP-4844 (blob transactions)
/// - Type 4: EIP-7702 (set code authorization transactions)
///
/// Any transaction with a type > 4 is considered a non-standard/system transaction,
/// typically used by L2s for special purposes (e.g., Optimism deposit transactions use type 126).
const MAX_STANDARD_TX_TYPE: u8 = 4;
/// A task that is responsible for caching and prewarming the cache by executing transactions
/// individually in parallel.
///
@@ -163,8 +177,8 @@ where
transaction_count_hint.min(max_concurrency)
};
// Spawn workers
let tx_sender = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
// Initialize worker handles container
let handles = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
// Distribute transactions to workers
let mut tx_index = 0usize;
@@ -179,18 +193,37 @@ where
}
let indexed_tx = IndexedTransaction { index: tx_index, tx };
let is_system_tx = indexed_tx.tx.tx().ty() > MAX_STANDARD_TX_TYPE;
// Send transaction to the workers
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled.
let _ = tx_sender.send(indexed_tx);
// System transactions (type > 4) in the first position set critical metadata
// that affects all subsequent transactions (e.g., L1 block info on L2s).
// Broadcast the first system transaction to all workers to ensure they have
// the critical state. This is particularly important for L2s like Optimism
// where the first deposit transaction (type 126) contains essential block metadata.
if tx_index == 0 && is_system_tx {
for handle in &handles {
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled. Sending to a disconnected worker is
// possible and harmless and should happen at most once due to
// the terminate_execution check above.
let _ = handle.send(indexed_tx.clone());
}
} else {
// Round-robin distribution for all other transactions
let worker_idx = tx_index % workers_needed;
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled. Sending to a disconnected worker is
// possible and harmless and should happen at most once due to
// the terminate_execution check above.
let _ = handles[worker_idx].send(indexed_tx);
}
tx_index += 1;
}
// drop sender and wait for all tasks to finish
// drop handle and wait for all tasks to finish and drop theirs
drop(done_tx);
drop(tx_sender);
drop(handles);
while done_rx.recv().is_ok() {}
let _ = actions_tx
@@ -204,7 +237,7 @@ where
}
/// If configured and the tx returned proof targets, emit the targets the transaction produced
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
if self.is_execution_terminated() {
// if execution is already terminated then we dont need to send more proof fetch
// messages
@@ -451,8 +484,6 @@ where
pub(super) terminate_execution: Arc<AtomicBool>,
pub(super) precompile_cache_disabled: bool,
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// Whether V2 proof calculation is enabled.
pub(super) v2_proofs_enabled: bool,
}
impl<N, P, Evm> PrewarmContext<N, P, Evm>
@@ -461,12 +492,10 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
Evm: ConfigureEvm<Primitives = N> + 'static,
{
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
/// execution, and whether V2 proofs are enabled.
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
/// execution.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn evm_for_ctx(
self,
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
let Self {
env,
evm_config,
@@ -476,7 +505,6 @@ where
terminate_execution,
precompile_cache_disabled,
precompile_cache_map,
v2_proofs_enabled,
} = self;
let mut state_provider = match provider.build() {
@@ -526,10 +554,10 @@ where
});
}
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
Some((evm, metrics, terminate_execution))
}
/// Accepts a [`CrossbeamReceiver`] of transactions and a handle to prewarm task. Executes
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
/// transactions and streams [`PrewarmTaskEvent::Outcome`] messages for each transaction.
///
/// This function processes transactions sequentially from the receiver and emits outcome events
@@ -541,16 +569,13 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn transact_batch<Tx>(
self,
txs: CrossbeamReceiver<IndexedTransaction<Tx>>,
txs: mpsc::Receiver<IndexedTransaction<Tx>>,
sender: Sender<PrewarmTaskEvent<N::Receipt>>,
done_tx: Sender<()>,
) where
Tx: ExecutableTxFor<Evm>,
{
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
else {
return
};
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
while let Ok(IndexedTransaction { index, tx }) = {
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
@@ -563,7 +588,6 @@ where
index,
tx_hash = %tx.tx().tx_hash(),
is_success = tracing::field::Empty,
gas_used = tracing::field::Empty,
)
.entered();
@@ -577,8 +601,7 @@ where
break
}
let (tx_env, tx) = tx.into_parts();
let res = match evm.transact(tx_env) {
let res = match evm.transact(&tx) {
Ok(res) => res,
Err(err) => {
trace!(
@@ -615,8 +638,7 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
.entered();
let (targets, storage_targets) =
multiproof_targets_from_state(res.state, v2_proofs_enabled);
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
metrics.prefetch_storage_targets.record(storage_targets as f64);
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
drop(_enter);
@@ -629,31 +651,35 @@ where
let _ = done_tx.send(());
}
/// Spawns worker tasks that pull transactions from a shared channel.
///
/// Returns the sender for distributing transactions to workers.
/// Spawns a worker task for transaction execution and returns its sender channel.
fn spawn_workers<Tx>(
self,
workers_needed: usize,
task_executor: &WorkloadExecutor,
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
done_tx: Sender<()>,
) -> CrossbeamSender<IndexedTransaction<Tx>>
) -> Vec<mpsc::Sender<IndexedTransaction<Tx>>>
where
Tx: ExecutableTxFor<Evm> + Send + 'static,
{
let (tx_sender, tx_receiver) = crossbeam_channel::unbounded();
let mut handles = Vec::with_capacity(workers_needed);
let mut receivers = Vec::with_capacity(workers_needed);
// Spawn workers that all pull from the shared receiver
for _ in 0..workers_needed {
let (tx, rx) = mpsc::channel();
handles.push(tx);
receivers.push(rx);
}
// Spawn a separate task spawning workers in parallel.
let executor = task_executor.clone();
let span = Span::current();
task_executor.spawn_blocking(move || {
let _enter = span.entered();
for idx in 0..workers_needed {
for (idx, rx) in receivers.into_iter().enumerate() {
let ctx = self.clone();
let actions_tx = actions_tx.clone();
let done_tx = done_tx.clone();
let rx = tx_receiver.clone();
let span = debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx);
executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -662,7 +688,7 @@ where
}
});
tx_sender
handles
}
/// Spawns a worker task for BAL slot prefetching.
@@ -757,22 +783,9 @@ where
}
}
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
/// on the given state.
fn multiproof_targets_from_state(
state: EvmState,
v2_enabled: bool,
) -> (VersionedMultiProofTargets, usize) {
if v2_enabled {
multiproof_targets_v2_from_state(state)
} else {
multiproof_targets_legacy_from_state(state)
}
}
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
/// given state.
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
let mut targets = MultiProofTargets::with_capacity(state.len());
let mut storage_targets = 0;
for (addr, account) in state {
@@ -802,50 +815,7 @@ fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProof
targets.insert(keccak256(addr), storage_set);
}
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
}
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
/// storage targets, based on the given state.
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
use reth_trie::proof_v2;
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
let mut targets = MultiProofTargetsV2::default();
let mut storage_target_count = 0;
for (addr, account) in state {
// if the account was not touched, or if the account was selfdestructed, do not
// fetch proofs for it
//
// Since selfdestruct can only happen in the same transaction, we can skip
// prefetching proofs for selfdestructed accounts
//
// See: https://eips.ethereum.org/EIPS/eip-6780
if !account.is_touched() || account.is_selfdestructed() {
continue
}
let hashed_address = keccak256(addr);
targets.account_targets.push(hashed_address.into());
let mut storage_slots = Vec::with_capacity(account.storage.len());
for (key, slot) in account.storage {
// do nothing if unchanged
if !slot.is_changed() {
continue
}
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
storage_slots.push(proof_v2::Target::from(hashed_slot));
}
storage_target_count += storage_slots.len();
if !storage_slots.is_empty() {
targets.storage_targets.insert(hashed_address, storage_slots);
}
}
(VersionedMultiProofTargets::V2(targets), storage_target_count)
(targets, storage_targets)
}
/// The events the pre-warm task can handle.
@@ -870,7 +840,7 @@ pub(super) enum PrewarmTaskEvent<R> {
/// The outcome of a pre-warm task
Outcome {
/// The prepared proof targets based on the evm state outcome
proof_targets: Option<VersionedMultiProofTargets>,
proof_targets: Option<MultiProofTargets>,
},
/// Finished executing all transactions
FinishedTxExecution {

View File

@@ -77,22 +77,8 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
receipt_with_bloom.encode_2718(&mut encode_buf);
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
match builder.push(indexed_receipt.index, &encode_buf) {
Ok(()) => {
received_count += 1;
}
Err(err) => {
// If a duplicate or out-of-bounds index is streamed, skip it and
// fall back to computing the receipt root from the full receipts
// vector later.
tracing::error!(
target: "engine::tree::payload_processor",
index = indexed_receipt.index,
?err,
"Receipt root task received invalid receipt index, skipping"
);
}
}
builder.push_unchecked(indexed_receipt.index, &encode_buf);
received_count += 1;
}
let Ok(root) = builder.finalize() else {

View File

@@ -1,34 +1,15 @@
//! Sparse Trie task related functionality.
use crate::tree::{
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
};
use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate};
use alloy_primitives::B256;
use alloy_rlp::Decodable;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_errors::ProviderError;
use reth_primitives_traits::Account;
use reth_revm::state::EvmState;
use reth_trie::{
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
};
use reth_trie_parallel::{
proof_task::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
root::ParallelStateRootError,
targets_v2::MultiProofTargetsV2,
};
use reth_trie::{updates::TrieUpdates, Nibbles};
use reth_trie_parallel::root::ParallelStateRootError;
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
SparseTrieExt,
ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrieInterface,
};
use revm_primitives::{hash_map::Entry, B256Map};
use smallvec::SmallVec;
use std::{
sync::mpsc,
@@ -57,8 +38,8 @@ where
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
A: SparseTrie + Send + Sync + Default,
S: SparseTrie + Send + Sync + Default + Clone,
A: SparseTrieInterface + Send + Sync + Default,
S: SparseTrieInterface + Send + Sync + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
@@ -116,8 +97,8 @@ where
debug!(
target: "engine::root",
num_updates,
account_proofs = update.multiproof.account_proofs_len(),
storage_proofs = update.multiproof.storage_proofs_len(),
account_proofs = update.multiproof.account_subtree.len(),
storage_proofs = update.multiproof.storages.len(),
"Updating sparse trie"
);
@@ -148,359 +129,6 @@ where
}
}
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
/// Sender for proof results.
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Receiver for proof results directly from workers.
proof_result_rx: CrossbeamReceiver<ProofResultMessage>,
/// Receives updates from execution and prewarming.
updates: CrossbeamReceiver<MultiProofMessage>,
/// `SparseStateTrie` used for computing the state root.
trie: SparseStateTrie<A, S>,
/// Handle to the proof worker pools (storage and account).
proof_worker_handle: ProofWorkerHandle,
/// Account trie updates.
account_updates: B256Map<LeafUpdate>,
/// Storage trie updates. hashed address -> slot -> update.
storage_updates: B256Map<B256Map<LeafUpdate>>,
/// Account updates that are blocked by storage root calculation or account reveal.
///
/// Those are being moved into `account_updates` once storage roots
/// are revealed and/or calculated.
///
/// Invariant: for each entry in `pending_account_updates` account must either be already
/// revealed in the trie or have an entry in `account_updates`.
///
/// Values can be either of:
/// - None: account had a storage update and is awaiting storage root calculation and/or
/// account node reveal to complete.
/// - Some(_): account was changed/destroyed and is awaiting storage root calculation/reveal
/// to complete.
pending_account_updates: B256Map<Option<Option<Account>>>,
/// Metrics for the sparse trie.
metrics: MultiProofTaskMetrics,
}
impl<A, S> SparseTrieCacheTask<A, S>
where
A: SparseTrieExt + Default,
S: SparseTrieExt + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
updates: CrossbeamReceiver<MultiProofMessage>,
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
sparse_state_trie: ClearedSparseStateTrie<A, S>,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
Self {
proof_result_tx,
proof_result_rx,
updates,
proof_worker_handle,
trie: sparse_state_trie.into_inner(),
account_updates: Default::default(),
storage_updates: Default::default(),
pending_account_updates: Default::default(),
metrics,
}
}
/// Runs the sparse trie task to completion.
///
/// This waits for new incoming [`MultiProofMessage`]s, applies updates to the trie and
/// schedules proof fetching when needed.
///
/// This concludes once the last state update has been received and processed.
///
/// # Returns
///
/// - State root computation outcome.
/// - `SparseStateTrie` that needs to be cleared and reused to avoid reallocations.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
pub(super) fn run(
mut self,
) -> (Result<StateRootComputeOutcome, ParallelStateRootError>, SparseStateTrie<A, S>) {
// run the main loop to completion
let result = self.run_inner();
(result, self.trie)
}
/// Inner function to run the sparse trie task to completion.
///
/// See [`Self::run`] for more information.
fn run_inner(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
let now = Instant::now();
let mut finished_state_updates = false;
loop {
crossbeam_channel::select_biased! {
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
self.on_proof_result(result)?;
},
recv(self.updates) -> message => {
let update = match message {
Ok(m) => m,
Err(_) => {
break
}
};
match update {
MultiProofMessage::PrefetchProofs(targets) => {
self.on_prewarm_targets(targets);
}
MultiProofMessage::StateUpdate(_, state) => {
self.on_state_update(state);
}
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
self.on_hashed_state_update(state);
}
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => {
finished_state_updates = true;
}
}
}
}
self.process_updates()?;
if finished_state_updates &&
self.account_updates.is_empty() &&
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
{
break;
}
}
// Process any remaining pending account updates.
if !self.pending_account_updates.is_empty() {
self.process_updates()?;
}
debug!(target: "engine::root", "All proofs processed, ending calculation");
let start = Instant::now();
let (state_root, trie_updates) =
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
})?;
let end = Instant::now();
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
Ok(StateRootComputeOutcome { state_root, trie_updates })
}
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
let VersionedMultiProofTargets::V2(targets) = targets else {
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
};
for target in targets.account_targets {
// Only touch accounts that are not yet present in the updates set.
self.account_updates.entry(target.key()).or_insert(LeafUpdate::Touched);
}
for (address, slots) in targets.storage_targets {
for slot in slots {
// Only touch storages that are not yet present in the updates set.
self.storage_updates
.entry(address)
.or_default()
.entry(slot.key())
.or_insert(LeafUpdate::Touched);
}
// Touch corresponding account leaf to make sure its revealed in accounts trie for
// storage root update.
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
}
}
/// Processes a state update and encodes all state changes as trie updates.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all,
fields(accounts = update.len())
)]
fn on_state_update(&mut self, update: EvmState) {
let hashed_state_update = evm_state_to_hashed_post_state(update);
self.on_hashed_state_update(hashed_state_update)
}
/// Processes a hashed state update and encodes all state changes as trie updates.
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
for (address, storage) in hashed_state_update.storages {
for (slot, value) in storage.storage {
let encoded = if value.is_zero() {
Vec::new()
} else {
alloy_rlp::encode_fixed_size(&value).to_vec()
};
self.storage_updates
.entry(address)
.or_default()
.insert(slot, LeafUpdate::Changed(encoded));
}
// Make sure account is tracked in `account_updates` so that it is revealed in accounts
// trie for storage root update.
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
// Make sure account is tracked in `pending_account_updates` so that once storage root
// is computed, it will be updated in the accounts trie.
self.pending_account_updates.entry(address).or_insert(None);
}
for (address, account) in hashed_state_update.accounts {
// Track account as touched.
//
// This might overwrite an existing update, which is fine, because storage root from it
// is already tracked in the trie and can be easily fetched again.
self.account_updates.insert(address, LeafUpdate::Touched);
// Track account in `pending_account_updates` so that once storage root is computed,
// it will be updated in the accounts trie.
self.pending_account_updates.insert(address, Some(account));
}
}
fn on_proof_result(
&mut self,
result: ProofResultMessage,
) -> Result<(), ParallelStateRootError> {
let ProofResult::V2(result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
})
}
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
fn process_updates(&mut self) -> Result<(), ProviderError> {
let mut targets = MultiProofTargetsV2::default();
for (addr, updates) in &mut self.storage_updates {
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
trie.update_leaves(updates, |path, min_len| {
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
})
.map_err(ProviderError::other)?;
// If all storage updates were processed, we can now compute the new storage root.
if updates.is_empty() {
let storage_root =
trie.root().expect("updates are drained, trie should be revealed by now");
// If there is a pending account update for this address with known info, we can
// encode it into proper update right away.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
Vec::new()
} else {
// TODO: optimize allocation
alloy_rlp::encode(
account.unwrap_or_default().into_trie_account(storage_root),
)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
}
// Now handle pending account updates that can be upgraded to a proper update.
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
let account = account.unwrap_or_default().into_trie_account(storage_root);
// TODO: optimize allocation
alloy_rlp::encode(account)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
false
});
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
targets.account_targets.push(Target::new(target).with_min_len(min_len));
})
.map_err(ProviderError::other)?;
if !targets.is_empty() {
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
})?;
}
Ok(())
}
}
/// Outcome of the state root computation, including the state root itself with
/// the trie updates.
#[derive(Debug)]
@@ -522,21 +150,14 @@ where
BPF: TrieNodeProviderFactory + Send + Sync,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
A: SparseTrie + Send + Sync + Default,
S: SparseTrie + Send + Sync + Default + Clone,
A: SparseTrieInterface + Send + Sync + Default,
S: SparseTrieInterface + Send + Sync + Default + Clone,
{
trace!(target: "engine::root::sparse", "Updating sparse trie");
let started_at = Instant::now();
// Reveal new accounts and storage slots.
match multiproof {
ProofResult::Legacy(decoded, _) => {
trie.reveal_decoded_multiproof(decoded)?;
}
ProofResult::V2(decoded_v2) => {
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
}
}
trie.reveal_decoded_multiproof(multiproof)?;
let reveal_multiproof_elapsed = started_at.elapsed();
trace!(
target: "engine::root::sparse",

View File

@@ -39,7 +39,7 @@ use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader, StorageChangeSetReader,
StateProviderFactory, StateReader,
};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
@@ -144,7 +144,6 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ ChangeSetReader
@@ -503,7 +502,6 @@ where
let root_time = Instant::now();
let mut maybe_state_root = None;
let mut state_root_task_failed = false;
match strategy {
StateRootStrategy::StateRootTask => {
@@ -522,12 +520,10 @@ where
block_state_root = ?block.header().state_root(),
"State root task returned incorrect state root"
);
state_root_task_failed = true;
}
}
Err(error) => {
debug!(target: "engine::tree::payload_validator", %error, "State root task failed");
state_root_task_failed = true;
}
}
}
@@ -572,11 +568,6 @@ where
self.compute_state_root_serial(overlay_factory.clone(), &hashed_state),
block
);
if state_root_task_failed {
self.metrics.block_validation.state_root_task_fallback_success_total.increment(1);
}
(root, updates, root_time.elapsed())
};
@@ -792,11 +783,6 @@ where
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
// Some executors may execute transactions that do not append receipts during the
// main loop (e.g., system transactions whose receipts are added during finalization).
// In that case, invoking the callback on every transaction would resend the previous
// receipt with the same index and can panic the ordered root builder.
let mut last_sent_len = 0usize;
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
@@ -823,14 +809,10 @@ where
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
let current_len = executor.receipts().len();
if current_len > last_sent_len {
last_sent_len = current_len;
// Send the latest receipt to the background task for incremental root computation.
if let Some(receipt) = executor.receipts().last() {
let tx_index = current_len - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
@@ -1112,13 +1094,10 @@ where
/// while the trie input computation is deferred until the overlay is actually needed.
///
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
///
/// Uses a cached overlay if available for the canonical head (the common case).
fn get_parent_lazy_overlay(
parent_hash: B256,
state: &EngineApiTreeState<N>,
) -> (Option<LazyOverlay>, B256) {
// Get blocks leading to the parent to determine the anchor
let (anchor_hash, blocks) =
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
@@ -1127,17 +1106,6 @@ where
return (None, anchor_hash);
}
// Try to use the cached overlay if it matches both parent hash and anchor
if let Some(cached) = state.tree_state.get_cached_overlay(parent_hash, anchor_hash) {
debug!(
target: "engine::tree::payload_validator",
%parent_hash,
%anchor_hash,
"Using cached canonical overlay"
);
return (Some(cached.overlay.clone()), cached.anchor_hash);
}
debug!(
target: "engine::tree::payload_validator",
%anchor_hash,
@@ -1368,7 +1336,6 @@ where
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ StateProviderFactory

View File

@@ -6,7 +6,7 @@ use alloy_primitives::{
map::{HashMap, HashSet},
BlockNumber, B256,
};
use reth_chain_state::{DeferredTrieData, EthPrimitives, ExecutedBlock, LazyOverlay};
use reth_chain_state::{EthPrimitives, ExecutedBlock};
use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader};
use std::{
collections::{btree_map, hash_map, BTreeMap, VecDeque},
@@ -38,12 +38,6 @@ pub struct TreeState<N: NodePrimitives = EthPrimitives> {
pub(crate) current_canonical_head: BlockNumHash,
/// The engine API variant of this handler
pub(crate) engine_kind: EngineApiKind,
/// Pre-computed lazy overlay for the canonical head.
///
/// This is optimistically prepared after the canonical head changes, so that
/// the next payload building on the canonical head can use it immediately
/// without recomputing.
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay>,
}
impl<N: NodePrimitives> TreeState<N> {
@@ -55,7 +49,6 @@ impl<N: NodePrimitives> TreeState<N> {
current_canonical_head,
parent_to_child: HashMap::default(),
engine_kind,
cached_canonical_overlay: None,
}
}
@@ -99,66 +92,6 @@ impl<N: NodePrimitives> TreeState<N> {
Some((parent_hash, blocks))
}
/// Prepares a cached lazy overlay for the current canonical head.
///
/// This should be called after the canonical head changes to optimistically
/// prepare the overlay for the next payload that will likely build on it.
///
/// Returns a clone of the [`LazyOverlay`] so the caller can spawn a background
/// task to trigger computation via [`LazyOverlay::get`]. This ensures the overlay
/// is actually computed before the next payload arrives.
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<LazyOverlay> {
let canonical_hash = self.current_canonical_head.hash;
// Get blocks leading to the canonical head
let Some((anchor_hash, blocks)) = self.blocks_by_hash(canonical_hash) else {
// Canonical head not in memory (persisted), no overlay needed
self.cached_canonical_overlay = None;
return None;
};
// Extract deferred trie data handles from blocks (newest to oldest)
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
let overlay = LazyOverlay::new(anchor_hash, handles);
self.cached_canonical_overlay = Some(PreparedCanonicalOverlay {
parent_hash: canonical_hash,
overlay: overlay.clone(),
anchor_hash,
});
debug!(
target: "engine::tree",
%canonical_hash,
%anchor_hash,
num_blocks = blocks.len(),
"Prepared cached canonical overlay"
);
Some(overlay)
}
/// Returns the cached overlay if it matches the requested parent hash and anchor.
///
/// Both parent hash and anchor hash must match to ensure the overlay is valid.
/// This prevents using a stale overlay after persistence has advanced the anchor.
pub(crate) fn get_cached_overlay(
&self,
parent_hash: B256,
expected_anchor: B256,
) -> Option<&PreparedCanonicalOverlay> {
self.cached_canonical_overlay.as_ref().filter(|cached| {
cached.parent_hash == parent_hash && cached.anchor_hash == expected_anchor
})
}
/// Invalidates the cached overlay.
///
/// Should be called when the anchor changes (e.g., after persistence).
pub(crate) fn invalidate_cached_overlay(&mut self) {
self.cached_canonical_overlay = None;
}
/// Insert executed block into the state.
pub(crate) fn insert_executed(&mut self, executed: ExecutedBlock<N>) {
let hash = executed.recovered_block().hash();
@@ -355,9 +288,6 @@ impl<N: NodePrimitives> TreeState<N> {
if let Some(finalized_num_hash) = finalized_num_hash {
self.prune_finalized_sidechains(finalized_num_hash);
}
// Invalidate the cached overlay since blocks were removed and the anchor may have changed
self.invalidate_cached_overlay();
}
/// Updates the canonical head to the given block.
@@ -425,39 +355,6 @@ impl<N: NodePrimitives> TreeState<N> {
}
}
/// Pre-computed lazy overlay for the canonical head block.
///
/// This is prepared **optimistically** when the canonical head changes, allowing
/// the next payload (which typically builds on the canonical head) to reuse
/// the pre-computed overlay immediately without re-traversing in-memory blocks.
///
/// The overlay captures deferred trie data handles from all in-memory blocks
/// between the canonical head and the persisted anchor. When a new payload
/// arrives building on the canonical head, this cached overlay can be used
/// directly instead of calling `blocks_by_hash` and collecting handles again.
///
/// # Invalidation
///
/// The cached overlay is invalidated when:
/// - Persistence completes (anchor changes)
/// - The canonical head changes to a different block
#[derive(Debug, Clone)]
pub struct PreparedCanonicalOverlay {
/// The block hash for which this overlay is prepared as a parent.
///
/// When a payload arrives with this parent hash, the overlay can be reused.
pub parent_hash: B256,
/// The pre-computed lazy overlay containing deferred trie data handles.
///
/// This is computed optimistically after `set_canonical_head` so subsequent
/// payloads don't need to re-collect the handles.
pub overlay: LazyOverlay,
/// The anchor hash (persisted ancestor) this overlay is based on.
///
/// Used to verify the overlay is still valid (anchor hasn't changed due to persistence).
pub anchor_hash: B256,
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -259,7 +259,6 @@ impl TestHarness {
current_canonical_head: blocks.last().unwrap().recovered_block().num_hash(),
parent_to_child,
engine_kind: EngineApiKind::Ethereum,
cached_canonical_overlay: None,
};
let last_executed_block = blocks.last().unwrap().clone();

View File

@@ -52,7 +52,7 @@ pub fn read_dir(
checksums.next().transpose()?.ok_or_eyre("Got less checksums than ERA files")?;
}
entries.sort_by_key(|(left, _)| *left);
entries.sort_by(|(left, _), (right, _)| left.cmp(right));
Ok(stream::iter(entries.into_iter().skip_while(move |(n, _)| *n < start_index).map(
move |(_, path)| {

View File

@@ -20,7 +20,6 @@ use reth_era::{
},
};
use reth_fs_util as fs;
use reth_primitives_traits::Block;
use reth_storage_api::{BlockNumReader, BlockReader, HeaderProvider};
use std::{
path::PathBuf,
@@ -296,11 +295,9 @@ where
return Err(eyre!("Expected block {expected_block_number}, got {actual_block_number}"));
}
// CompressedBody must contain the block *body* (rlp(body)), not the full block (rlp(block)).
let body = provider
.block_by_number(actual_block_number)?
.ok_or_else(|| eyre!("Block not found for block {}", actual_block_number))?
.into_body();
.ok_or_else(|| eyre!("Block body not found for block {}", actual_block_number))?;
let receipts = provider
.receipts_by_block(actual_block_number.into())?

View File

@@ -174,7 +174,7 @@ where
}
Commands::P2P(command) => runner.run_until_ctrl_c(command.execute::<N>()),
Commands::Config(command) => runner.run_until_ctrl_c(command.execute()),
Commands::Prune(command) => runner.run_command_until_exit(|ctx| command.execute::<N>(ctx)),
Commands::Prune(command) => runner.run_until_ctrl_c(command.execute::<N>()),
#[cfg(feature = "dev")]
Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()),
Commands::ReExecute(command) => runner.run_until_ctrl_c(command.execute::<N>(components)),

View File

@@ -17,11 +17,10 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
mod error;
pub use error::*;
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload};
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5};
pub use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6, ExecutionPayloadV1,
PayloadAttributes as EthPayloadAttributes,
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
};
use reth_engine_primitives::EngineTypes;
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
@@ -67,15 +66,13 @@ where
+ TryInto<ExecutionPayloadEnvelopeV2>
+ TryInto<ExecutionPayloadEnvelopeV3>
+ TryInto<ExecutionPayloadEnvelopeV4>
+ TryInto<ExecutionPayloadEnvelopeV5>
+ TryInto<ExecutionPayloadEnvelopeV6>,
+ TryInto<ExecutionPayloadEnvelopeV5>,
{
type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1;
type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2;
type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5;
type ExecutionPayloadEnvelopeV6 = ExecutionPayloadEnvelopeV6;
}
/// A default payload type for [`EthEngineTypes`]

View File

@@ -11,8 +11,8 @@ use alloy_primitives::{Address, B256, U256};
use alloy_rlp::Encodable;
use alloy_rpc_types_engine::{
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2,
ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
};
use core::convert::Infallible;
use reth_ethereum_primitives::EthPrimitives;
@@ -160,13 +160,6 @@ impl EthBuiltPayload {
execution_requests: requests.unwrap_or_default(),
})
}
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
///
/// Note: Amsterdam fork is not yet implemented, so this conversion is not yet supported.
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
unimplemented!("ExecutionPayloadEnvelopeV6 not yet supported")
}
}
impl<N: NodePrimitives> BuiltPayload for EthBuiltPayload<N> {
@@ -234,14 +227,6 @@ impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV5 {
}
}
impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV6 {
type Error = BuiltPayloadConversionError;
fn try_from(value: EthBuiltPayload) -> Result<Self, Self::Error> {
value.try_into_v6()
}
}
/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`].
#[derive(Clone, Default, Debug)]
pub enum BlobSidecars {

View File

@@ -59,7 +59,6 @@ std = [
"reth-storage-errors/std",
]
test-utils = [
"std",
"dep:parking_lot",
"dep:derive_more",
"reth-chainspec/test-utils",

View File

@@ -1,4 +1,3 @@
use alloy_consensus::TxType;
use alloy_evm::eth::receipt_builder::{ReceiptBuilder, ReceiptBuilderCtx};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm::Evm;
@@ -13,10 +12,13 @@ impl ReceiptBuilder for RethReceiptBuilder {
type Transaction = TransactionSigned;
type Receipt = Receipt;
fn build_receipt<E: Evm>(&self, ctx: ReceiptBuilderCtx<'_, TxType, E>) -> Self::Receipt {
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, .. } = ctx;
fn build_receipt<E: Evm>(
&self,
ctx: ReceiptBuilderCtx<'_, Self::Transaction, E>,
) -> Self::Receipt {
let ReceiptBuilderCtx { tx, result, cumulative_gas_used, .. } = ctx;
Receipt {
tx_type,
tx_type: tx.tx_type(),
// Success flag was added in `EIP-658: Embedding transaction status code in
// receipts`.
success: result.is_success(),

View File

@@ -1,6 +1,6 @@
use crate::EthEvmConfig;
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
use alloy_consensus::{Header, TxType};
use alloy_consensus::Header;
use alloy_eips::eip7685::Requests;
use alloy_evm::precompiles::PrecompilesMap;
use alloy_primitives::Bytes;
@@ -11,14 +11,14 @@ use reth_evm::{
block::{
BlockExecutionError, BlockExecutor, BlockExecutorFactory, BlockExecutorFor, ExecutableTx,
},
eth::{EthBlockExecutionCtx, EthEvmContext, EthTxResult},
eth::{EthBlockExecutionCtx, EthEvmContext},
ConfigureEngineEvm, ConfigureEvm, Database, EthEvm, EthEvmFactory, Evm, EvmEnvFor, EvmFactory,
ExecutableTxIterator, ExecutionCtxFor, RecoveredTx,
ExecutableTxIterator, ExecutionCtxFor,
};
use reth_execution_types::{BlockExecutionResult, ExecutionOutcome};
use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader};
use revm::{
context::result::{ExecutionResult, HaltReason, Output, ResultAndState, SuccessReason},
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
Inspector,
};
@@ -90,7 +90,6 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
type Evm = EthEvm<&'a mut State<DB>, I, PrecompilesMap>;
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Result = EthTxResult<HaltReason, TxType>;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
@@ -102,25 +101,25 @@ impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExec
fn execute_transaction_without_commit(
&mut self,
tx: impl ExecutableTx<Self>,
) -> Result<Self::Result, BlockExecutionError> {
Ok(EthTxResult {
result: ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
),
tx_type: tx.into_parts().1.tx().tx_type(),
blob_gas_used: 0,
})
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 0,
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(&mut self, _output: Self::Result) -> Result<u64, BlockExecutionError> {
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(0)
}

View File

@@ -456,22 +456,17 @@ pub struct EthereumPoolBuilder {
// TODO add options for txpool args
}
impl<Types, Node, Evm> PoolBuilder<Node, Evm> for EthereumPoolBuilder
impl<Types, Node> PoolBuilder<Node> for EthereumPoolBuilder
where
Types: NodeTypes<
ChainSpec: EthereumHardforks,
Primitives: NodePrimitives<SignedTx = TransactionSigned>,
>,
Node: FullNodeTypes<Types = Types>,
Evm: ConfigureEvm<Primitives = PrimitivesTy<Types>> + Clone + 'static,
{
type Pool = EthTransactionPool<Node::Provider, DiskFileBlobStore, Evm>;
type Pool = EthTransactionPool<Node::Provider, DiskFileBlobStore>;
async fn build_pool(
self,
ctx: &BuilderContext<Node>,
evm_config: Evm,
) -> eyre::Result<Self::Pool> {
async fn build_pool(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Pool> {
let pool_config = ctx.pool_config();
let blobs_disabled = ctx.config().txpool.disable_blobs_support ||
@@ -497,17 +492,17 @@ where
let blob_store =
reth_node_builder::components::create_blob_store_with_cache(ctx, blob_cache_size)?;
let validator =
TransactionValidationTaskExecutor::eth_builder(ctx.provider().clone(), evm_config)
.set_eip4844(!blobs_disabled)
.kzg_settings(ctx.kzg_settings()?)
.with_max_tx_input_bytes(ctx.config().txpool.max_tx_input_bytes)
.with_local_transactions_config(pool_config.local_transactions_config.clone())
.set_tx_fee_cap(ctx.config().rpc.rpc_tx_fee_cap)
.with_max_tx_gas_limit(ctx.config().txpool.max_tx_gas_limit)
.with_minimum_priority_fee(ctx.config().txpool.minimum_priority_fee)
.with_additional_tasks(ctx.config().txpool.additional_validation_tasks)
.build_with_tasks(ctx.task_executor().clone(), blob_store.clone());
let validator = TransactionValidationTaskExecutor::eth_builder(ctx.provider().clone())
.with_head_timestamp(ctx.head().timestamp)
.set_eip4844(!blobs_disabled)
.kzg_settings(ctx.kzg_settings()?)
.with_max_tx_input_bytes(ctx.config().txpool.max_tx_input_bytes)
.with_local_transactions_config(pool_config.local_transactions_config.clone())
.set_tx_fee_cap(ctx.config().rpc.rpc_tx_fee_cap)
.with_max_tx_gas_limit(ctx.config().txpool.max_tx_gas_limit)
.with_minimum_priority_fee(ctx.config().txpool.minimum_priority_fee)
.with_additional_tasks(ctx.config().txpool.additional_validation_tasks)
.build_with_tasks(ctx.task_executor().clone(), blob_store.clone());
if validator.validator().eip4844() {
// initializing the KZG settings can be expensive, this should be done upfront so that

View File

@@ -1,357 +0,0 @@
//! Tests for handling invalid payloads via Engine API.
//!
//! This module tests the scenario where a node receives invalid payloads (e.g., with modified
//! state roots) before receiving valid ones, ensuring the node can recover and continue.
use crate::utils::eth_payload_attributes;
use alloy_primitives::B256;
use alloy_rpc_types_engine::{ExecutionPayloadV3, PayloadStatusEnum};
use rand::{rngs::StdRng, Rng, SeedableRng};
use reth_chainspec::{ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::{setup_engine, transaction::TransactionTestContext};
use reth_node_ethereum::EthereumNode;
use reth_rpc_api::EngineApiClient;
use std::sync::Arc;
/// Tests that a node can handle receiving an invalid payload (with wrong state root)
/// followed by the correct payload, and continue operating normally.
///
/// Setup:
/// - Node 1: Produces valid payloads and advances the chain
/// - Node 2: Receives payloads from node 1, but we also inject modified payloads with invalid state
/// roots in between to verify error handling
#[tokio::test]
async fn can_handle_invalid_payload_then_valid() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
// Get engine API client for the receiver node
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building (advance_block waits for transactions)
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Build a valid payload on the producer
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send the invalid payload to the receiver - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status, invalid_payload.payload_inner.payload_inner.state_root
);
// The invalid payload should be rejected
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING status for invalid payload, got {:?}",
invalid_result.status
);
// Now send the valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload response: {:?}", valid_result.status);
// The valid payload should be accepted
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected VALID/SYNCING/ACCEPTED status for valid payload, got {:?}",
valid_result.status
);
// Update forkchoice on receiver to the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify the receiver node is at the expected block
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!(
"Test passed: Receiver successfully handled invalid payloads and synced to valid chain"
);
Ok(())
}
/// Tests that a node can handle multiple consecutive invalid payloads
/// before receiving a valid one.
#[tokio::test]
async fn can_handle_multiple_invalid_payloads() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Inject a transaction to allow block building
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
producer.rpc.inject_tx(raw_tx).await?;
// Produce a valid block
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Send multiple invalid payloads with different corruptions
for i in 0..3 {
// Create valid payload first, then corrupt the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
let result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Invalid payload {i}: status = {:?}", result.status);
assert!(
matches!(result.status, PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing),
"Expected INVALID or SYNCING for invalid payload {i}, got {:?}",
result.status
);
}
// Now send the valid payload
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload: status = {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload, got {:?}",
valid_result.status
);
// Finalize the valid block
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
println!("Test passed: Receiver handled multiple invalid payloads and accepted valid one");
Ok(())
}
/// Tests invalid payload handling with blocks that contain transactions.
///
/// This test sends real transactions to node 1, produces blocks with those transactions,
/// then sends invalid (corrupted state root) and valid payloads to node 2.
#[tokio::test]
async fn can_handle_invalid_payload_with_transactions() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let seed: [u8; 32] = rand::rng().random();
let mut rng = StdRng::from_seed(seed);
println!("Seed: {seed:?}");
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
2,
chain_spec.clone(),
false,
Default::default(),
eth_payload_attributes,
)
.await?;
let mut producer = nodes.pop().unwrap();
let receiver = nodes.pop().unwrap();
let receiver_engine = receiver.auth_server_handle().http_client();
// Create and send a transaction to the producer node
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
let tx_hash = producer.rpc.inject_tx(raw_tx).await?;
println!("Injected transaction {tx_hash}");
// Build a block containing the transaction
let payload = producer.advance_block().await?;
let valid_block = payload.block().clone();
// Verify the block contains a transaction
let tx_count = valid_block.body().transactions().count();
println!("Block contains {tx_count} transaction(s)");
assert!(tx_count > 0, "Block should contain at least one transaction");
// Create invalid payload by corrupting the state root
let mut invalid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let original_state_root = invalid_payload.payload_inner.payload_inner.state_root;
invalid_payload.payload_inner.payload_inner.state_root = B256::random_with(&mut rng);
// Send invalid payload - should be rejected
let invalid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
invalid_payload.clone(),
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!(
"Invalid payload (with tx) response: {:?} (state_root changed from {original_state_root} to {})",
invalid_result.status,
invalid_payload.payload_inner.payload_inner.state_root
);
assert!(
matches!(
invalid_result.status,
PayloadStatusEnum::Invalid { .. } | PayloadStatusEnum::Syncing
),
"Expected INVALID or SYNCING for invalid payload with transactions, got {:?}",
invalid_result.status
);
// Send valid payload - should be accepted
let valid_payload = ExecutionPayloadV3::from_block_unchecked(
valid_block.hash(),
&valid_block.clone().into_block(),
);
let valid_result = EngineApiClient::<reth_node_ethereum::EthEngineTypes>::new_payload_v3(
&receiver_engine,
valid_payload,
vec![],
valid_block.header().parent_beacon_block_root.unwrap_or_default(),
)
.await?;
println!("Valid payload (with tx) response: {:?}", valid_result.status);
assert!(
matches!(
valid_result.status,
PayloadStatusEnum::Valid | PayloadStatusEnum::Syncing | PayloadStatusEnum::Accepted
),
"Expected valid status for correct payload with transactions, got {:?}",
valid_result.status
);
// Update forkchoice
receiver.update_forkchoice(valid_block.hash(), valid_block.hash()).await?;
// Verify both nodes are at the same head
let receiver_head = receiver.block_hash(1);
let producer_head = producer.block_hash(1);
assert_eq!(
receiver_head, producer_head,
"Receiver should have synced to the same chain as producer"
);
println!("Test passed: Receiver handled invalid payloads with transactions correctly");
Ok(())
}

View File

@@ -4,12 +4,10 @@ mod blobs;
mod custom_genesis;
mod dev;
mod eth;
mod invalid_payload;
mod p2p;
mod pool;
mod prestate;
mod rpc;
mod selfdestruct;
mod utils;
const fn main() {}

View File

@@ -1,529 +0,0 @@
//! E2E tests for SELFDESTRUCT behavior and output state verification.
//!
//! These tests verify that:
//! - Pre-Dencun: SELFDESTRUCT clears storage and code, output state reflects this
//! - Post-Dencun (EIP-6780): SELFDESTRUCT only works in same-tx creation, state persists
//!
//! We disable prewarming to ensure deterministic cache behavior and verify the execution
//! output state contains the expected account status after SELFDESTRUCT.
use crate::utils::{eth_payload_attributes, eth_payload_attributes_shanghai};
use alloy_network::{EthereumWallet, TransactionBuilder};
use alloy_primitives::{bytes, Address, Bytes, TxKind, U256};
use alloy_provider::{Provider, ProviderBuilder};
use alloy_rpc_types_eth::TransactionRequest;
use futures::StreamExt;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_e2e_test_utils::setup_engine;
use reth_node_api::TreeConfig;
use reth_node_ethereum::EthereumNode;
use reth_revm::db::BundleAccount;
use std::sync::Arc;
const MAX_FEE_PER_GAS: u128 = 20_000_000_000;
const MAX_PRIORITY_FEE_PER_GAS: u128 = 1_000_000_000;
fn cancun_spec() -> Arc<ChainSpec> {
Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.cancun_activated()
.build(),
)
}
fn shanghai_spec() -> Arc<ChainSpec> {
Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
.shanghai_activated()
.build(),
)
}
fn deploy_tx(from: Address, nonce: u64, init_code: Bytes) -> TransactionRequest {
TransactionRequest::default()
.with_from(from)
.with_nonce(nonce)
.with_gas_limit(500_000)
.with_max_fee_per_gas(MAX_FEE_PER_GAS)
.with_max_priority_fee_per_gas(MAX_PRIORITY_FEE_PER_GAS)
.with_input(init_code)
.with_kind(TxKind::Create)
}
fn call_tx(from: Address, to: Address, nonce: u64) -> TransactionRequest {
TransactionRequest::default()
.with_from(from)
.with_to(to)
.with_nonce(nonce)
.with_gas_limit(100_000)
.with_max_fee_per_gas(MAX_FEE_PER_GAS)
.with_max_priority_fee_per_gas(MAX_PRIORITY_FEE_PER_GAS)
}
fn transfer_tx(from: Address, to: Address, nonce: u64, value: U256) -> TransactionRequest {
TransactionRequest::default()
.with_from(from)
.with_to(to)
.with_nonce(nonce)
.with_value(value)
.with_gas_limit(21_000)
.with_max_fee_per_gas(MAX_FEE_PER_GAS)
.with_max_priority_fee_per_gas(MAX_PRIORITY_FEE_PER_GAS)
}
/// Creates init code for a contract that selfdestructs during deployment (same tx).
/// This tests the EIP-6780 exception where SELFDESTRUCT in same tx as creation still works.
///
/// The contract:
/// 1. Stores 0x42 at slot 0
/// 2. Immediately selfdestructs to beneficiary (during init, before returning runtime)
fn selfdestruct_in_constructor_init_code() -> Bytes {
// Init code that selfdestructs during deployment:
// PUSH1 0x42, PUSH1 0x00, SSTORE (store 0x42 at slot 0)
// PUSH20 <beneficiary>, SELFDESTRUCT
let mut init = Vec::new();
init.extend_from_slice(&[0x60, 0x42, 0x60, 0x00, 0x55]); // PUSH1 0x42, PUSH1 0x00, SSTORE
init.extend_from_slice(&[
0x73, // PUSH20
0xde, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x01, // beneficiary address
]);
init.push(0xff); // SELFDESTRUCT
Bytes::from(init)
}
/// Creates init code for a simple contract that:
/// 1. Stores 0x42 at slot 0 during deployment
/// 2. On any call: selfdestructs to beneficiary
///
/// This simpler contract avoids complex branching logic.
fn selfdestruct_contract_init_code() -> Bytes {
// Runtime: just selfdestruct on any call
// PUSH20 <beneficiary>
// SELFDESTRUCT
let runtime = bytes!(
"73dead000000000000000000000000000000000001" // PUSH20 beneficiary
"ff" // SELFDESTRUCT
);
let runtime_len = runtime.len(); // 22 bytes
// Init code: SSTORE(0, 0x42), CODECOPY, RETURN
// Total init code before runtime = 17 bytes
let init_len: u8 = 17;
let mut init = Vec::new();
init.extend_from_slice(&[0x60, 0x42, 0x60, 0x00, 0x55]); // PUSH1 0x42, PUSH1 0x00, SSTORE
init.extend_from_slice(&[0x60, runtime_len as u8, 0x60, init_len, 0x60, 0x00, 0x39]); // CODECOPY
init.extend_from_slice(&[0x60, runtime_len as u8, 0x60, 0x00, 0xf3]); // RETURN
init.extend_from_slice(&runtime);
Bytes::from(init)
}
/// Tests SELFDESTRUCT behavior post-Dencun (Cancun+).
///
/// Post-Dencun (EIP-6780):
/// - SELFDESTRUCT only deletes contract if called in same tx as creation
/// - For existing contracts, SELFDESTRUCT only sends balance, code/storage persist
/// - The output state should NOT mark the account as destroyed
///
/// This test verifies:
/// 1. Contract deploys with storage
/// 2. SELFDESTRUCT in later tx does NOT delete code/storage
/// 3. Output state shows account is NOT destroyed
#[tokio::test]
async fn test_selfdestruct_post_dencun() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false);
let (mut nodes, _tasks, wallet) =
setup_engine::<EthereumNode>(1, cancun_spec(), false, tree_config, eth_payload_attributes)
.await?;
let mut node = nodes.pop().unwrap();
let signer = wallet.inner.clone();
let provider = ProviderBuilder::new()
.wallet(EthereumWallet::new(signer.clone()))
.connect_http(node.rpc_url());
// Deploy contract that stores 0x42 at slot 0 and selfdestructs on any call
let pending = provider
.send_transaction(deploy_tx(signer.address(), 0, selfdestruct_contract_init_code()))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Contract deployment should succeed");
let contract_address = receipt.contract_address.expect("Should have contract address");
// Consume the canonical notification for deployment block
let _ = node.canonical_stream.next().await;
// Trigger SELFDESTRUCT by calling the contract
let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 1)).await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Selfdestruct tx should succeed");
// Get the canonical notification for the selfdestruct block
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state: post-Dencun, account should NOT be destroyed
let account_state: Option<&BundleAccount> = execution_outcome.bundle.account(&contract_address);
assert!(
account_state.is_none() || !account_state.unwrap().was_destroyed(),
"Post-Dencun (EIP-6780): Account should NOT be destroyed when SELFDESTRUCT called on existing contract"
);
// Verify via RPC that code and storage persist
let code_after = provider.get_code_at(contract_address).await?;
assert!(!code_after.is_empty(), "Post-Dencun: Contract code should persist");
let slot0_after = provider.get_storage_at(contract_address, U256::ZERO).await?;
assert_eq!(slot0_after, U256::from(0x42), "Post-Dencun: Storage should persist");
// Send another transaction to the contract address in a new block.
// This tests cache behavior - if cache has stale data, execution would be incorrect.
// Post-Dencun: calling the contract should trigger SELFDESTRUCT again (but only transfer
// balance)
let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 2)).await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Second call to contract should succeed");
// Consume the canonical notification
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state still shows account NOT destroyed
let account_state: Option<&BundleAccount> = execution_outcome.bundle.account(&contract_address);
assert!(
account_state.is_none() || !account_state.unwrap().was_destroyed(),
"Post-Dencun: Account should still NOT be destroyed after second SELFDESTRUCT call"
);
// Verify code and storage still persist after the second call
let code_final = provider.get_code_at(contract_address).await?;
assert!(!code_final.is_empty(), "Post-Dencun: Contract code should still persist");
let slot0_final = provider.get_storage_at(contract_address, U256::ZERO).await?;
assert_eq!(slot0_final, U256::from(0x42), "Post-Dencun: Storage should still persist");
Ok(())
}
/// Tests SELFDESTRUCT in same transaction as creation (post-Dencun).
///
/// Post-Dencun (EIP-6780):
/// - SELFDESTRUCT during the same transaction as creation DOES delete the contract
/// - This is the exception to the rule that SELFDESTRUCT no longer deletes contracts
///
/// This test verifies:
/// 1. Contract selfdestructs during its constructor
/// 2. Contract is deleted (same-tx exception applies)
/// 3. No code or storage remains
/// 4. Since account never existed in DB before, bundle has no entry for it
#[tokio::test]
async fn test_selfdestruct_same_tx_post_dencun() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false);
let (mut nodes, _tasks, wallet) =
setup_engine::<EthereumNode>(1, cancun_spec(), false, tree_config, eth_payload_attributes)
.await?;
let mut node = nodes.pop().unwrap();
let signer = wallet.inner.clone();
let provider = ProviderBuilder::new()
.wallet(EthereumWallet::new(signer.clone()))
.connect_http(node.rpc_url());
// Deploy contract that selfdestructs during its constructor
let pending = provider
.send_transaction(deploy_tx(signer.address(), 0, selfdestruct_in_constructor_init_code()))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Contract deployment with selfdestruct should succeed");
// Calculate the contract address (CREATE uses sender + nonce)
let contract_address = signer.address().create(0);
// Get the canonical notification for the deployment block
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state: same-tx SELFDESTRUCT should destroy the account
let account_state: Option<&BundleAccount> = execution_outcome.bundle.account(&contract_address);
assert!(
account_state.is_none(),
"Post-Dencun same-tx: Account was created and selfdestructed in the same transaction, no trace in bundle state"
);
// Verify via RPC that code and storage are cleared
let code = provider.get_code_at(contract_address).await?;
assert!(code.is_empty(), "Post-Dencun same-tx: Contract code should be deleted");
let slot0 = provider.get_storage_at(contract_address, U256::ZERO).await?;
assert_eq!(slot0, U256::ZERO, "Post-Dencun same-tx: Storage should be cleared");
// Send ETH to the destroyed address in a new block to test cache behavior
let pending = provider
.send_transaction(transfer_tx(signer.address(), contract_address, 1, U256::from(1000)))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "ETH transfer to destroyed address should succeed");
// Consume the canonical notification
let _ = node.canonical_stream.next().await;
// Verify code is still empty and account received ETH
let code_final = provider.get_code_at(contract_address).await?;
assert!(code_final.is_empty(), "Post-Dencun same-tx: Contract code should remain deleted");
let balance = provider.get_balance(contract_address).await?;
assert_eq!(balance, U256::from(1000), "Post-Dencun same-tx: Account should have received ETH");
Ok(())
}
/// Tests SELFDESTRUCT behavior pre-Dencun (Shanghai).
///
/// Pre-Dencun:
/// - SELFDESTRUCT deletes contract code and storage regardless of when contract was created
/// - The output state MUST mark the account as destroyed
///
/// This test verifies:
/// 1. Contract deploys with storage
/// 2. SELFDESTRUCT deletes code and storage
/// 3. Output state shows account IS destroyed
#[tokio::test]
async fn test_selfdestruct_pre_dencun() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false);
let (mut nodes, _tasks, wallet) = setup_engine::<EthereumNode>(
1,
shanghai_spec(),
false,
tree_config,
eth_payload_attributes_shanghai,
)
.await?;
let mut node = nodes.pop().unwrap();
let signer = wallet.inner.clone();
let provider = ProviderBuilder::new()
.wallet(EthereumWallet::new(signer.clone()))
.connect_http(node.rpc_url());
// Deploy contract that stores 0x42 at slot 0 and selfdestructs on any call
let pending = provider
.send_transaction(deploy_tx(signer.address(), 0, selfdestruct_contract_init_code()))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Contract deployment should succeed");
let contract_address = receipt.contract_address.expect("Should have contract address");
// Consume the canonical notification for deployment block
let _ = node.canonical_stream.next().await;
// Trigger SELFDESTRUCT by calling the contract
let pending = provider.send_transaction(call_tx(signer.address(), contract_address, 1)).await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Selfdestruct tx should succeed");
// Get the canonical notification for the selfdestruct block
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state: pre-Dencun, account MUST be destroyed
let account_state: Option<&BundleAccount> = execution_outcome.bundle.account(&contract_address);
assert!(
account_state.is_some_and(|a: &BundleAccount| a.was_destroyed()),
"Pre-Dencun: Account MUST be marked as destroyed in output state"
);
// Verify via RPC that code and storage are cleared
let code_after = provider.get_code_at(contract_address).await?;
assert!(code_after.is_empty(), "Pre-Dencun: Contract code should be deleted");
let slot0_after = provider.get_storage_at(contract_address, U256::ZERO).await?;
assert_eq!(slot0_after, U256::ZERO, "Pre-Dencun: Storage should be cleared");
// Send ETH to the destroyed contract address in a new block.
// This tests cache behavior - the cache should correctly reflect the account was destroyed.
// Pre-Dencun: the contract no longer exists, so this is just a plain ETH transfer.
let pending = provider
.send_transaction(transfer_tx(signer.address(), contract_address, 2, U256::from(1000)))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "ETH transfer to destroyed contract address should succeed");
// Consume the canonical notification
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state shows the account exists (received ETH) but has no code
let account_state: Option<&BundleAccount> = execution_outcome.bundle.account(&contract_address);
// After receiving ETH, the account should exist with balance but no code
assert!(
account_state.is_some(),
"Pre-Dencun: Account should exist after receiving ETH (even though contract was destroyed)"
);
// Verify code is still empty (contract was destroyed, only ETH was received)
let code_final = provider.get_code_at(contract_address).await?;
assert!(code_final.is_empty(), "Pre-Dencun: Contract code should remain deleted");
// Verify storage is still cleared
let slot0_final = provider.get_storage_at(contract_address, U256::ZERO).await?;
assert_eq!(slot0_final, U256::ZERO, "Pre-Dencun: Storage should remain cleared");
// Verify the account now has the ETH balance we sent
let balance = provider.get_balance(contract_address).await?;
assert_eq!(balance, U256::from(1000), "Pre-Dencun: Account should have received ETH");
Ok(())
}
/// Tests SELFDESTRUCT in same transaction as creation, where account previously had ETH
/// (post-Dencun).
///
/// Post-Dencun (EIP-6780):
/// - The same-tx exception applies when the CONTRACT is created in that transaction
/// - Even if the address previously had ETH (as an EOA), deploying a contract there and
/// selfdestructing in the same tx DOES delete the contract
/// - The "created in same tx" refers to contract creation, not account existence
///
/// This test verifies:
/// 1. Send ETH to the future contract address (address has balance but no code)
/// 2. Deploy contract that selfdestructs during constructor to that address
/// 3. Contract is deleted (same-tx exception applies - contract was created this tx)
/// 4. Code and storage are cleared
/// 5. Since account existed in DB before (had ETH), bundle marks it as Destroyed
#[tokio::test]
async fn test_selfdestruct_same_tx_preexisting_account_post_dencun() -> eyre::Result<()> {
reth_tracing::init_test_tracing();
let tree_config = TreeConfig::default().without_prewarming(true).without_state_cache(false);
let (mut nodes, _tasks, wallet) =
setup_engine::<EthereumNode>(1, cancun_spec(), false, tree_config, eth_payload_attributes)
.await?;
let mut node = nodes.pop().unwrap();
let signer = wallet.inner.clone();
let provider = ProviderBuilder::new()
.wallet(EthereumWallet::new(signer.clone()))
.connect_http(node.rpc_url());
// Calculate where the contract will be deployed (CREATE uses sender + nonce)
// We'll use nonce 1 for deployment, so first send ETH with nonce 0
let future_contract_address = signer.address().create(1);
// Send ETH to the future contract address first (makes it a pre-existing account)
let pending = provider
.send_transaction(transfer_tx(
signer.address(),
future_contract_address,
0,
U256::from(1000),
))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "ETH transfer should succeed");
// Consume the canonical notification
let _ = node.canonical_stream.next().await;
// Verify the account exists and has balance
let balance_before = provider.get_balance(future_contract_address).await?;
assert_eq!(balance_before, U256::from(1000), "Account should have ETH before deployment");
// Now deploy contract that selfdestructs during its constructor to the same address
let pending = provider
.send_transaction(deploy_tx(signer.address(), 1, selfdestruct_in_constructor_init_code()))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "Contract deployment with selfdestruct should succeed");
// Verify deployment went to the expected address
assert_eq!(
receipt.contract_address,
Some(future_contract_address),
"Contract should be deployed to pre-computed address"
);
// Get the canonical notification for the deployment block
let notification = node.canonical_stream.next().await.unwrap();
let chain = notification.committed();
let execution_outcome = chain.execution_outcome();
// Verify the output state: same-tx exception DOES apply because contract was created this tx
// The account should be marked as destroyed. Since it had prior state (ETH balance),
// the bundle will contain it with status Destroyed and original_info set.
let account_state: Option<&BundleAccount> =
execution_outcome.bundle.account(&future_contract_address);
assert!(
account_state.is_some_and(|a| a.was_destroyed()),
"Post-Dencun same-tx with prior ETH: Account MUST be marked as destroyed"
);
// Verify via RPC that code and storage are cleared
let code = provider.get_code_at(future_contract_address).await?;
assert!(code.is_empty(), "Post-Dencun same-tx: Contract code should be deleted");
let slot0 = provider.get_storage_at(future_contract_address, U256::ZERO).await?;
assert_eq!(slot0, U256::ZERO, "Post-Dencun same-tx: Storage should be cleared");
// Balance should be zero (sent to beneficiary during SELFDESTRUCT)
let balance_after = provider.get_balance(future_contract_address).await?;
assert_eq!(
balance_after,
U256::ZERO,
"Post-Dencun same-tx: Balance should be zero (sent to beneficiary)"
);
// Send ETH to the destroyed address to verify cache behavior
let pending = provider
.send_transaction(transfer_tx(
signer.address(),
future_contract_address,
2,
U256::from(2000),
))
.await?;
node.advance_block().await?;
let receipt = pending.get_receipt().await?;
assert!(receipt.status(), "ETH transfer should succeed");
// Consume notification
let _ = node.canonical_stream.next().await;
// Verify the account received ETH and has no code (it's now just an EOA)
let balance_final = provider.get_balance(future_contract_address).await?;
assert_eq!(balance_final, U256::from(2000), "Account should have received ETH");
let code_final = provider.get_code_at(future_contract_address).await?;
assert!(code_final.is_empty(), "Code should remain empty after ETH transfer");
let slot0_final = provider.get_storage_at(future_contract_address, U256::ZERO).await?;
assert_eq!(slot0_final, U256::ZERO, "Storage should remain cleared");
Ok(())
}

View File

@@ -29,19 +29,6 @@ pub(crate) fn eth_payload_attributes(timestamp: u64) -> EthPayloadBuilderAttribu
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Helper function to create pre-Cancun (Shanghai) payload attributes.
/// No `parent_beacon_block_root` field.
pub(crate) fn eth_payload_attributes_shanghai(timestamp: u64) -> EthPayloadBuilderAttributes {
let attributes = PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: None,
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Advances node by producing blocks with random transactions.
pub(crate) async fn advance_with_random_transactions<Provider>(
node: &mut NodeHelperType<EthereumNode, Provider>,

View File

@@ -1,7 +1,5 @@
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, TxEnvFor};
use alloy_evm::{block::ExecutableTxParts, RecoveredTx};
use crate::{execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor};
use rayon::prelude::*;
use reth_primitives_traits::TxTy;
/// [`ConfigureEvm`] extension providing methods for executing payloads.
pub trait ConfigureEngineEvm<ExecutionData>: ConfigureEvm {
@@ -63,16 +61,11 @@ where
/// Iterator over executable transactions.
pub trait ExecutableTxIterator<Evm: ConfigureEvm>:
ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered = Self::Recovered>>
ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
{
/// HACK: for some reason, this duplicated AT is the only way to enforce the inner Recovered:
/// Send + Sync bound. Effectively alias for `Self::Tx::Recovered`.
type Recovered: RecoveredTx<TxTy<Evm::Primitives>> + Send + Sync;
}
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T
where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm, Recovered: Send + Sync>>,
impl<T, Evm: ConfigureEvm> ExecutableTxIterator<Evm> for T where
T: ExecutableTxTuple<Tx: ExecutableTxFor<Evm>>
{
type Recovered = <T::Tx as ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>>>::Recovered;
}

View File

@@ -6,7 +6,7 @@ use alloy_consensus::{BlockHeader, Header};
use alloy_eips::eip2718::WithEncoded;
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
use alloy_evm::{
block::{CommitChanges, ExecutableTxParts},
block::{CommitChanges, ExecutableTx},
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
};
use alloy_primitives::{Address, B256};
@@ -75,11 +75,9 @@ pub trait Executor<DB: Database>: Sized {
where
I: IntoIterator<Item = &'a RecoveredBlock<<Self::Primitives as NodePrimitives>::Block>>,
{
let blocks_iter = blocks.into_iter();
let capacity = blocks_iter.size_hint().0;
let mut results = Vec::with_capacity(capacity);
let mut results = Vec::new();
let mut first_block = None;
for block in blocks_iter {
for block in blocks {
if first_block.is_none() {
first_block = Some(block.header().number());
}
@@ -403,31 +401,49 @@ where
/// Conversions for executable transactions.
pub trait ExecutorTx<Executor: BlockExecutor> {
/// Converts the transaction into a tuple of [`TxEnvFor`] and [`Recovered`].
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>);
/// Converts the transaction into [`ExecutableTx`].
fn as_executable(&self) -> impl ExecutableTx<Executor>;
/// Converts the transaction into [`Recovered`].
fn into_recovered(self) -> Recovered<Executor::Transaction>;
}
impl<Executor: BlockExecutor> ExecutorTx<Executor>
for WithEncoded<Recovered<Executor::Transaction>>
{
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.to_tx_env(), self.1)
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
self.1
}
}
impl<Executor: BlockExecutor> ExecutorTx<Executor> for Recovered<Executor::Transaction> {
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Self) {
(self.to_tx_env(), self)
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Self {
self
}
}
impl<Executor> ExecutorTx<Executor>
for WithTxEnv<<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>>
impl<T, Executor> ExecutorTx<Executor>
for WithTxEnv<<<Executor as BlockExecutor>::Evm as Evm>::Tx, T>
where
Executor: BlockExecutor<Transaction: Clone>,
T: ExecutorTx<Executor> + Clone,
Executor: BlockExecutor,
<<Executor as BlockExecutor>::Evm as Evm>::Tx: Clone,
Self: RecoveredTx<Executor::Transaction>,
{
fn into_parts(self) -> (<Executor::Evm as Evm>::Tx, Recovered<Executor::Transaction>) {
(self.tx_env, Arc::unwrap_or_clone(self.tx))
fn as_executable(&self) -> impl ExecutableTx<Executor> {
self
}
fn into_recovered(self) -> Recovered<Executor::Transaction> {
Arc::unwrap_or_clone(self.tx).into_recovered()
}
}
@@ -463,11 +479,10 @@ where
&ExecutionResult<<<Self::Executor as BlockExecutor>::Evm as Evm>::HaltReason>,
) -> CommitChanges,
) -> Result<Option<u64>, BlockExecutionError> {
let (tx_env, tx) = tx.into_parts();
if let Some(gas_used) =
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
self.executor.execute_transaction_with_commit_condition(tx.as_executable(), f)?
{
self.transactions.push(tx);
self.transactions.push(tx.into_recovered());
Ok(Some(gas_used))
} else {
Ok(None)
@@ -594,20 +609,20 @@ where
}
}
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTxParts`] for
/// block executor.
/// A helper trait marking a 'static type that can be converted into an [`ExecutableTx`] for block
/// executor.
pub trait ExecutableTxFor<Evm: ConfigureEvm>:
ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
impl<T, Evm: ConfigureEvm> ExecutableTxFor<Evm> for T where
T: ExecutableTxParts<TxEnvFor<Evm>, TxTy<Evm::Primitives>> + RecoveredTx<TxTy<Evm::Primitives>>
T: ToTxEnv<TxEnvFor<Evm>> + RecoveredTx<TxTy<Evm::Primitives>>
{
}
/// A container for a transaction and a transaction environment.
#[derive(Debug)]
#[derive(Debug, Clone)]
pub struct WithTxEnv<TxEnv, T> {
/// The transaction environment for EVM.
pub tx_env: TxEnv,
@@ -615,12 +630,6 @@ pub struct WithTxEnv<TxEnv, T> {
pub tx: Arc<T>,
}
impl<TxEnv: Clone, T> Clone for WithTxEnv<TxEnv, T> {
fn clone(&self) -> Self {
Self { tx_env: self.tx_env.clone(), tx: self.tx.clone() }
}
}
impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
fn tx(&self) -> &Tx {
self.tx.tx()
@@ -631,11 +640,9 @@ impl<TxEnv, Tx, T: RecoveredTx<Tx>> RecoveredTx<Tx> for WithTxEnv<TxEnv, T> {
}
}
impl<TxEnv, T: RecoveredTx<Tx>, Tx> ExecutableTxParts<TxEnv, Tx> for WithTxEnv<TxEnv, T> {
type Recovered = Arc<T>;
fn into_parts(self) -> (TxEnv, Self::Recovered) {
(self.tx_env, self.tx)
impl<TxEnv: Clone, T> ToTxEnv<TxEnv> for WithTxEnv<TxEnv, T> {
fn to_tx_env(&self) -> TxEnv {
self.tx_env.clone()
}
}

Some files were not shown because too many files have changed in this diff Show More