mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
185 Commits
pr-21331
...
disable-pr
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d701d70dc1 | ||
|
|
53f922927a | ||
|
|
f1f3980d29 | ||
|
|
6946f26d77 | ||
|
|
f663d1d110 | ||
|
|
f4943abf73 | ||
|
|
102a6944ba | ||
|
|
1592e51d34 | ||
|
|
4280ccf470 | ||
|
|
05ab98107c | ||
|
|
49128ed28f | ||
|
|
f74e594292 | ||
|
|
e7d4a05e36 | ||
|
|
9382a4c713 | ||
|
|
28409558f9 | ||
|
|
5ef32726db | ||
|
|
60c3bef1e8 | ||
|
|
af96eeae56 | ||
|
|
5528aae8f6 | ||
|
|
83364aa2d6 | ||
|
|
749a742bcf | ||
|
|
2970624413 | ||
|
|
7e18aa4be8 | ||
|
|
9f8c22e2c3 | ||
|
|
3d699ac9c6 | ||
|
|
9be31d504d | ||
|
|
34cc65cfe6 | ||
|
|
6e161f0fc9 | ||
|
|
63a3e18404 | ||
|
|
7d10e791b2 | ||
|
|
a9b2c1d454 | ||
|
|
9127563914 | ||
|
|
a500fb22ba | ||
|
|
e869cd4670 | ||
|
|
de69654b73 | ||
|
|
8d28c4c8f2 | ||
|
|
bfe778ab51 | ||
|
|
e523a76fb8 | ||
|
|
cd12ae58f2 | ||
|
|
370a548f34 | ||
|
|
781128eece | ||
|
|
435d915422 | ||
|
|
3ec065295e | ||
|
|
e1bc6d0f08 | ||
|
|
29072639d6 | ||
|
|
f90b5c8a7f | ||
|
|
d4fa6806b7 | ||
|
|
63742ab4ae | ||
|
|
08122bc1ea | ||
|
|
83afaf1aa7 | ||
|
|
d72300c685 | ||
|
|
faf64c712e | ||
|
|
b3d532ce9d | ||
|
|
9d064be77e | ||
|
|
e3c256340e | ||
|
|
d0df549ddb | ||
|
|
7ccb43ea13 | ||
|
|
20f48b1e50 | ||
|
|
0470c65e6c | ||
|
|
9de1f0905e | ||
|
|
327a1a6681 | ||
|
|
b8f27b73ad | ||
|
|
7ec5ff6483 | ||
|
|
f98af4ad9f | ||
|
|
d8e912f66b | ||
|
|
0572c4e0ca | ||
|
|
67a7a1c2d1 | ||
|
|
2b1833576b | ||
|
|
5592c362d4 | ||
|
|
6beec25f43 | ||
|
|
19bf580f93 | ||
|
|
796ba6d5dc | ||
|
|
5307dfc22b | ||
|
|
f380ed1581 | ||
|
|
f7313c755c | ||
|
|
3bc2191590 | ||
|
|
320f2a6015 | ||
|
|
70bfdafd26 | ||
|
|
e9fe0283a9 | ||
|
|
92b8857625 | ||
|
|
2d71243cf6 | ||
|
|
732bf712aa | ||
|
|
0901c2ca8b | ||
|
|
2352158b3d | ||
|
|
1a98605ce6 | ||
|
|
8d37f76d23 | ||
|
|
2d9cf4c989 | ||
|
|
f5ca71d2fb | ||
|
|
8d58c98034 | ||
|
|
50e0591540 | ||
|
|
013dfdf8c8 | ||
|
|
effa0ab4c7 | ||
|
|
543a85e9f3 | ||
|
|
88eb0beeb2 | ||
|
|
747c0169a7 | ||
|
|
497985ca86 | ||
|
|
48a999a81b | ||
|
|
d53858b3e2 | ||
|
|
6aa91b0020 | ||
|
|
e0a0a0d5fb | ||
|
|
231292b58e | ||
|
|
42765890b5 | ||
|
|
8417ddc0e8 | ||
|
|
1ca62d0696 | ||
|
|
928bf37297 | ||
|
|
aa5b12af44 | ||
|
|
f12acf17e6 | ||
|
|
2e05cec84b | ||
|
|
9eaa5a6303 | ||
|
|
ba8c8354e5 | ||
|
|
af3601c65d | ||
|
|
bff11ab663 | ||
|
|
08cd1cbda6 | ||
|
|
e4e05e9ef9 | ||
|
|
c8245594bc | ||
|
|
ed40ce8c4c | ||
|
|
1e734936d8 | ||
|
|
11d9f38077 | ||
|
|
226ce14ca1 | ||
|
|
a6e1dea2d7 | ||
|
|
71ed68e944 | ||
|
|
adecbd7814 | ||
|
|
26a37f3c00 | ||
|
|
0bfa7fa5fa | ||
|
|
18bec10a0b | ||
|
|
1e33821e19 | ||
|
|
da92733be8 | ||
|
|
c41c8e6cae | ||
|
|
1ccc174e7b | ||
|
|
f1459fcf91 | ||
|
|
94235d64a8 | ||
|
|
7fe60017cf | ||
|
|
f9ec2fafa0 | ||
|
|
768a687189 | ||
|
|
b87cde5479 | ||
|
|
ab685579f0 | ||
|
|
c7faafd183 | ||
|
|
935a2cc056 | ||
|
|
507cf58db0 | ||
|
|
6cfd369d17 | ||
|
|
934f462d01 | ||
|
|
d4f28b02ff | ||
|
|
963bfeeeed | ||
|
|
adbe6d9da0 | ||
|
|
6d19c0ed8e | ||
|
|
4baf2baec4 | ||
|
|
0b5f79e8c9 | ||
|
|
afe164baca | ||
|
|
31fdbe914c | ||
|
|
6870747246 | ||
|
|
0ad8c772e1 | ||
|
|
5440d0d89a | ||
|
|
0eea4d76e9 | ||
|
|
8a1702cd74 | ||
|
|
7feb56d5f6 | ||
|
|
0aa922c4e8 | ||
|
|
ccff9a08f0 | ||
|
|
eb788cc7cf | ||
|
|
fb05a0654f | ||
|
|
d5a36dcc00 | ||
|
|
ffbef9e3cd | ||
|
|
820c112e8e | ||
|
|
9285f7eafc | ||
|
|
9a4c6d8a11 | ||
|
|
963c26550a | ||
|
|
3648483512 | ||
|
|
ab418642b4 | ||
|
|
decb56fae1 | ||
|
|
ee1ec8f9f0 | ||
|
|
d7bf87da52 | ||
|
|
dd0c6d279f | ||
|
|
c137ed836f | ||
|
|
a543752f7d | ||
|
|
b814893221 | ||
|
|
fcef82261d | ||
|
|
d3846d98a9 | ||
|
|
1f536cce65 | ||
|
|
0ddaf1b26c | ||
|
|
830cd5e355 | ||
|
|
f77d7d5983 | ||
|
|
a2237c534e | ||
|
|
1bd8fab887 | ||
|
|
22a68756c7 | ||
|
|
d99c0ffd62 | ||
|
|
ad476e2b5c |
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@@ -37,7 +37,7 @@ crates/storage/db/ @joshieDo
|
||||
crates/storage/errors/ @joshieDo
|
||||
crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tokio-util/ @mattsse
|
||||
|
||||
@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
|
||||
done
|
||||
|
||||
# Make sure we don't rebuild images on the CI jobs
|
||||
git apply ../.github/assets/hive/no_sim_build.diff
|
||||
git apply ../.github/scripts/hive/no_sim_build.diff
|
||||
go build .
|
||||
mv ./hive ../hive_assets/
|
||||
54
.github/workflows/docker-git.yml
vendored
54
.github/workflows/docker-git.yml
vendored
@@ -1,54 +0,0 @@
|
||||
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
|
||||
# images from this workflow will be tagged with the git sha of the branch used
|
||||
# and will NOT tag it as `latest`.
|
||||
|
||||
name: docker-git
|
||||
|
||||
on:
|
||||
workflow_dispatch: {}
|
||||
|
||||
env:
|
||||
REPO_NAME: ${{ github.repository_owner }}/reth
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
GIT_SHA: ${{ github.sha }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build and push
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: 'Build and push the git-sha-tagged reth image'
|
||||
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
|
||||
- name: 'Build and push the git-sha-tagged op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
65
.github/workflows/docker-nightly.yml
vendored
65
.github/workflows/docker-nightly.yml
vendored
@@ -1,65 +0,0 @@
|
||||
# Publishes the nightly Docker image.
|
||||
|
||||
name: docker-nightly
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
env:
|
||||
REPO_NAME: ${{ github.repository_owner }}/reth
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
name: build and push
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: 'Build and push the nightly reth image'
|
||||
command: 'make PROFILE=maxperf docker-build-push-nightly'
|
||||
- name: 'Build and push the nightly edge profiling reth image'
|
||||
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
|
||||
- name: 'Build and push the nightly profiling reth image'
|
||||
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
|
||||
- name: 'Build and push the nightly op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
|
||||
- name: 'Build and push the nightly edge profiling op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
|
||||
- name: 'Build and push the nightly profiling op-reth image'
|
||||
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Remove bloatware
|
||||
uses: laverdet/remove-bloatware@v1.0.0
|
||||
with:
|
||||
docker: true
|
||||
lang: rust
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
155
.github/workflows/docker.yml
vendored
155
.github/workflows/docker.yml
vendored
@@ -1,4 +1,9 @@
|
||||
# Publishes the Docker image.
|
||||
# Publishes Docker images.
|
||||
#
|
||||
# Triggers:
|
||||
# - Push tag v*: builds release (RC or latest)
|
||||
# - Schedule: builds nightly + profiling
|
||||
# - Manual: builds git-sha or nightly
|
||||
|
||||
name: docker
|
||||
|
||||
@@ -6,84 +11,94 @@ on:
|
||||
push:
|
||||
tags:
|
||||
- v*
|
||||
|
||||
env:
|
||||
IMAGE_NAME: ${{ github.repository_owner }}/reth
|
||||
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
|
||||
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
schedule:
|
||||
- cron: "0 1 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
build_type:
|
||||
description: "Build type"
|
||||
required: true
|
||||
type: choice
|
||||
options:
|
||||
- git-sha
|
||||
- nightly
|
||||
default: git-sha
|
||||
dry_run:
|
||||
description: "Skip pushing images (dry run)"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
build-rc:
|
||||
if: contains(github.ref, '-rc')
|
||||
name: build and push as release candidate
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: "Build and push reth image"
|
||||
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
|
||||
- name: "Build and push op-reth image"
|
||||
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
|
||||
build:
|
||||
if: ${{ !contains(github.ref, '-rc') }}
|
||||
name: build and push as latest
|
||||
name: Build Docker images
|
||||
runs-on: ubuntu-24.04
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
build:
|
||||
- name: "Build and push reth image"
|
||||
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
|
||||
- name: "Build and push op-reth image"
|
||||
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
|
||||
- name: Set up Depot CLI
|
||||
uses: depot/setup-action@v1
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v3
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Get git info for vergen
|
||||
id: git
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
- name: Log in to Docker
|
||||
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
|
||||
echo "dirty=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Determine build parameters
|
||||
id: params
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
- name: Set up Docker builder
|
||||
run: |
|
||||
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
|
||||
docker buildx create --use --name cross-builder
|
||||
- name: Build and push ${{ matrix.build.name }}
|
||||
run: ${{ matrix.build.command }}
|
||||
REGISTRY="ghcr.io/${{ github.repository_owner }}"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
|
||||
|
||||
# Add 'latest' tag for non-RC releases
|
||||
if [[ ! "$VERSION" =~ -rc ]]; then
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
# git-sha build
|
||||
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
uses: depot/bake-action@v1
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
with:
|
||||
project: ${{ vars.DEPOT_PROJECT_ID }}
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
|
||||
21
.github/workflows/e2e.yml
vendored
21
.github/workflows/e2e.yml
vendored
@@ -44,3 +44,24 @@ jobs:
|
||||
--exclude 'op-reth' \
|
||||
--exclude 'reth' \
|
||||
-E 'binary(e2e_testsuite)'
|
||||
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run RocksDB e2e tests
|
||||
run: |
|
||||
cargo nextest run \
|
||||
--locked --features "edge" \
|
||||
-p reth-e2e-test-utils \
|
||||
-E 'binary(rocksdb)'
|
||||
|
||||
10
.github/workflows/hive.yml
vendored
10
.github/workflows/hive.yml
vendored
@@ -58,11 +58,11 @@ jobs:
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ./hive_assets
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
|
||||
- name: Build hive assets
|
||||
if: steps.cache-hive.outputs.cache-hit != 'true'
|
||||
run: .github/assets/hive/build_simulators.sh
|
||||
run: .github/scripts/hive/build_simulators.sh
|
||||
|
||||
- name: Load cached Docker images
|
||||
if: steps.cache-hive.outputs.cache-hit == 'true'
|
||||
@@ -213,7 +213,7 @@ jobs:
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
run: .github/assets/hive/load_images.sh
|
||||
run: .github/scripts/hive/load_images.sh
|
||||
|
||||
- name: Move hive binary
|
||||
run: |
|
||||
@@ -241,11 +241,11 @@ jobs:
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
|
||||
|
||||
- name: Print simulator output
|
||||
if: ${{ failure() }}
|
||||
|
||||
10
.github/workflows/integration.yml
vendored
10
.github/workflows/integration.yml
vendored
@@ -22,7 +22,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }}
|
||||
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
env:
|
||||
@@ -30,13 +30,17 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
network: ["ethereum", "optimism"]
|
||||
storage: ["stable", "edge"]
|
||||
exclude:
|
||||
- network: optimism
|
||||
storage: edge
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Install Geth
|
||||
run: .github/assets/install_geth.sh
|
||||
run: .github/scripts/install_geth.sh
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
@@ -46,7 +50,7 @@ jobs:
|
||||
name: Run tests
|
||||
run: |
|
||||
cargo nextest run \
|
||||
--locked --features "asm-keccak ${{ matrix.network }}" \
|
||||
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
|
||||
--workspace --exclude ef-tests \
|
||||
-E "kind(test) and not binary(e2e_testsuite)"
|
||||
- if: matrix.network == 'optimism'
|
||||
|
||||
2
.github/workflows/label-pr.yml
vendored
2
.github/workflows/label-pr.yml
vendored
@@ -19,5 +19,5 @@ jobs:
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/assets/label_pr.js')
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
await label_pr({github, context})
|
||||
|
||||
4
.github/workflows/lint.yml
vendored
4
.github/workflows/lint.yml
vendored
@@ -76,7 +76,7 @@ jobs:
|
||||
- name: Run Wasm checks
|
||||
run: |
|
||||
sudo apt update && sudo apt install gcc-multilib
|
||||
.github/assets/check_wasm.sh
|
||||
.github/scripts/check_wasm.sh
|
||||
|
||||
riscv:
|
||||
runs-on: depot-ubuntu-latest
|
||||
@@ -94,7 +94,7 @@ jobs:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- name: Run RISC-V checks
|
||||
run: .github/assets/check_rv32imac.sh
|
||||
run: .github/scripts/check_rv32imac.sh
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
|
||||
2
.github/workflows/prepare-reth.yml
vendored
2
.github/workflows/prepare-reth.yml
vendored
@@ -43,7 +43,7 @@ jobs:
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: .github/assets/hive/Dockerfile
|
||||
file: .github/scripts/hive/Dockerfile
|
||||
tags: ${{ inputs.image_tag }}
|
||||
outputs: type=docker,dest=./artifacts/reth_image.tar
|
||||
build-args: |
|
||||
|
||||
11
CLAUDE.md
11
CLAUDE.md
@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
|
||||
|
||||
2. **Linting**: Run clippy with all features
|
||||
```bash
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
|
||||
```
|
||||
|
||||
3. **Testing**: Use nextest for faster test execution
|
||||
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
|
||||
Before submitting changes, ensure:
|
||||
|
||||
1. **Format Check**: `cargo +nightly fmt --all --check`
|
||||
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
|
||||
2. **Clippy**: No warnings
|
||||
3. **Tests Pass**: All unit and integration tests
|
||||
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
|
||||
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
|
||||
|
||||
|
||||
### Opening PRs against <https://github.com/paradigmxyz/reth>
|
||||
|
||||
Label PRs appropriately, first check the available labels and then apply the relevant ones:
|
||||
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
}
|
||||
```
|
||||
|
||||
5. **Run checks**:
|
||||
5. **Run checks** (IMPORTANT!):
|
||||
```bash
|
||||
cargo +nightly fmt --all
|
||||
cargo clippy --all-features
|
||||
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
|
||||
cargo test -p reth-discv4
|
||||
```
|
||||
|
||||
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
cargo +nightly fmt --all
|
||||
|
||||
# Run lints
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
|
||||
cargo +nightly clippy --workspace --all-features
|
||||
|
||||
# Run tests
|
||||
cargo nextest run --workspace
|
||||
|
||||
942
Cargo.lock
generated
942
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
66
Cargo.toml
66
Cargo.toml
@@ -77,6 +77,10 @@ members = [
|
||||
"crates/optimism/cli",
|
||||
"crates/optimism/consensus",
|
||||
"crates/optimism/evm/",
|
||||
"crates/optimism/examples/custom-node",
|
||||
"crates/optimism/examples/engine-api-access",
|
||||
"crates/optimism/examples/exex-hello-world",
|
||||
"crates/optimism/examples/op-db-access",
|
||||
"crates/optimism/flashblocks/",
|
||||
"crates/optimism/hardforks/",
|
||||
"crates/optimism/node/",
|
||||
@@ -145,7 +149,6 @@ members = [
|
||||
"examples/beacon-api-sse/",
|
||||
"examples/bsc-p2p",
|
||||
"examples/custom-dev-node/",
|
||||
"examples/custom-node/",
|
||||
"examples/custom-engine-types/",
|
||||
"examples/custom-evm/",
|
||||
"examples/custom-hardforks/",
|
||||
@@ -154,10 +157,7 @@ members = [
|
||||
"examples/custom-payload-builder/",
|
||||
"examples/custom-rlpx-subprotocol",
|
||||
"examples/custom-rpc-middleware",
|
||||
"examples/custom-node",
|
||||
"examples/db-access",
|
||||
"examples/engine-api-access",
|
||||
"examples/exex-hello-world",
|
||||
"examples/exex-subscription",
|
||||
"examples/exex-test",
|
||||
"examples/full-contract-state",
|
||||
@@ -168,7 +168,6 @@ members = [
|
||||
"examples/node-builder-api/",
|
||||
"examples/node-custom-rpc/",
|
||||
"examples/node-event-hooks/",
|
||||
"examples/op-db-access/",
|
||||
"examples/polygon-p2p/",
|
||||
"examples/rpc-db/",
|
||||
"examples/precompile-cache/",
|
||||
@@ -481,18 +480,18 @@ revm-primitives = { version = "22.0.0", default-features = false }
|
||||
revm-interpreter = { version = "32.0.0", default-features = false }
|
||||
revm-database-interface = { version = "9.0.0", default-features = false }
|
||||
op-revm = { version = "15.0.0", default-features = false }
|
||||
revm-inspectors = "0.34.0"
|
||||
revm-inspectors = "0.34.1"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.4"
|
||||
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.4", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-dyn-abi = "1.5.2"
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-sol-macro = "1.5.0"
|
||||
alloy-sol-types = { version = "1.5.0", default-features = false }
|
||||
alloy-trie = { version = "0.9.1", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
@@ -561,7 +560,7 @@ humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.11.2"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.2", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
@@ -588,15 +587,15 @@ tracing-appender = "0.2"
|
||||
url = { version = "2.3", default-features = false }
|
||||
zstd = "0.13"
|
||||
byteorder = "1"
|
||||
mini-moka = "0.10"
|
||||
fixed-cache = { version = "0.1.7", features = ["stats"] }
|
||||
moka = "0.12"
|
||||
tar-no-std = { version = "0.3.2", default-features = false }
|
||||
miniz_oxide = { version = "0.8.4", default-features = false }
|
||||
tar-no-std = { version = "0.4.2", default-features = false }
|
||||
miniz_oxide = { version = "0.9.0", default-features = false }
|
||||
chrono = "0.4.41"
|
||||
|
||||
# metrics
|
||||
metrics = "0.24.0"
|
||||
metrics-derive = "0.1"
|
||||
metrics-derive = "0.1.1"
|
||||
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
|
||||
metrics-process = "2.1.0"
|
||||
metrics-util = { default-features = false, version = "0.20.0" }
|
||||
@@ -608,7 +607,7 @@ quote = "1.0"
|
||||
# tokio
|
||||
tokio = { version = "1.44.2", default-features = false }
|
||||
tokio-stream = "0.1.11"
|
||||
tokio-tungstenite = "0.26.2"
|
||||
tokio-tungstenite = "0.28.0"
|
||||
tokio-util = { version = "0.7.4", features = ["codec"] }
|
||||
|
||||
# async
|
||||
@@ -621,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
|
||||
hyper = "1.3"
|
||||
hyper-util = "0.1.5"
|
||||
pin-project = "1.0.12"
|
||||
reqwest = { version = "0.12", default-features = false }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
tracing-futures = "0.2"
|
||||
tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
@@ -641,7 +640,6 @@ jsonrpsee-types = "0.26.0"
|
||||
http = "1.0"
|
||||
http-body = "1.0"
|
||||
http-body-util = "0.1.2"
|
||||
jsonwebtoken = "9"
|
||||
proptest-arbitrary-interop = "0.1.0"
|
||||
|
||||
# crypto
|
||||
@@ -655,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
|
||||
c-kzg = "2.1.5"
|
||||
|
||||
# config
|
||||
toml = "0.8"
|
||||
toml = "0.9"
|
||||
|
||||
# rocksdb
|
||||
rocksdb = { version = "0.24" }
|
||||
@@ -671,19 +669,19 @@ tracing-opentelemetry = "0.32"
|
||||
# misc-testing
|
||||
arbitrary = "1.3"
|
||||
assert_matches = "1.5.0"
|
||||
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
|
||||
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
|
||||
insta = "1.41"
|
||||
proptest = "1.7"
|
||||
proptest-derive = "0.5"
|
||||
proptest-derive = "0.7"
|
||||
similar-asserts = { version = "1.5.0", features = ["serde"] }
|
||||
tempfile = "3.20"
|
||||
test-fuzz = "7"
|
||||
rstest = "0.24.0"
|
||||
rstest = "0.26.1"
|
||||
test-case = "3"
|
||||
|
||||
# ssz encoding
|
||||
ethereum_ssz = "0.9.0"
|
||||
ethereum_ssz_derive = "0.9.0"
|
||||
ethereum_ssz = "0.10.1"
|
||||
ethereum_ssz_derive = "0.10.1"
|
||||
|
||||
# allocators
|
||||
jemalloc_pprof = { version = "0.8", default-features = false }
|
||||
@@ -695,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
|
||||
aes = "0.8.1"
|
||||
ahash = "0.8"
|
||||
anyhow = "1.0"
|
||||
bindgen = { version = "0.71", default-features = false }
|
||||
block-padding = "0.3.2"
|
||||
bindgen = { version = "0.72", default-features = false }
|
||||
block-padding = "0.3"
|
||||
cc = "1.2.15"
|
||||
cipher = "0.4.3"
|
||||
comfy-table = "7.0"
|
||||
concat-kdf = "0.1.0"
|
||||
crossbeam-channel = "0.5.13"
|
||||
crossterm = "0.28.0"
|
||||
crossterm = "0.29.0"
|
||||
csv = "1.3.0"
|
||||
ctrlc = "3.4"
|
||||
ctr = "0.9.2"
|
||||
@@ -715,7 +713,7 @@ hmac = "0.12.1"
|
||||
human_bytes = "0.4.1"
|
||||
indexmap = "2"
|
||||
interprocess = "2.2.0"
|
||||
lz4_flex = { version = "0.11", default-features = false }
|
||||
lz4_flex = { version = "0.12", default-features = false }
|
||||
memmap2 = "0.9.4"
|
||||
mev-share-sse = { version = "0.5.0", default-features = false }
|
||||
num-traits = "0.2.15"
|
||||
@@ -723,17 +721,17 @@ page_size = "0.6.0"
|
||||
parity-scale-codec = "3.2.1"
|
||||
plain_hasher = "0.2"
|
||||
pretty_assertions = "1.4"
|
||||
ratatui = { version = "0.29", default-features = false }
|
||||
ringbuffer = "0.15.0"
|
||||
ratatui = { version = "0.30", default-features = false }
|
||||
ringbuffer = "0.16.0"
|
||||
rmp-serde = "1.3"
|
||||
roaring = "0.10.2"
|
||||
roaring = "0.11.3"
|
||||
rolling-file = "0.2.0"
|
||||
sha3 = "0.10.5"
|
||||
snap = "1.1.1"
|
||||
socket2 = { version = "0.5", default-features = false }
|
||||
sysinfo = { version = "0.33", default-features = false }
|
||||
socket2 = { version = "0.6", default-features = false }
|
||||
sysinfo = { version = "0.38", default-features = false }
|
||||
tracing-journald = "0.3"
|
||||
tracing-logfmt = "0.3.3"
|
||||
tracing-logfmt = "=0.3.5"
|
||||
tracing-samply = "0.1"
|
||||
tracing-subscriber = { version = "0.3", default-features = false }
|
||||
tracing-tracy = "0.11"
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
# This image is meant to enable cross-architecture builds.
|
||||
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
|
||||
# locatable in `./dist/bin/$TARGETARCH`
|
||||
FROM --platform=$TARGETPLATFORM ubuntu:22.04
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
# Filled by docker buildx
|
||||
ARG TARGETARCH
|
||||
|
||||
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/reth"]
|
||||
84
Dockerfile.depot
Normal file
84
Dockerfile.depot
Normal file
@@ -0,0 +1,84 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
|
||||
# Usage:
|
||||
# reth: --build-arg BINARY=reth
|
||||
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
|
||||
|
||||
FROM rust:1 AS builder
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
RUN apt-get update && apt-get install -y libclang-dev pkg-config
|
||||
|
||||
# Install sccache for compilation caching
|
||||
RUN cargo install sccache --locked
|
||||
ENV RUSTC_WRAPPER=sccache
|
||||
ENV SCCACHE_DIR=/sccache
|
||||
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
|
||||
|
||||
# Binary to build (reth or op-reth)
|
||||
ARG BINARY=reth
|
||||
|
||||
# Manifest path for the binary
|
||||
ARG MANIFEST_PATH=bin/reth
|
||||
|
||||
# Build profile, release by default
|
||||
ARG BUILD_PROFILE=release
|
||||
ENV BUILD_PROFILE=$BUILD_PROFILE
|
||||
|
||||
# Extra Cargo flags
|
||||
ARG RUSTFLAGS=""
|
||||
ENV RUSTFLAGS="$RUSTFLAGS"
|
||||
|
||||
# Extra Cargo features
|
||||
ARG FEATURES=""
|
||||
ENV FEATURES=$FEATURES
|
||||
|
||||
# Git info for vergen (since .git is excluded from Docker context)
|
||||
ARG VERGEN_GIT_SHA=""
|
||||
ARG VERGEN_GIT_DESCRIBE=""
|
||||
ARG VERGEN_GIT_DIRTY="false"
|
||||
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
|
||||
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
|
||||
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
|
||||
|
||||
# Build application
|
||||
COPY --exclude=.git . .
|
||||
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
|
||||
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
|
||||
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
|
||||
|
||||
RUN sccache --show-stats || true
|
||||
|
||||
# Copy binary to a known location (ARG not resolved in COPY)
|
||||
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
|
||||
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
|
||||
cp /app/target/release/$BINARY /app/binary
|
||||
|
||||
FROM ubuntu:24.04 AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
# Binary name for entrypoint
|
||||
ARG BINARY=reth
|
||||
|
||||
# Install runtime dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Copy binary from build stage and create canonical symlink for entrypoint
|
||||
COPY --from=builder /app/binary /usr/local/bin/
|
||||
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
|
||||
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
|
||||
chmod +x /usr/local/bin/$BINARY
|
||||
|
||||
# Copy licenses
|
||||
COPY LICENSE-* ./
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/reth-binary"]
|
||||
@@ -1,15 +0,0 @@
|
||||
# This image is meant to enable cross-architecture builds.
|
||||
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
|
||||
# locatable in `./dist/bin/$TARGETARCH`
|
||||
FROM --platform=$TARGETPLATFORM ubuntu:22.04
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
|
||||
|
||||
# Filled by docker buildx
|
||||
ARG TARGETARCH
|
||||
|
||||
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
|
||||
|
||||
EXPOSE 30303 30303/udp 9001 8545 8546
|
||||
ENTRYPOINT ["/usr/local/bin/op-reth"]
|
||||
134
Makefile
134
Makefile
@@ -35,9 +35,6 @@ EEST_TESTS_TAG := v4.5.0
|
||||
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
|
||||
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
|
||||
|
||||
# The docker image name
|
||||
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
|
||||
|
||||
##@ Help
|
||||
|
||||
.PHONY: help
|
||||
@@ -242,137 +239,6 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
|
||||
--features "$(FEATURES)" \
|
||||
--profile "$(PROFILE)"
|
||||
|
||||
##@ Docker
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push
|
||||
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
|
||||
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push-git-sha
|
||||
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
|
||||
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: docker-build-push-latest
|
||||
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
|
||||
$(call docker_build_push,$(GIT_TAG),latest)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: docker-build-push-nightly
|
||||
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call docker_build_push,nightly,nightly)
|
||||
|
||||
.PHONY: docker-build-push-nightly-edge-profiling
|
||||
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
|
||||
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
|
||||
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define docker_build_push
|
||||
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
|
||||
|
||||
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
|
||||
|
||||
docker buildx build --file ./Dockerfile.cross . \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(1) \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(2) \
|
||||
--provenance=false \
|
||||
--push
|
||||
endef
|
||||
|
||||
##@ Optimism docker
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push
|
||||
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
|
||||
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push-git-sha
|
||||
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
|
||||
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --driver docker-container --name cross-builder`
|
||||
.PHONY: op-docker-build-push-latest
|
||||
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
|
||||
$(call op_docker_build_push,$(GIT_TAG),latest)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: op-docker-build-push-nightly
|
||||
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call op_docker_build_push,nightly,nightly)
|
||||
|
||||
.PHONY: op-docker-build-push-nightly-edge-profiling
|
||||
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
|
||||
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
|
||||
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: docker-build-push-nightly-profiling
|
||||
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
|
||||
$(call docker_build_push,nightly-profiling,nightly-profiling)
|
||||
|
||||
# Note: This requires a buildx builder with emulation support. For example:
|
||||
#
|
||||
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
|
||||
# `docker buildx create --use --name cross-builder`
|
||||
.PHONY: op-docker-build-push-nightly-profiling
|
||||
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
|
||||
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
|
||||
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define op_docker_build_push
|
||||
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
|
||||
|
||||
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
|
||||
|
||||
docker buildx build --file ./DockerfileOp.cross . \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(1) \
|
||||
--tag $(DOCKER_IMAGE_NAME):$(2) \
|
||||
--provenance=false \
|
||||
--push
|
||||
endef
|
||||
|
||||
##@ Other
|
||||
|
||||
.PHONY: clean
|
||||
|
||||
@@ -56,7 +56,7 @@ ctrlc.workspace = true
|
||||
shlex.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
nix = { version = "0.31", features = ["signal", "process"] }
|
||||
|
||||
[features]
|
||||
default = ["jemalloc"]
|
||||
|
||||
@@ -274,10 +274,10 @@ impl Args {
|
||||
/// Get the default RPC URL for a given chain
|
||||
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
|
||||
match chain.id() {
|
||||
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
|
||||
8453 => "https://base.reth.rs/rpc", // base
|
||||
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
|
||||
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
|
||||
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
|
||||
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
|
||||
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
|
||||
|
||||
# reqwest
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
# tower
|
||||
tower.workspace = true
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
|
||||
@@ -22,29 +22,6 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
@@ -237,50 +214,3 @@ const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@
|
||||
//! This command fetches transactions from existing blocks and packs them into a single
|
||||
//! large block using the `testing_buildBlockV1` RPC endpoint.
|
||||
|
||||
use crate::authenticated_transport::AuthenticatedTransportConnect;
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
|
||||
};
|
||||
use alloy_eips::{BlockNumberOrTag, Typed2718};
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -130,13 +132,24 @@ impl<S: TransactionSource> TransactionCollector<S> {
|
||||
/// Collect transactions starting from the given block number.
|
||||
///
|
||||
/// Skips blob transactions (type 3) and collects until target gas is reached.
|
||||
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
|
||||
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
|
||||
let mut transactions: Vec<Bytes> = Vec::new();
|
||||
/// Returns a `CollectionResult` with transactions, gas info, and next block.
|
||||
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
|
||||
self.collect_gas(start_block, self.target_gas).await
|
||||
}
|
||||
|
||||
/// Collect transactions up to a specific gas target.
|
||||
///
|
||||
/// This is used both for initial collection and for retry top-ups.
|
||||
pub async fn collect_gas(
|
||||
&self,
|
||||
start_block: u64,
|
||||
gas_target: u64,
|
||||
) -> eyre::Result<CollectionResult> {
|
||||
let mut transactions: Vec<RawTransaction> = Vec::new();
|
||||
let mut total_gas: u64 = 0;
|
||||
let mut current_block = start_block;
|
||||
|
||||
while total_gas < self.target_gas {
|
||||
while total_gas < gas_target {
|
||||
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
|
||||
else {
|
||||
warn!(block = current_block, "Block not found, stopping");
|
||||
@@ -149,12 +162,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
|
||||
continue;
|
||||
}
|
||||
|
||||
if total_gas + tx.gas_used <= self.target_gas {
|
||||
transactions.push(tx.raw);
|
||||
if total_gas + tx.gas_used <= gas_target {
|
||||
total_gas += tx.gas_used;
|
||||
transactions.push(tx);
|
||||
}
|
||||
|
||||
if total_gas >= self.target_gas {
|
||||
if total_gas >= gas_target {
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -162,7 +175,7 @@ impl<S: TransactionSource> TransactionCollector<S> {
|
||||
current_block += 1;
|
||||
|
||||
// Stop early if remaining gas is under 1M (close enough to target)
|
||||
let remaining_gas = self.target_gas.saturating_sub(total_gas);
|
||||
let remaining_gas = gas_target.saturating_sub(total_gas);
|
||||
if remaining_gas < 1_000_000 {
|
||||
break;
|
||||
}
|
||||
@@ -170,12 +183,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
|
||||
|
||||
info!(
|
||||
total_txs = transactions.len(),
|
||||
total_gas,
|
||||
gas_sent = total_gas,
|
||||
next_block = current_block,
|
||||
"Finished collecting transactions"
|
||||
);
|
||||
|
||||
Ok((transactions, total_gas, current_block))
|
||||
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -202,13 +215,26 @@ pub struct Command {
|
||||
jwt_secret: std::path::PathBuf,
|
||||
|
||||
/// Target gas to pack into the block.
|
||||
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
|
||||
target_gas: u64,
|
||||
|
||||
/// Starting block number to fetch transactions from.
|
||||
/// If not specified, starts from the engine's latest block.
|
||||
/// Block number to start fetching transactions from (required).
|
||||
///
|
||||
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
|
||||
/// The command collects transactions from historical blocks starting at this number
|
||||
/// to pack into large blocks.
|
||||
///
|
||||
/// How to determine this value:
|
||||
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
|
||||
/// - If gas limit ramping has already been performed: use the block number that was the chain
|
||||
/// tip BEFORE ramping began (you must track this yourself)
|
||||
///
|
||||
/// Using a block after ramping started will cause transaction collection to fail
|
||||
/// because those blocks contain synthetic transactions that cannot be replayed.
|
||||
#[arg(long, value_name = "FROM_BLOCK")]
|
||||
from_block: Option<u64>,
|
||||
from_block: u64,
|
||||
|
||||
/// Execute the payload (call newPayload + forkchoiceUpdated).
|
||||
/// If false, only builds the payload and prints it.
|
||||
@@ -237,6 +263,80 @@ struct BuiltPayload {
|
||||
envelope: ExecutionPayloadEnvelopeV4,
|
||||
block_hash: B256,
|
||||
timestamp: u64,
|
||||
/// The actual gas used in the built block.
|
||||
gas_used: u64,
|
||||
}
|
||||
|
||||
/// Result of collecting transactions from blocks.
|
||||
#[derive(Debug)]
|
||||
pub struct CollectionResult {
|
||||
/// Collected transactions with their gas info.
|
||||
pub transactions: Vec<RawTransaction>,
|
||||
/// Total gas sent (sum of historical `gas_used` for all collected txs).
|
||||
pub gas_sent: u64,
|
||||
/// Next block number to continue collecting from.
|
||||
pub next_block: u64,
|
||||
}
|
||||
|
||||
/// Constants for retry logic.
|
||||
const MAX_BUILD_RETRIES: u32 = 5;
|
||||
/// Maximum retries for fetching a transaction batch.
|
||||
const MAX_FETCH_RETRIES: u32 = 5;
|
||||
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
|
||||
const MIN_TARGET_SLACK: u64 = 1_000_000;
|
||||
/// Maximum gas to request in retries (10x target as safety cap).
|
||||
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
|
||||
|
||||
/// Fetches a batch of transactions with retry logic.
|
||||
///
|
||||
/// Returns `None` if all retries are exhausted.
|
||||
async fn fetch_batch_with_retry<S: TransactionSource>(
|
||||
collector: &TransactionCollector<S>,
|
||||
block: u64,
|
||||
) -> Option<CollectionResult> {
|
||||
for attempt in 1..=MAX_FETCH_RETRIES {
|
||||
match collector.collect(block).await {
|
||||
Ok(result) => return Some(result),
|
||||
Err(e) => {
|
||||
if attempt == MAX_FETCH_RETRIES {
|
||||
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
|
||||
return None;
|
||||
}
|
||||
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
|
||||
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Outcome of a build attempt check.
|
||||
enum RetryOutcome {
|
||||
/// Payload is close enough to target gas.
|
||||
Success,
|
||||
/// Max retries reached, accept what we have.
|
||||
MaxRetries,
|
||||
/// Need more transactions with the specified gas amount.
|
||||
NeedMore(u64),
|
||||
}
|
||||
|
||||
/// Buffer for receiving transaction batches from the fetcher.
|
||||
///
|
||||
/// This abstracts over the channel to allow the main loop to request
|
||||
/// batches on demand, including for retries.
|
||||
struct TxBuffer {
|
||||
receiver: mpsc::Receiver<CollectionResult>,
|
||||
}
|
||||
|
||||
impl TxBuffer {
|
||||
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
|
||||
Self { receiver }
|
||||
}
|
||||
|
||||
/// Take the next available batch from the fetcher.
|
||||
async fn take_batch(&mut self) -> Option<CollectionResult> {
|
||||
self.receiver.recv().await
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -284,7 +384,7 @@ impl Command {
|
||||
format!("Failed to create output directory: {:?}", self.output_dir)
|
||||
})?;
|
||||
|
||||
let start_block = self.from_block.unwrap_or(parent_number);
|
||||
let start_block = self.from_block;
|
||||
|
||||
// Use pipelined execution when generating multiple payloads
|
||||
if self.count > 1 {
|
||||
@@ -297,19 +397,20 @@ impl Command {
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
// Single payload - collect transactions and build
|
||||
// Single payload - collect transactions and build with retry
|
||||
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
|
||||
let collector = TransactionCollector::new(tx_source, self.target_gas);
|
||||
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
|
||||
let result = collector.collect(start_block).await?;
|
||||
|
||||
if transactions.is_empty() {
|
||||
if result.transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected"));
|
||||
}
|
||||
|
||||
self.execute_sequential(
|
||||
self.execute_sequential_with_retry(
|
||||
&auth_provider,
|
||||
&testing_provider,
|
||||
transactions,
|
||||
&collector,
|
||||
result,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
@@ -320,32 +421,34 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Sequential execution path for single payload or no-execute mode.
|
||||
async fn execute_sequential(
|
||||
/// Sequential execution path with retry logic for underfilled payloads.
|
||||
async fn execute_sequential_with_retry<S: TransactionSource>(
|
||||
&self,
|
||||
auth_provider: &RootProvider<AnyNetwork>,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
transactions: Vec<Bytes>,
|
||||
collector: &TransactionCollector<S>,
|
||||
initial_result: CollectionResult,
|
||||
mut parent_hash: B256,
|
||||
mut parent_timestamp: u64,
|
||||
) -> eyre::Result<()> {
|
||||
for i in 0..self.count {
|
||||
info!(
|
||||
payload = i + 1,
|
||||
total = self.count,
|
||||
parent_hash = %parent_hash,
|
||||
parent_timestamp = parent_timestamp,
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
let mut current_result = initial_result;
|
||||
|
||||
for i in 0..self.count {
|
||||
let built = self
|
||||
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
|
||||
.build_with_retry(
|
||||
testing_provider,
|
||||
collector,
|
||||
&mut current_result,
|
||||
i,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
self.save_payload(&built)?;
|
||||
|
||||
if self.execute || self.count > 1 {
|
||||
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
|
||||
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
|
||||
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
|
||||
info!(payload = i + 1, "Payload executed successfully");
|
||||
}
|
||||
@@ -356,7 +459,62 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pipelined execution - fetches transactions and builds payloads in background.
|
||||
/// Build a payload with retry logic when `gas_used` is below target.
|
||||
///
|
||||
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
|
||||
/// are needed to hit the target gas.
|
||||
async fn build_with_retry<S: TransactionSource>(
|
||||
&self,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
collector: &TransactionCollector<S>,
|
||||
result: &mut CollectionResult,
|
||||
index: u64,
|
||||
parent_hash: B256,
|
||||
parent_timestamp: u64,
|
||||
) -> eyre::Result<BuiltPayload> {
|
||||
for attempt in 1..=MAX_BUILD_RETRIES {
|
||||
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
|
||||
let gas_sent = result.gas_sent;
|
||||
|
||||
info!(
|
||||
payload = index + 1,
|
||||
attempt,
|
||||
tx_count = tx_bytes.len(),
|
||||
gas_sent,
|
||||
parent_hash = %parent_hash,
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
|
||||
let built = Self::build_payload_static(
|
||||
testing_provider,
|
||||
&tx_bytes,
|
||||
index,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
|
||||
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
|
||||
RetryOutcome::NeedMore(additional_gas) => {
|
||||
let additional =
|
||||
collector.collect_gas(result.next_block, additional_gas).await?;
|
||||
result.transactions.extend(additional.transactions);
|
||||
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
|
||||
result.next_block = additional.next_block;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
warn!(payload = index + 1, "Retry loop exited without returning a payload");
|
||||
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
|
||||
}
|
||||
|
||||
/// Pipelined execution - fetches transactions in background, builds with retry.
|
||||
///
|
||||
/// The fetcher continuously produces transaction batches. The main loop consumes them,
|
||||
/// builds payloads with retry logic (requesting more transactions if underfilled),
|
||||
/// and executes each payload before moving to the next.
|
||||
async fn execute_pipelined(
|
||||
&self,
|
||||
auth_provider: &RootProvider<AnyNetwork>,
|
||||
@@ -365,167 +523,229 @@ impl Command {
|
||||
initial_parent_hash: B256,
|
||||
initial_parent_timestamp: u64,
|
||||
) -> eyre::Result<()> {
|
||||
// Create channel for transaction batches (one batch per payload)
|
||||
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
|
||||
// Create channel for transaction batches - fetcher sends CollectionResult
|
||||
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
|
||||
|
||||
// Spawn background task to continuously fetch transaction batches
|
||||
let rpc_url = self.rpc_url.clone();
|
||||
let target_gas = self.target_gas;
|
||||
let count = self.count;
|
||||
|
||||
let fetcher_handle = tokio::spawn(async move {
|
||||
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
|
||||
Ok(source) => source,
|
||||
Err(e) => {
|
||||
warn!(error = %e, "Failed to create transaction source");
|
||||
return;
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let collector = TransactionCollector::new(tx_source, target_gas);
|
||||
let mut current_block = start_block;
|
||||
|
||||
for payload_idx in 0..count {
|
||||
match collector.collect(current_block).await {
|
||||
Ok((transactions, total_gas, next_block)) => {
|
||||
info!(
|
||||
payload = payload_idx + 1,
|
||||
tx_count = transactions.len(),
|
||||
total_gas,
|
||||
blocks = format!("{}..{}", current_block, next_block),
|
||||
"Fetched transactions"
|
||||
);
|
||||
current_block = next_block;
|
||||
|
||||
if tx_sender.send(transactions).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
let mut parent_timestamp = initial_parent_timestamp;
|
||||
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
|
||||
|
||||
for i in 0..self.count {
|
||||
let is_last = i == self.count - 1;
|
||||
|
||||
// Get current payload (either from pending build or build now)
|
||||
let current_payload = if let Some(handle) = pending_build.take() {
|
||||
handle.await??
|
||||
} else {
|
||||
// First payload - wait for transactions and build synchronously
|
||||
let transactions = tx_receiver
|
||||
.recv()
|
||||
.await
|
||||
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
|
||||
|
||||
if transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
|
||||
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
|
||||
if batch.transactions.is_empty() {
|
||||
info!(block = current_block, "Reached chain tip, stopping fetcher");
|
||||
break;
|
||||
}
|
||||
|
||||
info!(
|
||||
payload = i + 1,
|
||||
total = self.count,
|
||||
parent_hash = %parent_hash,
|
||||
parent_timestamp = parent_timestamp,
|
||||
tx_count = transactions.len(),
|
||||
"Building payload via testing_buildBlockV1"
|
||||
tx_count = batch.transactions.len(),
|
||||
gas_sent = batch.gas_sent,
|
||||
blocks = format!("{}..{}", current_block, batch.next_block),
|
||||
"Fetched transaction batch"
|
||||
);
|
||||
self.build_payload(
|
||||
current_block = batch.next_block;
|
||||
|
||||
if tx_sender.send(batch).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Some(current_block)
|
||||
});
|
||||
|
||||
// Transaction buffer: holds transactions from batches + any extras from retries
|
||||
let mut tx_buffer = TxBuffer::new(tx_receiver);
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
let mut parent_timestamp = initial_parent_timestamp;
|
||||
|
||||
for i in 0..self.count {
|
||||
// Get initial batch of transactions for this payload
|
||||
let Some(mut result) = tx_buffer.take_batch().await else {
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"Transaction source exhausted, stopping"
|
||||
);
|
||||
break;
|
||||
};
|
||||
|
||||
if result.transactions.is_empty() {
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"No more transactions available, stopping"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Build with retry - may need to request more transactions
|
||||
let built = self
|
||||
.build_with_retry_buffered(
|
||||
testing_provider,
|
||||
&transactions,
|
||||
&mut tx_buffer,
|
||||
&mut result,
|
||||
i,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?
|
||||
};
|
||||
.await?;
|
||||
|
||||
self.save_payload(¤t_payload)?;
|
||||
self.save_payload(&built)?;
|
||||
|
||||
let current_block_hash = current_payload.block_hash;
|
||||
let current_timestamp = current_payload.timestamp;
|
||||
let current_block_hash = built.block_hash;
|
||||
let current_timestamp = built.timestamp;
|
||||
|
||||
// Execute current payload first
|
||||
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
|
||||
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
|
||||
// Execute payload
|
||||
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
|
||||
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
|
||||
info!(payload = i + 1, "Payload executed successfully");
|
||||
|
||||
// Start building next payload in background (if not last) - AFTER execution
|
||||
if !is_last {
|
||||
// Get transactions for next payload (should already be fetched or fetching)
|
||||
let next_transactions = tx_receiver
|
||||
.recv()
|
||||
.await
|
||||
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
|
||||
|
||||
if next_transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
|
||||
}
|
||||
|
||||
let testing_provider = testing_provider.clone();
|
||||
let next_index = i + 1;
|
||||
let total = self.count;
|
||||
|
||||
pending_build = Some(tokio::spawn(async move {
|
||||
info!(
|
||||
payload = next_index + 1,
|
||||
total = total,
|
||||
parent_hash = %current_block_hash,
|
||||
parent_timestamp = current_timestamp,
|
||||
tx_count = next_transactions.len(),
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
|
||||
Self::build_payload_static(
|
||||
&testing_provider,
|
||||
&next_transactions,
|
||||
next_index,
|
||||
current_block_hash,
|
||||
current_timestamp,
|
||||
)
|
||||
.await
|
||||
}));
|
||||
}
|
||||
|
||||
parent_hash = current_block_hash;
|
||||
parent_timestamp = current_timestamp;
|
||||
}
|
||||
|
||||
// Clean up the fetcher task
|
||||
drop(tx_receiver);
|
||||
drop(tx_buffer);
|
||||
let _ = fetcher_handle.await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a single payload via `testing_buildBlockV1`.
|
||||
async fn build_payload(
|
||||
/// Build a payload with retry logic, using the buffered transaction source.
|
||||
async fn build_with_retry_buffered(
|
||||
&self,
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
transactions: &[Bytes],
|
||||
tx_buffer: &mut TxBuffer,
|
||||
result: &mut CollectionResult,
|
||||
index: u64,
|
||||
parent_hash: B256,
|
||||
parent_timestamp: u64,
|
||||
) -> eyre::Result<BuiltPayload> {
|
||||
Self::build_payload_static(
|
||||
testing_provider,
|
||||
transactions,
|
||||
index,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await
|
||||
for attempt in 1..=MAX_BUILD_RETRIES {
|
||||
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
|
||||
let gas_sent = result.gas_sent;
|
||||
|
||||
info!(
|
||||
payload = index + 1,
|
||||
attempt,
|
||||
tx_count = tx_bytes.len(),
|
||||
gas_sent,
|
||||
parent_hash = %parent_hash,
|
||||
"Building payload via testing_buildBlockV1"
|
||||
);
|
||||
|
||||
let built = Self::build_payload_static(
|
||||
testing_provider,
|
||||
&tx_bytes,
|
||||
index,
|
||||
parent_hash,
|
||||
parent_timestamp,
|
||||
)
|
||||
.await?;
|
||||
|
||||
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
|
||||
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
|
||||
RetryOutcome::NeedMore(additional_gas) => {
|
||||
let mut collected_gas = 0u64;
|
||||
while collected_gas < additional_gas {
|
||||
if let Some(batch) = tx_buffer.take_batch().await {
|
||||
collected_gas += batch.gas_sent;
|
||||
result.transactions.extend(batch.transactions);
|
||||
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
|
||||
result.next_block = batch.next_block;
|
||||
} else {
|
||||
warn!("Transaction fetcher exhausted, proceeding with available transactions");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
warn!(payload = index + 1, "Retry loop exited without returning a payload");
|
||||
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
|
||||
}
|
||||
|
||||
/// Static version for use in spawned tasks.
|
||||
/// Determines the outcome of a build attempt.
|
||||
fn check_retry_outcome(
|
||||
&self,
|
||||
built: &BuiltPayload,
|
||||
index: u64,
|
||||
attempt: u32,
|
||||
gas_sent: u64,
|
||||
) -> RetryOutcome {
|
||||
let gas_used = built.gas_used;
|
||||
|
||||
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
attempts = attempt,
|
||||
"Payload built successfully"
|
||||
);
|
||||
return RetryOutcome::Success;
|
||||
}
|
||||
|
||||
if attempt == MAX_BUILD_RETRIES {
|
||||
warn!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
gas_sent,
|
||||
"Underfilled after max retries, accepting payload"
|
||||
);
|
||||
return RetryOutcome::MaxRetries;
|
||||
}
|
||||
|
||||
if gas_used == 0 {
|
||||
warn!(
|
||||
payload = index + 1,
|
||||
"Zero gas used in payload, requesting fixed chunk of additional transactions"
|
||||
);
|
||||
return RetryOutcome::NeedMore(self.target_gas);
|
||||
}
|
||||
|
||||
let gas_sent_needed_total =
|
||||
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
|
||||
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
|
||||
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
|
||||
|
||||
if additional == 0 {
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
target_gas = self.target_gas,
|
||||
"No additional transactions needed based on ratio"
|
||||
);
|
||||
return RetryOutcome::Success;
|
||||
}
|
||||
|
||||
let ratio = gas_used as f64 / gas_sent as f64;
|
||||
info!(
|
||||
payload = index + 1,
|
||||
gas_used,
|
||||
gas_sent,
|
||||
ratio = format!("{:.4}", ratio),
|
||||
additional_gas = additional,
|
||||
"Underfilled, collecting more transactions for retry"
|
||||
);
|
||||
RetryOutcome::NeedMore(additional)
|
||||
}
|
||||
|
||||
/// Build a single payload via `testing_buildBlockV1`.
|
||||
async fn build_payload_static(
|
||||
testing_provider: &RootProvider<AnyNetwork>,
|
||||
transactions: &[Bytes],
|
||||
@@ -563,8 +783,9 @@ impl Command {
|
||||
let block_hash = inner.block_hash;
|
||||
let block_number = inner.block_number;
|
||||
let timestamp = inner.timestamp;
|
||||
let gas_used = inner.gas_used;
|
||||
|
||||
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
|
||||
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
|
||||
}
|
||||
|
||||
/// Save a payload to disk.
|
||||
|
||||
@@ -1,6 +1,56 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use crate::valid_payload::call_forkchoice_updated;
|
||||
use eyre::Result;
|
||||
use std::io::{BufReader, Read};
|
||||
|
||||
/// Read input from either a file path or stdin.
|
||||
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
|
||||
Ok(match path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly.
|
||||
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
|
||||
match jwt_secret {
|
||||
Some(secret) => {
|
||||
// Try to read as file first
|
||||
match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
// If file read fails, use the string directly
|
||||
Err(_) => Ok(Some(secret.to_string())),
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
|
||||
///
|
||||
/// Examples: "30000000", "30M", "1G", "2G"
|
||||
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
|
||||
let s = s.trim();
|
||||
if s.is_empty() {
|
||||
return Err(eyre::eyre!("empty value"));
|
||||
}
|
||||
|
||||
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
|
||||
(prefix, 1_000_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
|
||||
(prefix, 1_000_000u64)
|
||||
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
|
||||
(prefix, 1_000u64)
|
||||
} else {
|
||||
(s, 1u64)
|
||||
};
|
||||
|
||||
let base: u64 = num_str.trim().parse()?;
|
||||
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
|
||||
}
|
||||
use alloy_consensus::Header;
|
||||
use alloy_eips::eip4844::kzg_to_versioned_hash;
|
||||
use alloy_primitives::{Address, B256};
|
||||
@@ -194,3 +244,50 @@ pub(crate) async fn get_payload_with_sidecar(
|
||||
_ => panic!("This tool does not support getPayload versions past v5"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_plain_number() {
|
||||
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("1").unwrap(), 1);
|
||||
assert_eq!(parse_gas_limit("0").unwrap(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_k_suffix() {
|
||||
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
|
||||
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
|
||||
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_m_suffix() {
|
||||
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
|
||||
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
|
||||
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_g_suffix() {
|
||||
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
|
||||
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_with_whitespace() {
|
||||
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
|
||||
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gas_limit_errors() {
|
||||
assert!(parse_gas_limit("").is_err());
|
||||
assert!(parse_gas_limit("abc").is_err());
|
||||
assert!(parse_gas_limit("G").is_err());
|
||||
assert!(parse_gas_limit("-1G").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ pub use generate_big_block::{
|
||||
mod new_payload_fcu;
|
||||
mod new_payload_only;
|
||||
mod output;
|
||||
mod persistence_waiter;
|
||||
mod replay_payloads;
|
||||
mod send_invalid_payload;
|
||||
mod send_payload;
|
||||
|
||||
@@ -15,28 +15,23 @@ use crate::{
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
|
||||
};
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_network::Ethereum;
|
||||
use alloy_provider::{Provider, RootProvider};
|
||||
use alloy_pubsub::SubscriptionStream;
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_provider::Provider;
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use clap::Parser;
|
||||
use eyre::{Context, OptionExt};
|
||||
use futures::StreamExt;
|
||||
use humantime::parse_duration;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// `reth benchmark new-payload-fcu` command
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -105,7 +100,11 @@ impl Command {
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let sub = self.setup_persistence_subscription().await?;
|
||||
let ws_url = derive_ws_rpc_url(
|
||||
self.benchmark.ws_rpc_url.as_deref(),
|
||||
&self.benchmark.engine_rpc_url,
|
||||
)?;
|
||||
let sub = setup_persistence_subscription(ws_url).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
@@ -245,293 +244,22 @@ impl Command {
|
||||
results.into_iter().unzip();
|
||||
|
||||
if let Some(ref path) = self.benchmark.output {
|
||||
write_benchmark_results(path, &gas_output_results, combined_results)?;
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
let gas_output = TotalGasOutput::new(gas_output_results)?;
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
|
||||
info!(
|
||||
total_duration=?gas_output.total_duration,
|
||||
total_gas_used=?gas_output.total_gas_used,
|
||||
blocks_processed=?gas_output.blocks_processed,
|
||||
"Total Ggas/s: {:.4}",
|
||||
gas_output.total_gigagas_per_second()
|
||||
total_gas_used = gas_output.total_gas_used,
|
||||
total_duration = ?gas_output.total_duration,
|
||||
execution_duration = ?gas_output.execution_duration,
|
||||
blocks_processed = gas_output.blocks_processed,
|
||||
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
|
||||
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the websocket RPC URL used for the persistence subscription.
|
||||
///
|
||||
/// Preference:
|
||||
/// - If `--ws-rpc-url` is provided, use it directly.
|
||||
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
|
||||
///
|
||||
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
|
||||
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
|
||||
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
|
||||
/// (http→ws, https→wss) and force the port to 8546.
|
||||
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
|
||||
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
|
||||
let parsed: Url = ws_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
|
||||
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
|
||||
Ok(parsed)
|
||||
} else {
|
||||
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
engine_url = %self.benchmark.engine_rpc_url,
|
||||
%derived,
|
||||
"Derived WebSocket RPC URL from engine RPC URL"
|
||||
);
|
||||
Ok(derived)
|
||||
}
|
||||
}
|
||||
|
||||
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
|
||||
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
|
||||
let ws_url = self.derive_ws_rpc_url()?;
|
||||
|
||||
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
|
||||
|
||||
let ws_connect = WsConnect::new(ws_url.to_string());
|
||||
let client = RpcClient::connect_pubsub(ws_connect)
|
||||
.await
|
||||
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
|
||||
let provider: RootProvider<Ethereum> = RootProvider::new(client);
|
||||
|
||||
let subscription = provider
|
||||
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
|
||||
.await
|
||||
.wrap_err("Failed to subscribe to persistence notifications")?;
|
||||
|
||||
info!("Subscribed to persistence notifications");
|
||||
|
||||
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an engine API URL to the default RPC websocket URL.
|
||||
///
|
||||
/// Transformations:
|
||||
/// - `http` → `ws`
|
||||
/// - `https` → `wss`
|
||||
/// - `ws` / `wss` keep their scheme
|
||||
/// - Port is always set to `8546`, reth's default RPC websocket port.
|
||||
///
|
||||
/// This is used when we only know the engine API URL (typically `:8551`) but
|
||||
/// need to connect to the node's WS RPC endpoint for persistence events.
|
||||
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
|
||||
let url: Url = engine_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
|
||||
|
||||
let mut ws_url = url.clone();
|
||||
|
||||
match ws_url.scheme() {
|
||||
"http" => ws_url
|
||||
.set_scheme("ws")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
|
||||
"https" => ws_url
|
||||
.set_scheme("wss")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
|
||||
"ws" | "wss" => {}
|
||||
scheme => {
|
||||
return Err(eyre::eyre!(
|
||||
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
|
||||
|
||||
Ok(ws_url)
|
||||
}
|
||||
|
||||
/// Waits until the persistence subscription reports that `target` has been persisted.
|
||||
///
|
||||
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
|
||||
/// - the subscription stream ends unexpectedly, or
|
||||
/// - `timeout` elapses before `target` is observed.
|
||||
async fn wait_for_persistence(
|
||||
stream: &mut SubscriptionStream<BlockNumHash>,
|
||||
target: u64,
|
||||
last_persisted: &mut u64,
|
||||
timeout: Duration,
|
||||
) -> eyre::Result<()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
while *last_persisted < target {
|
||||
match stream.next().await {
|
||||
Some(persisted) => {
|
||||
*last_persisted = persisted.number;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted_block = ?last_persisted,
|
||||
"Received persistence notification"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
eyre::eyre!(
|
||||
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
|
||||
target,
|
||||
timeout,
|
||||
last_persisted
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
|
||||
/// The provider must be kept alive for the subscription to continue receiving events.
|
||||
struct PersistenceSubscription {
|
||||
_provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
}
|
||||
|
||||
impl PersistenceSubscription {
|
||||
const fn new(
|
||||
provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
) -> Self {
|
||||
Self { _provider: provider, stream }
|
||||
}
|
||||
|
||||
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Encapsulates the block waiting logic.
|
||||
///
|
||||
/// Provides a simple `on_block()` interface that handles both:
|
||||
/// - Fixed duration waits (when `wait_time` is set)
|
||||
/// - Persistence-based waits (when `subscription` is set)
|
||||
///
|
||||
/// For persistence mode, waits after every `(threshold + 1)` blocks.
|
||||
struct PersistenceWaiter {
|
||||
wait_time: Option<Duration>,
|
||||
subscription: Option<PersistenceSubscription>,
|
||||
blocks_sent: u64,
|
||||
last_persisted: u64,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PersistenceWaiter {
|
||||
const fn with_duration(wait_time: Duration) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: None,
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold: 0,
|
||||
timeout: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
const fn with_subscription(
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: None,
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Called once per block. Waits based on the configured mode.
|
||||
#[allow(clippy::manual_is_multiple_of)]
|
||||
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(ref mut subscription) = self.subscription else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.blocks_sent += 1;
|
||||
|
||||
if self.blocks_sent % (self.threshold + 1) == 0 {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
target_block = ?block_number,
|
||||
last_persisted = self.last_persisted,
|
||||
blocks_sent = self.blocks_sent,
|
||||
"Waiting for persistence"
|
||||
);
|
||||
|
||||
wait_for_persistence(
|
||||
subscription.stream_mut(),
|
||||
block_number,
|
||||
&mut self.last_persisted,
|
||||
self.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted = self.last_persisted,
|
||||
"Persistence caught up"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_engine_url_to_ws_url() {
|
||||
// http -> ws, always uses port 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "ws://localhost:8546/");
|
||||
|
||||
// https -> wss
|
||||
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "wss://localhost:8546/");
|
||||
|
||||
// Custom engine port still maps to 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
|
||||
assert_eq!(result.port(), Some(8546));
|
||||
|
||||
// Already ws passthrough
|
||||
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
|
||||
assert_eq!(result.scheme(), "ws");
|
||||
|
||||
// Invalid inputs
|
||||
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
|
||||
assert!(engine_url_to_ws_url("not a valid url").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_waiter_with_duration() {
|
||||
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
|
||||
|
||||
let start = Instant::now();
|
||||
waiter.on_block(1).await.unwrap();
|
||||
waiter.on_block(2).await.unwrap();
|
||||
waiter.on_block(3).await.unwrap();
|
||||
|
||||
// Should have waited ~3ms total
|
||||
assert!(start.elapsed() >= Duration::from_millis(3));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use csv::Writer;
|
||||
use eyre::OptionExt;
|
||||
use reth_primitives_traits::constants::GIGAGAS;
|
||||
use serde::{ser::SerializeStruct, Deserialize, Serialize};
|
||||
use std::{path::Path, time::Duration};
|
||||
use std::{fs, path::Path, time::Duration};
|
||||
use tracing::info;
|
||||
|
||||
/// This is the suffix for gas output csv files.
|
||||
@@ -158,29 +158,58 @@ pub(crate) struct TotalGasRow {
|
||||
pub(crate) struct TotalGasOutput {
|
||||
/// The total gas used in the benchmark.
|
||||
pub(crate) total_gas_used: u64,
|
||||
/// The total duration of the benchmark.
|
||||
/// The total wall-clock duration of the benchmark (includes wait times).
|
||||
pub(crate) total_duration: Duration,
|
||||
/// The total gas used per second.
|
||||
pub(crate) total_gas_per_second: f64,
|
||||
/// The total execution-only duration (excludes wait times).
|
||||
pub(crate) execution_duration: Duration,
|
||||
/// The number of blocks processed.
|
||||
pub(crate) blocks_processed: u64,
|
||||
}
|
||||
|
||||
impl TotalGasOutput {
|
||||
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
|
||||
/// Create a new [`TotalGasOutput`] from gas rows only.
|
||||
///
|
||||
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
|
||||
/// `execution_duration` will equal `total_duration`.
|
||||
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
|
||||
// the duration is obtained from the last row
|
||||
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
|
||||
let blocks_processed = rows.len() as u64;
|
||||
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
|
||||
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
|
||||
|
||||
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
|
||||
Ok(Self {
|
||||
total_gas_used,
|
||||
total_duration,
|
||||
execution_duration: total_duration,
|
||||
blocks_processed,
|
||||
})
|
||||
}
|
||||
|
||||
/// Return the total gigagas per second.
|
||||
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
|
||||
///
|
||||
/// - `rows`: Used for total gas and wall-clock duration
|
||||
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
|
||||
pub(crate) fn with_combined_results(
|
||||
rows: Vec<TotalGasRow>,
|
||||
combined_results: &[CombinedResult],
|
||||
) -> eyre::Result<Self> {
|
||||
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
|
||||
let blocks_processed = rows.len() as u64;
|
||||
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
|
||||
|
||||
// Sum execution-only time from combined results
|
||||
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
|
||||
|
||||
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
|
||||
}
|
||||
|
||||
/// Return the total gigagas per second based on wall-clock time.
|
||||
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
|
||||
self.total_gas_per_second / GIGAGAS as f64
|
||||
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
|
||||
}
|
||||
|
||||
/// Return the execution-only gigagas per second (excludes wait times).
|
||||
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
|
||||
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,8 +221,10 @@ impl TotalGasOutput {
|
||||
pub(crate) fn write_benchmark_results(
|
||||
output_dir: &Path,
|
||||
gas_results: &[TotalGasRow],
|
||||
combined_results: Vec<CombinedResult>,
|
||||
combined_results: &[CombinedResult],
|
||||
) -> eyre::Result<()> {
|
||||
fs::create_dir_all(output_dir)?;
|
||||
|
||||
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
|
||||
info!("Writing engine api call latency output to file: {:?}", output_path);
|
||||
let mut writer = Writer::from_path(&output_path)?;
|
||||
|
||||
304
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
304
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
@@ -0,0 +1,304 @@
|
||||
//! Persistence waiting utilities for benchmarks.
|
||||
//!
|
||||
//! Provides waiting behavior to control benchmark pacing:
|
||||
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
|
||||
//! - **Persistence-based waits**: Wait for blocks to be persisted using
|
||||
//! `reth_subscribePersistedBlock` subscription
|
||||
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_network::Ethereum;
|
||||
use alloy_provider::{Provider, RootProvider};
|
||||
use alloy_pubsub::SubscriptionStream;
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use eyre::Context;
|
||||
use futures::StreamExt;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Default `WebSocket` RPC port for reth.
|
||||
const DEFAULT_WS_RPC_PORT: u16 = 8546;
|
||||
use url::Url;
|
||||
|
||||
/// Default timeout for waiting on persistence.
|
||||
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
|
||||
/// Returns the websocket RPC URL used for the persistence subscription.
|
||||
///
|
||||
/// Preference:
|
||||
/// - If `ws_rpc_url` is provided, use it directly.
|
||||
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
|
||||
///
|
||||
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
|
||||
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
|
||||
/// Since we may only have the engine URL by default, we convert the scheme
|
||||
/// (http→ws, https→wss) and force the port to 8546.
|
||||
pub(crate) fn derive_ws_rpc_url(
|
||||
ws_rpc_url: Option<&str>,
|
||||
engine_rpc_url: &str,
|
||||
) -> eyre::Result<Url> {
|
||||
if let Some(ws_url) = ws_rpc_url {
|
||||
let parsed: Url = ws_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
|
||||
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
|
||||
Ok(parsed)
|
||||
} else {
|
||||
let derived = engine_url_to_ws_url(engine_rpc_url)?;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
engine_url = %engine_rpc_url,
|
||||
%derived,
|
||||
"Derived WebSocket RPC URL from engine RPC URL"
|
||||
);
|
||||
Ok(derived)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an engine API URL to the default RPC websocket URL.
|
||||
///
|
||||
/// Transformations:
|
||||
/// - `http` → `ws`
|
||||
/// - `https` → `wss`
|
||||
/// - `ws` / `wss` keep their scheme
|
||||
/// - Port is always set to `8546`, reth's default RPC websocket port.
|
||||
///
|
||||
/// This is used when we only know the engine API URL (typically `:8551`) but
|
||||
/// need to connect to the node's WS RPC endpoint for persistence events.
|
||||
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
|
||||
let url: Url = engine_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
|
||||
|
||||
let mut ws_url = url.clone();
|
||||
|
||||
match ws_url.scheme() {
|
||||
"http" => ws_url
|
||||
.set_scheme("ws")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
|
||||
"https" => ws_url
|
||||
.set_scheme("wss")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
|
||||
"ws" | "wss" => {}
|
||||
scheme => {
|
||||
return Err(eyre::eyre!(
|
||||
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
ws_url
|
||||
.set_port(Some(DEFAULT_WS_RPC_PORT))
|
||||
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
|
||||
|
||||
Ok(ws_url)
|
||||
}
|
||||
|
||||
/// Waits until the persistence subscription reports that `target` has been persisted.
|
||||
///
|
||||
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
|
||||
/// - the subscription stream ends unexpectedly, or
|
||||
/// - `timeout` elapses before `target` is observed.
|
||||
async fn wait_for_persistence(
|
||||
stream: &mut SubscriptionStream<BlockNumHash>,
|
||||
target: u64,
|
||||
last_persisted: &mut u64,
|
||||
timeout: Duration,
|
||||
) -> eyre::Result<()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
while *last_persisted < target {
|
||||
match stream.next().await {
|
||||
Some(persisted) => {
|
||||
*last_persisted = persisted.number;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted_block = ?last_persisted,
|
||||
"Received persistence notification"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
eyre::eyre!(
|
||||
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
|
||||
target,
|
||||
timeout,
|
||||
last_persisted
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
|
||||
/// The provider must be kept alive for the subscription to continue receiving events.
|
||||
pub(crate) struct PersistenceSubscription {
|
||||
_provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
}
|
||||
|
||||
impl PersistenceSubscription {
|
||||
const fn new(
|
||||
provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
) -> Self {
|
||||
Self { _provider: provider, stream }
|
||||
}
|
||||
|
||||
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
|
||||
pub(crate) async fn setup_persistence_subscription(
|
||||
ws_url: Url,
|
||||
) -> eyre::Result<PersistenceSubscription> {
|
||||
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
|
||||
|
||||
let ws_connect = WsConnect::new(ws_url.to_string());
|
||||
let client = RpcClient::connect_pubsub(ws_connect)
|
||||
.await
|
||||
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
|
||||
let provider: RootProvider<Ethereum> = RootProvider::new(client);
|
||||
|
||||
let subscription = provider
|
||||
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
|
||||
.await
|
||||
.wrap_err("Failed to subscribe to persistence notifications")?;
|
||||
|
||||
info!("Subscribed to persistence notifications");
|
||||
|
||||
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
|
||||
}
|
||||
|
||||
/// Encapsulates the block waiting logic.
|
||||
///
|
||||
/// Provides a simple `on_block()` interface that handles both:
|
||||
/// - Fixed duration waits (when `wait_time` is set)
|
||||
/// - Persistence-based waits (when `subscription` is set)
|
||||
///
|
||||
/// For persistence mode, waits after every `(threshold + 1)` blocks.
|
||||
pub(crate) struct PersistenceWaiter {
|
||||
wait_time: Option<Duration>,
|
||||
subscription: Option<PersistenceSubscription>,
|
||||
blocks_sent: u64,
|
||||
last_persisted: u64,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PersistenceWaiter {
|
||||
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: None,
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold: 0,
|
||||
timeout: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn with_subscription(
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: None,
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Called once per block. Waits based on the configured mode.
|
||||
#[allow(clippy::manual_is_multiple_of)]
|
||||
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let Some(ref mut subscription) = self.subscription else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.blocks_sent += 1;
|
||||
|
||||
if self.blocks_sent % (self.threshold + 1) == 0 {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
target_block = ?block_number,
|
||||
last_persisted = self.last_persisted,
|
||||
blocks_sent = self.blocks_sent,
|
||||
"Waiting for persistence"
|
||||
);
|
||||
|
||||
wait_for_persistence(
|
||||
subscription.stream_mut(),
|
||||
block_number,
|
||||
&mut self.last_persisted,
|
||||
self.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted = self.last_persisted,
|
||||
"Persistence caught up"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn test_engine_url_to_ws_url() {
|
||||
// http -> ws, always uses port 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "ws://localhost:8546/");
|
||||
|
||||
// https -> wss
|
||||
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "wss://localhost:8546/");
|
||||
|
||||
// Custom engine port still maps to 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
|
||||
assert_eq!(result.port(), Some(8546));
|
||||
|
||||
// Already ws passthrough
|
||||
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
|
||||
assert_eq!(result.scheme(), "ws");
|
||||
|
||||
// Invalid inputs
|
||||
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
|
||||
assert!(engine_url_to_ws_url("not a valid url").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_waiter_with_duration() {
|
||||
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
|
||||
|
||||
let start = Instant::now();
|
||||
waiter.on_block(1).await.unwrap();
|
||||
waiter.on_block(2).await.unwrap();
|
||||
waiter.on_block(3).await.unwrap();
|
||||
|
||||
// Should have waited ~3ms total
|
||||
assert!(start.elapsed() >= Duration::from_millis(3));
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,27 @@
|
||||
//!
|
||||
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
|
||||
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
|
||||
//!
|
||||
//! Supports configurable waiting behavior:
|
||||
//! - **`--wait-time`**: Fixed sleep interval between blocks.
|
||||
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
|
||||
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
|
||||
//! threshold. This ensures the benchmark doesn't outpace persistence.
|
||||
//!
|
||||
//! Both options can be used together or independently.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::output::GasRampPayloadFile,
|
||||
bench::{
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
|
||||
TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
},
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
@@ -14,11 +31,16 @@ use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use reqwest::Url;
|
||||
use humantime::parse_duration;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use std::path::PathBuf;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
/// `reth bench replay-payloads` command
|
||||
///
|
||||
@@ -51,6 +73,42 @@ pub struct Command {
|
||||
/// These are replayed before the main payloads to warm up the gas limit.
|
||||
#[arg(long, value_name = "GAS_RAMP_DIR")]
|
||||
gas_ramp_dir: Option<PathBuf>,
|
||||
|
||||
/// Optional output directory for benchmark results (CSV files).
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: Option<PathBuf>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
|
||||
wait_time: Option<Duration>,
|
||||
|
||||
/// Wait for blocks to be persisted before sending the next batch.
|
||||
///
|
||||
/// When enabled, waits for every Nth block to be persisted using the
|
||||
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
|
||||
/// doesn't outpace persistence.
|
||||
///
|
||||
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
wait_for_persistence: bool,
|
||||
|
||||
/// Engine persistence threshold used for deciding when to wait for persistence.
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
|
||||
/// at blocks 3, 6, 9, etc.
|
||||
#[arg(
|
||||
long = "persistence-threshold",
|
||||
value_name = "PERSISTENCE_THRESHOLD",
|
||||
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
persistence_threshold: u64,
|
||||
|
||||
/// Optional `WebSocket` RPC URL for persistence subscription.
|
||||
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
|
||||
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
|
||||
ws_rpc_url: Option<String>,
|
||||
}
|
||||
|
||||
/// A loaded payload ready for execution.
|
||||
@@ -78,6 +136,33 @@ impl Command {
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
|
||||
}
|
||||
if self.wait_for_persistence {
|
||||
info!(
|
||||
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
|
||||
self.persistence_threshold + 1,
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options (duration takes precedence)
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
|
||||
let sub = setup_persistence_subscription(ws_url).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
PERSISTENCE_CHECKPOINT_TIMEOUT,
|
||||
))
|
||||
}
|
||||
(None, false) => None,
|
||||
};
|
||||
|
||||
// Set up authenticated engine provider
|
||||
let jwt =
|
||||
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
|
||||
@@ -144,6 +229,11 @@ impl Command {
|
||||
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
|
||||
|
||||
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(payload.block_number).await?;
|
||||
}
|
||||
|
||||
parent_hash = payload.file.block_hash;
|
||||
}
|
||||
|
||||
@@ -151,22 +241,112 @@ impl Command {
|
||||
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
for (i, payload) in payloads.iter().enumerate() {
|
||||
info!(
|
||||
let envelope = &payload.envelope;
|
||||
let block_hash = payload.block_hash;
|
||||
let execution_payload = &envelope.envelope_inner.execution_payload;
|
||||
let inner_payload = &execution_payload.payload_inner.payload_inner;
|
||||
|
||||
let gas_used = inner_payload.gas_used;
|
||||
let gas_limit = inner_payload.gas_limit;
|
||||
let block_number = inner_payload.block_number;
|
||||
let transaction_count =
|
||||
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
|
||||
|
||||
debug!(
|
||||
payload = i + 1,
|
||||
total = payloads.len(),
|
||||
index = payload.index,
|
||||
block_hash = %payload.block_hash,
|
||||
block_hash = %block_hash,
|
||||
"Executing payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
|
||||
let start = Instant::now();
|
||||
|
||||
info!(payload = i + 1, "Payload executed successfully");
|
||||
parent_hash = payload.block_hash;
|
||||
debug!(
|
||||
method = "engine_newPayloadV4",
|
||||
block_hash = %block_hash,
|
||||
"Sending newPayload"
|
||||
);
|
||||
|
||||
let status = auth_provider
|
||||
.new_payload_v4(
|
||||
execution_payload.clone(),
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
|
||||
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
|
||||
|
||||
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
transaction_count,
|
||||
new_payload_result,
|
||||
fcu_latency,
|
||||
total_latency,
|
||||
};
|
||||
|
||||
let current_duration = total_benchmark_duration.elapsed();
|
||||
info!(%combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
}
|
||||
|
||||
let gas_row =
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((gas_row, combined_result));
|
||||
|
||||
debug!(?status, ?fcu_result, "Payload executed successfully");
|
||||
parent_hash = block_hash;
|
||||
}
|
||||
|
||||
info!(count = payloads.len(), "All payloads replayed successfully");
|
||||
// Drop waiter - we don't need to wait for final blocks to persist
|
||||
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
|
||||
drop(waiter);
|
||||
|
||||
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
|
||||
results.into_iter().unzip();
|
||||
|
||||
if let Some(ref path) = self.output {
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
info!(
|
||||
total_gas_used = gas_output.total_gas_used,
|
||||
total_duration = ?gas_output.total_duration,
|
||||
execution_duration = ?gas_output.execution_duration,
|
||||
blocks_processed = gas_output.blocks_processed,
|
||||
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
|
||||
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -284,49 +464,4 @@ impl Command {
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
async fn execute_payload_v4(
|
||||
&self,
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
envelope: &ExecutionPayloadEnvelopeV4,
|
||||
parent_hash: B256,
|
||||
) -> eyre::Result<()> {
|
||||
let block_hash =
|
||||
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
|
||||
|
||||
debug!(
|
||||
method = "engine_newPayloadV4",
|
||||
block_hash = %block_hash,
|
||||
"Sending newPayload"
|
||||
);
|
||||
|
||||
let status = provider
|
||||
.new_payload_v4(
|
||||
envelope.envelope_inner.execution_payload.clone(),
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
info!(?status, "newPayloadV4 response");
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
|
||||
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
|
||||
|
||||
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
|
||||
info!(?fcu_result, "forkchoiceUpdatedV3 response");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
mod invalidation;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
@@ -10,7 +11,7 @@ use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
use op_alloy_consensus::OpTxEnvelope;
|
||||
use reth_cli_runner::CliContext;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
use std::io::Write;
|
||||
|
||||
/// Command for generating and sending an invalid `engine_newPayload` request.
|
||||
///
|
||||
@@ -180,27 +181,6 @@ enum Mode {
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Read input from either a file or stdin
|
||||
fn read_input(&self) -> Result<String> {
|
||||
Ok(match &self.path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly
|
||||
fn load_jwt_secret(&self) -> Result<Option<String>> {
|
||||
match &self.jwt_secret {
|
||||
Some(secret) => match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
Err(_) => Ok(Some(secret.clone())),
|
||||
},
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Build `InvalidationConfig` from command flags
|
||||
const fn build_invalidation_config(&self) -> InvalidationConfig {
|
||||
InvalidationConfig {
|
||||
@@ -236,8 +216,8 @@ impl Command {
|
||||
|
||||
/// Execute the command
|
||||
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
|
||||
let block_json = self.read_input()?;
|
||||
let jwt_secret = self.load_jwt_secret()?;
|
||||
let block_json = read_input(self.path.as_deref())?;
|
||||
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
|
||||
|
||||
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
|
||||
.into_inner()
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
use op_alloy_consensus::OpTxEnvelope;
|
||||
use reth_cli_runner::CliContext;
|
||||
use std::io::{BufReader, Read, Write};
|
||||
use std::io::Write;
|
||||
|
||||
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
|
||||
/// block.
|
||||
@@ -51,38 +52,13 @@ enum Mode {
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Read input from either a file or stdin
|
||||
fn read_input(&self) -> Result<String> {
|
||||
Ok(match &self.path {
|
||||
Some(path) => reth_fs_util::read_to_string(path)?,
|
||||
None => String::from_utf8(
|
||||
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
|
||||
)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load JWT secret from either a file or use the provided string directly
|
||||
fn load_jwt_secret(&self) -> Result<Option<String>> {
|
||||
match &self.jwt_secret {
|
||||
Some(secret) => {
|
||||
// Try to read as file first
|
||||
match std::fs::read_to_string(secret) {
|
||||
Ok(contents) => Ok(Some(contents.trim().to_string())),
|
||||
// If file read fails, use the string directly
|
||||
Err(_) => Ok(Some(secret.clone())),
|
||||
}
|
||||
}
|
||||
None => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the generate payload command
|
||||
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
|
||||
// Load block
|
||||
let block_json = self.read_input()?;
|
||||
let block_json = read_input(self.path.as_deref())?;
|
||||
|
||||
// Load JWT secret
|
||||
let jwt_secret = self.load_jwt_secret()?;
|
||||
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
|
||||
|
||||
// Parse the block
|
||||
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
|
||||
|
||||
@@ -260,7 +260,9 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(?status, ?params, "Invalid {method}",);
|
||||
panic!("Invalid {method}: {status:?}");
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
|
||||
format!("Invalid {method}: {status:?}"),
|
||||
))))
|
||||
}
|
||||
if status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
|
||||
@@ -25,8 +25,8 @@
|
||||
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
|
||||
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
|
||||
//! integration for memory profiling.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
|
||||
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
|
||||
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
|
||||
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
|
||||
//! when enabling this.
|
||||
//!
|
||||
|
||||
@@ -206,11 +206,33 @@ impl DeferredTrieData {
|
||||
Default::default(), // prefix_sets are per-block, not cumulative
|
||||
);
|
||||
// Only trigger COW clone if there's actually data to add.
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
|
||||
#[cfg(feature = "rayon")]
|
||||
{
|
||||
rayon::join(
|
||||
|| {
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state)
|
||||
.extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
},
|
||||
|| {
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes)
|
||||
.extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state)
|
||||
.extend_ref_and_sort(&sorted_hashed_state);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes)
|
||||
.extend_ref_and_sort(&sorted_trie_updates);
|
||||
}
|
||||
}
|
||||
overlay
|
||||
}
|
||||
@@ -243,53 +265,8 @@ impl DeferredTrieData {
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// When the `rayon` feature is enabled, uses parallel collection and merge:
|
||||
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
|
||||
/// 2. Merges hashed state and trie updates in parallel with each other
|
||||
/// 3. Uses tree reduction within each merge for O(log n) depth
|
||||
#[cfg(feature = "rayon")]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
// Early exit: no ancestors means just wrap current block's data
|
||||
if ancestors.is_empty() {
|
||||
return TrieInputSorted::new(
|
||||
Arc::new(sorted_trie_updates.clone()),
|
||||
Arc::new(sorted_hashed_state.clone()),
|
||||
Default::default(),
|
||||
);
|
||||
}
|
||||
|
||||
// Collect ancestor data, unzipping states and updates into Arc slices
|
||||
let (states, updates): (Vec<_>, Vec<_>) = ancestors
|
||||
.iter()
|
||||
.map(|a| {
|
||||
let data = a.wait_cloned();
|
||||
(data.hashed_state, data.trie_updates)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
// Merge state and nodes in parallel with each other using tree reduction
|
||||
let (state, nodes) = rayon::join(
|
||||
|| {
|
||||
let mut merged = HashedPostStateSorted::merge_parallel(&states);
|
||||
merged.extend_ref_and_sort(sorted_hashed_state);
|
||||
merged
|
||||
},
|
||||
|| {
|
||||
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
|
||||
merged.extend_ref_and_sort(sorted_trie_updates);
|
||||
merged
|
||||
},
|
||||
);
|
||||
|
||||
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
|
||||
}
|
||||
|
||||
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
@@ -307,8 +284,17 @@ impl DeferredTrieData {
|
||||
}
|
||||
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
#[cfg(feature = "rayon")]
|
||||
rayon::join(
|
||||
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|
||||
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
}
|
||||
|
||||
overlay
|
||||
}
|
||||
|
||||
@@ -17,7 +17,10 @@ use reth_primitives_traits::{
|
||||
SignedTransaction,
|
||||
};
|
||||
use reth_storage_api::StateProviderBox;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
|
||||
use reth_trie::{
|
||||
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
|
||||
TrieInputSorted,
|
||||
};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
use tokio::sync::{broadcast, watch};
|
||||
|
||||
@@ -948,22 +951,36 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
|
||||
match blocks {
|
||||
[] => Chain::default(),
|
||||
[first, rest @ ..] => {
|
||||
let trie_data_handle = first.trie_data_handle();
|
||||
let mut chain = Chain::from_block(
|
||||
first.recovered_block().clone(),
|
||||
ExecutionOutcome::from((
|
||||
first.execution_outcome().clone(),
|
||||
first.block_number(),
|
||||
)),
|
||||
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
|
||||
LazyTrieData::deferred(move || {
|
||||
let trie_data = trie_data_handle.wait_cloned();
|
||||
SortedTrieData {
|
||||
hashed_state: trie_data.hashed_state,
|
||||
trie_updates: trie_data.trie_updates,
|
||||
}
|
||||
}),
|
||||
);
|
||||
for exec in rest {
|
||||
let trie_data_handle = exec.trie_data_handle();
|
||||
chain.append_block(
|
||||
exec.recovered_block().clone(),
|
||||
ExecutionOutcome::from((
|
||||
exec.execution_outcome().clone(),
|
||||
exec.block_number(),
|
||||
)),
|
||||
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
|
||||
LazyTrieData::deferred(move || {
|
||||
let trie_data = trie_data_handle.wait_cloned();
|
||||
SortedTrieData {
|
||||
hashed_state: trie_data.hashed_state,
|
||||
trie_updates: trie_data.trie_updates,
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
chain
|
||||
|
||||
@@ -25,6 +25,7 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
|
||||
/// Re-export for convenience
|
||||
pub use reth_ethereum_forks::*;
|
||||
|
||||
pub use alloy_evm::EvmLimitParams;
|
||||
pub use api::EthChainSpec;
|
||||
pub use info::ChainInfo;
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
|
||||
@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
|
||||
}
|
||||
}
|
||||
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,6 +78,7 @@ lz4.workspace = true
|
||||
zstd.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
parking_lot.workspace = true
|
||||
tar.workspace = true
|
||||
tracing.workspace = true
|
||||
backon.workspace = true
|
||||
@@ -132,4 +133,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]
|
||||
|
||||
@@ -121,13 +121,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
|
||||
let (db, sfp) = match access {
|
||||
AccessRights::RW => (
|
||||
Arc::new(init_db(db_path, self.db.database_args())?),
|
||||
init_db(db_path, self.db.database_args())?,
|
||||
StaticFileProviderBuilder::read_write(sf_path)
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
|
||||
(open_db_read_only(&db_path, self.db.database_args())?, {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
@@ -160,16 +160,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
fn create_provider_factory<N: CliNodeTypes>(
|
||||
&self,
|
||||
config: &Config,
|
||||
db: Arc<DatabaseEnv>,
|
||||
db: DatabaseEnv,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
access: AccessRights,
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let prune_modes = config.prune.segments.clone();
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
|
||||
db,
|
||||
self.chain.clone(),
|
||||
static_file_provider,
|
||||
@@ -200,7 +200,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
|
||||
|
||||
// Builds and executes an unwind-only pipeline
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
|
||||
.add_stages(DefaultStages::new(
|
||||
factory.clone(),
|
||||
tip_rx,
|
||||
@@ -229,7 +229,7 @@ pub struct Environment<N: NodeTypes> {
|
||||
/// Configuration for reth node
|
||||
pub config: Config,
|
||||
/// Provider factory.
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
/// Datadir path.
|
||||
pub data_dir: ChainPath<DataDirPath>,
|
||||
}
|
||||
@@ -261,8 +261,8 @@ impl AccessRights {
|
||||
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
|
||||
type FullTypesAdapter<T> = FullNodeTypesAdapter<
|
||||
T,
|
||||
Arc<DatabaseEnv>,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
DatabaseEnv,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
>;
|
||||
|
||||
/// Helper trait with a common set of requirements for the
|
||||
|
||||
@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
@@ -90,7 +89,7 @@ impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use std::{hash::Hasher, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
@@ -36,7 +36,7 @@ impl RocksDbTable {
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
|
||||
@@ -16,7 +16,6 @@ use std::{
|
||||
hash::Hash,
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -56,7 +55,7 @@ impl Command {
|
||||
/// then written to a file in the output directory.
|
||||
pub fn execute<T: NodeTypes>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("Make sure the node is not running when running `reth db diff`!");
|
||||
// open second db
|
||||
|
||||
@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
|
||||
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
|
||||
use reth_db_common::{DbTool, ListFilter};
|
||||
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
|
||||
use std::{cell::RefCell, sync::Arc};
|
||||
use std::cell::RefCell;
|
||||
use tracing::error;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -55,7 +55,7 @@ impl Command {
|
||||
/// Execute `db list` command
|
||||
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
self.table.view(&ListTableViewer { tool, args: &self })
|
||||
}
|
||||
@@ -89,7 +89,7 @@ impl Command {
|
||||
}
|
||||
|
||||
struct ListTableViewer<'a, N: NodeTypes> {
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
args: &'a Command,
|
||||
}
|
||||
|
||||
@@ -101,8 +101,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
|
||||
// We may be using the tui for a long time
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
|
||||
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
|
||||
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let total_entries = stats.entries();
|
||||
let final_entry_idx = total_entries.saturating_sub(1);
|
||||
if self.args.skip > final_entry_idx {
|
||||
|
||||
@@ -17,6 +17,7 @@ mod get;
|
||||
mod list;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
/// DB List TUI
|
||||
@@ -65,6 +66,8 @@ pub enum Subcommands {
|
||||
Settings(settings::Command),
|
||||
/// Gets storage size information for an account
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
}
|
||||
|
||||
/// Initializes a provider factory with specified access rights, and then execute with the provided
|
||||
@@ -198,6 +201,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::State(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
412
crates/cli/commands/src/db/state.rs
Normal file
412
crates/cli/commands/src/db/state.rs
Normal file
@@ -0,0 +1,412 @@
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use clap::Parser;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRO},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_provider::providers::ProviderNodeTypes;
|
||||
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{error, info};
|
||||
|
||||
/// Log progress every 5 seconds
|
||||
const LOG_INTERVAL: Duration = Duration::from_secs(30);
|
||||
|
||||
/// The arguments for the `reth db state` command
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// The account address to get state for
|
||||
address: Address,
|
||||
|
||||
/// Block number to query state at (uses current state if not provided)
|
||||
#[arg(long, short)]
|
||||
block: Option<BlockNumber>,
|
||||
|
||||
/// Maximum number of storage slots to display
|
||||
#[arg(long, short, default_value = "100")]
|
||||
limit: usize,
|
||||
|
||||
/// Output format (table, json, csv)
|
||||
#[arg(long, short, default_value = "table")]
|
||||
format: OutputFormat,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db state` command
|
||||
pub fn execute<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
self,
|
||||
tool: &DbTool<N>,
|
||||
) -> eyre::Result<()> {
|
||||
let address = self.address;
|
||||
let limit = self.limit;
|
||||
|
||||
if let Some(block) = self.block {
|
||||
self.execute_historical(tool, address, block, limit)
|
||||
} else {
|
||||
self.execute_current(tool, address, limit)
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_current<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let entries = tool.provider_factory.db_ref().view(|tx| {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>((account, entries))
|
||||
})??;
|
||||
|
||||
let (account, storage_entries) = entries;
|
||||
|
||||
self.print_results(address, None, account, &storage_entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn execute_historical<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
block: BlockNumber,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let provider = tool.provider_factory.history_by_block_number(block)?;
|
||||
|
||||
// Get account info at that block
|
||||
let account = provider.basic_account(&address)?;
|
||||
|
||||
// Check storage settings to determine where history is stored
|
||||
let storage_settings = tool.provider_factory.cached_storage_settings();
|
||||
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
|
||||
|
||||
// For historical queries, enumerate keys from history indices only
|
||||
// (not PlainStorageState, which reflects current state)
|
||||
let mut storage_keys = BTreeSet::new();
|
||||
|
||||
if history_in_rocksdb {
|
||||
error!(
|
||||
target: "reth::cli",
|
||||
"Historical storage queries with RocksDB backend are not yet supported. \
|
||||
Use MDBX for storage history or query current state without --block."
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Collect keys from MDBX StorageChangeSets using parallel scanning
|
||||
self.collect_mdbx_storage_keys_parallel(tool, address, &mut storage_keys)?;
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
block = block,
|
||||
total_keys = storage_keys.len(),
|
||||
"Found storage keys to query"
|
||||
);
|
||||
|
||||
// Now query each key at the historical block using the StateProvider
|
||||
// This handles both MDBX and RocksDB backends transparently
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
for (idx, key) in storage_keys.iter().enumerate() {
|
||||
match provider.storage(address, *key) {
|
||||
Ok(Some(value)) if value != U256::ZERO => {
|
||||
entries.push((*key, value));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
block = block,
|
||||
keys_total = storage_keys.len(),
|
||||
slots_scanned = idx,
|
||||
slots_found = entries.len(),
|
||||
"Scanning historical storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
self.print_results(address, Some(block), account, &entries);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Collects storage keys from MDBX StorageChangeSets using parallel block range scanning.
|
||||
fn collect_mdbx_storage_keys_parallel<N: NodeTypesWithDB + ProviderNodeTypes>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
address: Address,
|
||||
keys: &mut BTreeSet<B256>,
|
||||
) -> eyre::Result<()> {
|
||||
const CHUNK_SIZE: u64 = 500_000; // 500k blocks per thread
|
||||
let num_threads = std::thread::available_parallelism()
|
||||
.map(|p| p.get().saturating_sub(1).max(1))
|
||||
.unwrap_or(4);
|
||||
|
||||
// Get the current tip block
|
||||
let tip = tool.provider_factory.provider()?.best_block_number()?;
|
||||
|
||||
if tip == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
tip,
|
||||
chunk_size = CHUNK_SIZE,
|
||||
num_threads,
|
||||
"Starting parallel MDBX changeset scan"
|
||||
);
|
||||
|
||||
// Shared state for collecting keys
|
||||
let collected_keys: Mutex<BTreeSet<B256>> = Mutex::new(BTreeSet::new());
|
||||
let total_entries_scanned = Mutex::new(0usize);
|
||||
|
||||
// Create chunk ranges
|
||||
let mut chunks: Vec<(u64, u64)> = Vec::new();
|
||||
let mut start = 0u64;
|
||||
while start <= tip {
|
||||
let end = (start + CHUNK_SIZE - 1).min(tip);
|
||||
chunks.push((start, end));
|
||||
start = end + 1;
|
||||
}
|
||||
|
||||
let chunks_ref = &chunks;
|
||||
let next_chunk = Mutex::new(0usize);
|
||||
let next_chunk_ref = &next_chunk;
|
||||
let collected_keys_ref = &collected_keys;
|
||||
let total_entries_ref = &total_entries_scanned;
|
||||
|
||||
thread::scope(|s| {
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|thread_id| {
|
||||
s.spawn(move || {
|
||||
loop {
|
||||
// Get next chunk to process
|
||||
let chunk_idx = {
|
||||
let mut idx = next_chunk_ref.lock();
|
||||
if *idx >= chunks_ref.len() {
|
||||
return Ok::<_, eyre::Report>(());
|
||||
}
|
||||
let current = *idx;
|
||||
*idx += 1;
|
||||
current
|
||||
};
|
||||
|
||||
let (chunk_start, chunk_end) = chunks_ref[chunk_idx];
|
||||
|
||||
// Open a new read transaction for this chunk
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
let mut changeset_cursor =
|
||||
tx.cursor_read::<tables::StorageChangeSets>()?;
|
||||
let start_key =
|
||||
reth_db_api::models::BlockNumberAddress((chunk_start, address));
|
||||
let end_key =
|
||||
reth_db_api::models::BlockNumberAddress((chunk_end, address));
|
||||
|
||||
let mut local_keys = BTreeSet::new();
|
||||
let mut entries_in_chunk = 0usize;
|
||||
|
||||
if let Ok(walker) = changeset_cursor.walk_range(start_key..=end_key)
|
||||
{
|
||||
for (block_addr, storage_entry) in walker.flatten() {
|
||||
if block_addr.address() == address {
|
||||
local_keys.insert(storage_entry.key);
|
||||
}
|
||||
entries_in_chunk += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Merge into global state
|
||||
collected_keys_ref.lock().extend(local_keys);
|
||||
*total_entries_ref.lock() += entries_in_chunk;
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
thread_id,
|
||||
chunk_start,
|
||||
chunk_end,
|
||||
entries_in_chunk,
|
||||
"Thread completed chunk"
|
||||
);
|
||||
|
||||
Ok::<_, eyre::Report>(())
|
||||
})??;
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
for handle in handles {
|
||||
handle.join().map_err(|_| eyre::eyre!("Thread panicked"))??;
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>(())
|
||||
})?;
|
||||
|
||||
let final_keys = collected_keys.into_inner();
|
||||
let total = *total_entries_scanned.lock();
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
total_entries = total,
|
||||
unique_keys = final_keys.len(),
|
||||
"Finished parallel MDBX changeset scan"
|
||||
);
|
||||
|
||||
keys.extend(final_keys);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn print_results(
|
||||
&self,
|
||||
address: Address,
|
||||
block: Option<BlockNumber>,
|
||||
account: Option<reth_primitives_traits::Account>,
|
||||
storage: &[(alloy_primitives::B256, U256)],
|
||||
) {
|
||||
match self.format {
|
||||
OutputFormat::Table => {
|
||||
println!("Account: {address}");
|
||||
if let Some(b) = block {
|
||||
println!("Block: {b}");
|
||||
} else {
|
||||
println!("Block: latest");
|
||||
}
|
||||
println!();
|
||||
|
||||
if let Some(acc) = account {
|
||||
println!("Nonce: {}", acc.nonce);
|
||||
println!("Balance: {} wei", acc.balance);
|
||||
if let Some(code_hash) = acc.bytecode_hash {
|
||||
println!("Code hash: {code_hash}");
|
||||
}
|
||||
} else {
|
||||
println!("Account not found");
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("Storage ({} slots):", storage.len());
|
||||
println!("{:-<130}", "");
|
||||
println!("{:<66} | {:<64}", "Slot", "Value");
|
||||
println!("{:-<130}", "");
|
||||
for (key, value) in storage {
|
||||
println!("{key} | {value:#066x}");
|
||||
}
|
||||
}
|
||||
OutputFormat::Json => {
|
||||
let output = serde_json::json!({
|
||||
"address": address.to_string(),
|
||||
"block": block,
|
||||
"account": account.map(|a| serde_json::json!({
|
||||
"nonce": a.nonce,
|
||||
"balance": a.balance.to_string(),
|
||||
"code_hash": a.bytecode_hash.map(|h| h.to_string()),
|
||||
})),
|
||||
"storage": storage.iter().map(|(k, v)| {
|
||||
serde_json::json!({
|
||||
"key": k.to_string(),
|
||||
"value": format!("{v:#066x}"),
|
||||
})
|
||||
}).collect::<Vec<_>>(),
|
||||
});
|
||||
println!("{}", serde_json::to_string_pretty(&output).unwrap());
|
||||
}
|
||||
OutputFormat::Csv => {
|
||||
println!("slot,value");
|
||||
for (key, value) in storage {
|
||||
println!("{key},{value:#066x}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, clap::ValueEnum)]
|
||||
pub enum OutputFormat {
|
||||
#[default]
|
||||
Table,
|
||||
Json,
|
||||
Csv,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parse_state_args() {
|
||||
let cmd = Command::try_parse_from([
|
||||
"state",
|
||||
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045",
|
||||
"--block",
|
||||
"1000000",
|
||||
])
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
cmd.address,
|
||||
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045".parse::<Address>().unwrap()
|
||||
);
|
||||
assert_eq!(cmd.block, Some(1000000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_state_args_no_block() {
|
||||
let cmd = Command::try_parse_from(["state", "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"])
|
||||
.unwrap();
|
||||
assert_eq!(cmd.block, None);
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,7 @@ use reth_provider::{
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db stats` command
|
||||
@@ -48,7 +48,7 @@ impl Command {
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
if self.checksum {
|
||||
let checksum_report = self.checksum_report(tool)?;
|
||||
@@ -72,7 +72,7 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
) -> eyre::Result<ComfyTable> {
|
||||
@@ -92,10 +92,10 @@ impl Command {
|
||||
db_tables.sort();
|
||||
let mut total_size = 0;
|
||||
for db_table in db_tables {
|
||||
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
|
||||
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
|
||||
|
||||
let stats = tx
|
||||
.inner
|
||||
.inner()
|
||||
.db_stat(table_db.dbi())
|
||||
.wrap_err(format!("Could not find table: {db_table}"))?;
|
||||
|
||||
@@ -136,9 +136,9 @@ impl Command {
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)));
|
||||
table.add_row(row);
|
||||
|
||||
let freelist = tx.inner.env().freelist()?;
|
||||
let freelist = tx.inner().env().freelist()?;
|
||||
let pagesize =
|
||||
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
|
||||
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
|
||||
let freelist_size = freelist * pagesize;
|
||||
|
||||
let mut row = Row::new();
|
||||
@@ -205,6 +205,16 @@ impl Command {
|
||||
.add_cell(Cell::new(human_bytes(total_size as f64)))
|
||||
.add_cell(Cell::new(human_bytes(total_pending as f64)));
|
||||
table.add_row(row);
|
||||
|
||||
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
|
||||
let mut row = Row::new();
|
||||
row.add_cell(Cell::new("WAL"))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(""))
|
||||
.add_cell(Cell::new(human_bytes(wal_size as f64)))
|
||||
.add_cell(Cell::new(""));
|
||||
table.add_row(row);
|
||||
}
|
||||
|
||||
table
|
||||
|
||||
@@ -227,8 +227,9 @@ where
|
||||
|
||||
// Handle errors
|
||||
if let Err(err) = res {
|
||||
error!("{:?}", err)
|
||||
error!("{err}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
|
||||
) -> io::Result<()>
|
||||
where
|
||||
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
|
||||
io::Error: From<B::Error>,
|
||||
{
|
||||
let mut last_tick = Instant::now();
|
||||
let mut running = true;
|
||||
|
||||
@@ -26,6 +26,14 @@ pub struct ImportCommand<C: ChainSpecParser> {
|
||||
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
|
||||
chunk_len: Option<u64>,
|
||||
|
||||
/// Fail immediately when an invalid block is encountered.
|
||||
///
|
||||
/// By default, the import will stop at the last valid block if an invalid block is
|
||||
/// encountered during execution or validation, leaving the database at the last valid
|
||||
/// block state. When this flag is set, the import will instead fail with an error.
|
||||
#[arg(long, verbatim_doc_comment)]
|
||||
fail_on_invalid_block: bool,
|
||||
|
||||
/// The path(s) to block file(s) for import.
|
||||
///
|
||||
/// The online stages (headers and bodies) are replaced by a file import, after which the
|
||||
@@ -52,7 +60,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
|
||||
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
|
||||
|
||||
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
|
||||
let import_config = ImportConfig {
|
||||
no_state: self.no_state,
|
||||
chunk_len: self.chunk_len,
|
||||
fail_on_invalid_block: self.fail_on_invalid_block,
|
||||
};
|
||||
|
||||
let executor = components.evm_config().clone();
|
||||
let consensus = Arc::new(components.consensus().clone());
|
||||
@@ -81,7 +93,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
total_decoded_blocks += result.total_decoded_blocks;
|
||||
total_decoded_txns += result.total_decoded_txns;
|
||||
|
||||
if !result.is_complete() {
|
||||
// Check if we stopped due to an invalid block
|
||||
if result.stopped_on_invalid_block {
|
||||
info!(target: "reth::cli",
|
||||
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
|
||||
result.last_valid_block.unwrap_or(0),
|
||||
result.bad_block.unwrap_or(0),
|
||||
path.display(),
|
||||
result.total_imported_blocks,
|
||||
result.total_imported_txns);
|
||||
// Stop importing further files and exit successfully
|
||||
break;
|
||||
}
|
||||
|
||||
if !result.is_successful() {
|
||||
return Err(eyre::eyre!(
|
||||
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
|
||||
path.display(),
|
||||
@@ -98,7 +123,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
}
|
||||
|
||||
info!(target: "reth::cli",
|
||||
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
|
||||
"Import complete. Total: {}/{} blocks, {}/{} transactions",
|
||||
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
|
||||
|
||||
Ok(())
|
||||
@@ -139,4 +164,20 @@ mod tests {
|
||||
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
|
||||
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_import_command_with_fail_on_invalid_block() {
|
||||
let args: ImportCommand<EthereumChainSpecParser> =
|
||||
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
|
||||
assert!(args.fail_on_invalid_block);
|
||||
assert_eq!(args.paths.len(), 1);
|
||||
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_import_command_default_stops_on_invalid_block() {
|
||||
let args: ImportCommand<EthereumChainSpecParser> =
|
||||
ImportCommand::parse_from(["reth", "chain.rlp"]);
|
||||
assert!(!args.fail_on_invalid_block);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,11 +22,11 @@ use reth_provider::{
|
||||
StageCheckpointReader,
|
||||
};
|
||||
use reth_prune::PruneModes;
|
||||
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
|
||||
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::{path::Path, sync::Arc};
|
||||
use tokio::sync::watch;
|
||||
use tracing::{debug, error, info};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Configuration for importing blocks from RLP files.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -35,6 +35,9 @@ pub struct ImportConfig {
|
||||
pub no_state: bool,
|
||||
/// Chunk byte length to read from file.
|
||||
pub chunk_len: Option<u64>,
|
||||
/// If true, fail immediately when an invalid block is encountered.
|
||||
/// By default (false), the import stops at the last valid block and exits successfully.
|
||||
pub fail_on_invalid_block: bool,
|
||||
}
|
||||
|
||||
/// Result of an import operation.
|
||||
@@ -48,6 +51,12 @@ pub struct ImportResult {
|
||||
pub total_imported_blocks: usize,
|
||||
/// Total number of transactions imported into the database.
|
||||
pub total_imported_txns: usize,
|
||||
/// Whether the import was stopped due to an invalid block.
|
||||
pub stopped_on_invalid_block: bool,
|
||||
/// The block number that was invalid, if any.
|
||||
pub bad_block: Option<u64>,
|
||||
/// The last valid block number when stopped due to invalid block.
|
||||
pub last_valid_block: Option<u64>,
|
||||
}
|
||||
|
||||
impl ImportResult {
|
||||
@@ -56,6 +65,14 @@ impl ImportResult {
|
||||
self.total_decoded_blocks == self.total_imported_blocks &&
|
||||
self.total_decoded_txns == self.total_imported_txns
|
||||
}
|
||||
|
||||
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
|
||||
///
|
||||
/// In stop-on-invalid-block mode, a partial import is considered successful if we
|
||||
/// stopped due to an invalid block (leaving the DB at the last valid block).
|
||||
pub fn is_successful(&self) -> bool {
|
||||
self.is_complete() || self.stopped_on_invalid_block
|
||||
}
|
||||
}
|
||||
|
||||
/// Imports blocks from an RLP-encoded file into the database.
|
||||
@@ -103,6 +120,11 @@ where
|
||||
let static_file_producer =
|
||||
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
|
||||
|
||||
// Track if we stopped due to an invalid block
|
||||
let mut stopped_on_invalid_block = false;
|
||||
let mut bad_block_number: Option<u64> = None;
|
||||
let mut last_valid_block_number: Option<u64> = None;
|
||||
|
||||
while let Some(file_client) =
|
||||
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
|
||||
{
|
||||
@@ -137,12 +159,51 @@ where
|
||||
|
||||
// Run pipeline
|
||||
info!(target: "reth::import", "Starting sync pipeline");
|
||||
tokio::select! {
|
||||
res = pipeline.run() => res?,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
if import_config.fail_on_invalid_block {
|
||||
// Original behavior: fail on unwind
|
||||
tokio::select! {
|
||||
res = pipeline.run() => res?,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
}
|
||||
} else {
|
||||
// Default behavior: Use run_loop() to handle unwinds gracefully
|
||||
let result = tokio::select! {
|
||||
res = pipeline.run_loop() => res,
|
||||
_ = tokio::signal::ctrl_c() => {
|
||||
info!(target: "reth::import", "Import interrupted by user");
|
||||
break;
|
||||
},
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(ControlFlow::Unwind { target, bad_block }) => {
|
||||
// An invalid block was encountered; stop at last valid block
|
||||
let bad = bad_block.block.number;
|
||||
warn!(
|
||||
target: "reth::import",
|
||||
bad_block = bad,
|
||||
last_valid_block = target,
|
||||
"Invalid block encountered during import; stopping at last valid block"
|
||||
);
|
||||
stopped_on_invalid_block = true;
|
||||
bad_block_number = Some(bad);
|
||||
last_valid_block_number = Some(target);
|
||||
break;
|
||||
}
|
||||
Ok(ControlFlow::Continue { block_number }) => {
|
||||
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
|
||||
}
|
||||
Ok(ControlFlow::NoProgress { block_number }) => {
|
||||
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
|
||||
}
|
||||
Err(e) => {
|
||||
// Propagate other pipeline errors
|
||||
return Err(e.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sealed_header = provider_factory
|
||||
@@ -160,9 +221,20 @@ where
|
||||
total_decoded_txns,
|
||||
total_imported_blocks,
|
||||
total_imported_txns,
|
||||
stopped_on_invalid_block,
|
||||
bad_block: bad_block_number,
|
||||
last_valid_block: last_valid_block_number,
|
||||
};
|
||||
|
||||
if !result.is_complete() {
|
||||
if result.stopped_on_invalid_block {
|
||||
info!(target: "reth::import",
|
||||
total_imported_blocks,
|
||||
total_imported_txns,
|
||||
bad_block = ?result.bad_block,
|
||||
last_valid_block = ?result.last_valid_block,
|
||||
"Import stopped at last valid block due to invalid block"
|
||||
);
|
||||
} else if !result.is_complete() {
|
||||
error!(target: "reth::import",
|
||||
total_decoded_blocks,
|
||||
total_imported_blocks,
|
||||
|
||||
@@ -2,7 +2,7 @@ use futures::Future;
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_node_builder::{NodeBuilder, WithLaunchContext};
|
||||
use std::{fmt, sync::Arc};
|
||||
use std::fmt;
|
||||
|
||||
/// A trait for launching a reth node with custom configuration strategies.
|
||||
///
|
||||
@@ -30,7 +30,7 @@ where
|
||||
/// * `builder_args` - Extension arguments for configuration
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>>;
|
||||
}
|
||||
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
|
||||
where
|
||||
C: ChainSpecParser,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
@@ -77,13 +77,13 @@ where
|
||||
C: ChainSpecParser,
|
||||
Ext: clap::Args + fmt::Debug,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>> {
|
||||
(self.func)(builder, builder_args)
|
||||
|
||||
@@ -206,7 +206,7 @@ where
|
||||
let db_path = data_dir.db();
|
||||
|
||||
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
|
||||
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
|
||||
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
|
||||
|
||||
if with_unused_ports {
|
||||
node_config = node_config.with_unused_ports();
|
||||
|
||||
34
crates/cli/commands/src/p2p/enode.rs
Normal file
34
crates/cli/commands/src/p2p/enode.rs
Normal file
@@ -0,0 +1,34 @@
|
||||
//! Enode identifier command
|
||||
|
||||
use clap::Parser;
|
||||
use reth_cli_util::get_secret_key;
|
||||
use reth_network_peers::NodeRecord;
|
||||
use std::{
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr},
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
/// Print the enode identifier for a given secret key.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Path to the secret key file for discovery.
|
||||
pub discovery_secret: PathBuf,
|
||||
|
||||
/// Optional IP address to include in the enode URL.
|
||||
///
|
||||
/// If not provided, defaults to 0.0.0.0.
|
||||
#[arg(long)]
|
||||
pub ip: Option<IpAddr>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the enode command.
|
||||
pub fn execute(self) -> eyre::Result<()> {
|
||||
let sk = get_secret_key(&self.discovery_secret)?;
|
||||
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
|
||||
let addr = SocketAddr::new(ip, 30303);
|
||||
let enr = NodeRecord::from_secret_key(addr, &sk);
|
||||
println!("{enr}");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -18,6 +18,7 @@ use reth_node_core::{
|
||||
};
|
||||
|
||||
pub mod bootnode;
|
||||
pub mod enode;
|
||||
pub mod rlpx;
|
||||
|
||||
/// `reth p2p` command
|
||||
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Subcommands::Bootnode(command) => {
|
||||
command.execute().await?;
|
||||
}
|
||||
Subcommands::Enode(command) => {
|
||||
command.execute()?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
Subcommands::Body { args, .. } => Some(&args.chain),
|
||||
Subcommands::Rlpx(_) => None,
|
||||
Subcommands::Bootnode(_) => None,
|
||||
Subcommands::Enode(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
|
||||
Rlpx(rlpx::Command),
|
||||
/// Bootnode command
|
||||
Bootnode(bootnode::Command),
|
||||
/// Print enode identifier
|
||||
Enode(enode::Command),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
@@ -225,4 +232,16 @@ mod tests {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_enode_cmd_with_ip() {
|
||||
let _args: Command<EthereumChainSpecParser> =
|
||||
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,26 +1,62 @@
|
||||
//! Command that runs pruning without any limits.
|
||||
//! Command that runs pruning.
|
||||
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{args::MetricArgs, version::version_metadata};
|
||||
use reth_node_metrics::{
|
||||
chain::ChainSpecInfo,
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
use reth_prune::PrunerBuilder;
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
/// Prunes according to the configuration without any limits
|
||||
/// Prunes according to the configuration
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct PruneCommand<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
env: EnvironmentArgs<C>,
|
||||
|
||||
/// Prometheus metrics configuration.
|
||||
#[command(flatten)]
|
||||
metrics: MetricArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
|
||||
/// Execute the `prune` command
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
|
||||
self,
|
||||
ctx: CliContext,
|
||||
) -> eyre::Result<()> {
|
||||
let env = self.env.init::<N>(AccessRights::RW)?;
|
||||
let provider_factory = env.provider_factory;
|
||||
let config = env.config.prune;
|
||||
let data_dir = env.data_dir;
|
||||
|
||||
if let Some(listen_addr) = self.metrics.prometheus {
|
||||
let config = MetricServerConfig::new(
|
||||
listen_addr,
|
||||
VersionInfo {
|
||||
version: version_metadata().cargo_pkg_version.as_ref(),
|
||||
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
|
||||
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
|
||||
git_sha: version_metadata().vergen_git_sha.as_ref(),
|
||||
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
|
||||
build_profile: version_metadata().build_profile_name.as_ref(),
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
|
||||
MetricServer::new(config).serve().await?;
|
||||
}
|
||||
|
||||
// Copy data from database to static files
|
||||
info!(target: "reth::cli", "Copying data from database to static files...");
|
||||
@@ -33,13 +69,43 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
// Delete data which has been copied to static files.
|
||||
if let Some(prune_tip) = lowest_static_file_height {
|
||||
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
|
||||
// Run the pruner according to the configuration, and don't enforce any limits on it
|
||||
|
||||
// Use batched pruning with a limit to bound memory, running in a loop until complete.
|
||||
const DELETE_LIMIT: usize = 200_000;
|
||||
let mut pruner = PrunerBuilder::new(config)
|
||||
.delete_limit(usize::MAX)
|
||||
.delete_limit(DELETE_LIMIT)
|
||||
.build_with_provider_factory(provider_factory);
|
||||
|
||||
pruner.run(prune_tip)?;
|
||||
info!(target: "reth::cli", "Pruned data from database");
|
||||
let mut total_pruned = 0usize;
|
||||
loop {
|
||||
let output = pruner.run(prune_tip)?;
|
||||
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
|
||||
total_pruned = total_pruned.saturating_add(batch_pruned);
|
||||
|
||||
// Check if all segments are finished (not just the overall progress,
|
||||
// since the pruner sets overall progress from the last segment only)
|
||||
let all_segments_finished =
|
||||
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
|
||||
|
||||
if all_segments_finished {
|
||||
break;
|
||||
}
|
||||
|
||||
if batch_pruned == 0 {
|
||||
return Err(eyre::eyre!(
|
||||
"pruner made no progress but reported more data remaining; \
|
||||
aborting to prevent infinite loop"
|
||||
));
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
batch_pruned,
|
||||
total_pruned,
|
||||
"Pruning batch complete, continuing..."
|
||||
);
|
||||
}
|
||||
info!(target: "reth::cli", total_pruned, "Pruned data from database");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
|
||||
consensus: C,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
C: FullConsensus<E::Primitives> + 'static,
|
||||
{
|
||||
@@ -39,7 +39,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -10,10 +10,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: BlockNumber,
|
||||
to: BlockNumber,
|
||||
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -9,10 +9,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: u64,
|
||||
to: u64,
|
||||
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
|
||||
consensus: impl FullConsensus<N::Primitives> + 'static,
|
||||
) -> Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
{
|
||||
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
|
||||
|
||||
@@ -59,7 +59,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -158,7 +158,7 @@ enum Subcommands {
|
||||
|
||||
impl Subcommands {
|
||||
/// Returns the block to unwind to. The returned block will stay in database.
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
factory: ProviderFactory<N>,
|
||||
) -> eyre::Result<u64> {
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
//! Collection of methods for block validation.
|
||||
|
||||
use alloy_consensus::{BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_consensus::{BlockHeader as _, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams};
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks};
|
||||
use reth_consensus::{ConsensusError, TxGasLimitTooHighErr};
|
||||
use reth_consensus::ConsensusError;
|
||||
use reth_primitives_traits::{
|
||||
constants::{
|
||||
GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT,
|
||||
},
|
||||
transaction::TxHashRef,
|
||||
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
|
||||
Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader,
|
||||
};
|
||||
|
||||
@@ -146,7 +143,7 @@ pub fn validate_block_pre_execution<B, ChainSpec>(
|
||||
) -> Result<(), ConsensusError>
|
||||
where
|
||||
B: Block,
|
||||
ChainSpec: EthereumHardforks,
|
||||
ChainSpec: EthChainSpec + EthereumHardforks,
|
||||
{
|
||||
post_merge_hardfork_fields(block, chain_spec)?;
|
||||
|
||||
@@ -154,19 +151,6 @@ where
|
||||
if let Err(error) = block.ensure_transaction_root_valid() {
|
||||
return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
|
||||
}
|
||||
// EIP-7825 validation
|
||||
if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) {
|
||||
for tx in block.body().transactions() {
|
||||
if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA {
|
||||
return Err(TxGasLimitTooHighErr {
|
||||
tx_hash: *tx.tx_hash(),
|
||||
gas_limit: tx.gas_limit(),
|
||||
max_allowed: MAX_TX_GAS_LIMIT_OSAKA,
|
||||
}
|
||||
.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ auto_impl.workspace = true
|
||||
derive_more.workspace = true
|
||||
futures.workspace = true
|
||||
eyre.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls"] }
|
||||
reqwest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
tokio = { workspace = true, features = ["time"] }
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -38,12 +38,8 @@ pub trait BlockProvider: Send + Sync + 'static {
|
||||
offset: usize,
|
||||
) -> impl Future<Output = eyre::Result<B256>> + Send {
|
||||
async move {
|
||||
let stored_hash = previous_block_hashes
|
||||
.len()
|
||||
.checked_sub(offset)
|
||||
.and_then(|index| previous_block_hashes.get(index));
|
||||
if let Some(hash) = stored_hash {
|
||||
return Ok(*hash);
|
||||
if let Some(hash) = get_hash_at_offset(previous_block_hashes, offset) {
|
||||
return Ok(hash);
|
||||
}
|
||||
|
||||
// Return zero hash if the chain isn't long enough to have the block at the offset.
|
||||
@@ -83,7 +79,7 @@ where
|
||||
/// Spawn the client to start sending FCUs and new payloads by periodically fetching recent
|
||||
/// blocks.
|
||||
pub async fn run(self) {
|
||||
let mut previous_block_hashes = AllocRingBuffer::new(64);
|
||||
let mut previous_block_hashes = AllocRingBuffer::new(65);
|
||||
let mut block_stream = {
|
||||
let (tx, rx) = mpsc::channel::<P::Block>(64);
|
||||
let block_provider = self.block_provider.clone();
|
||||
@@ -99,7 +95,7 @@ where
|
||||
let block_hash = payload.block_hash();
|
||||
let block_number = payload.block_number();
|
||||
|
||||
previous_block_hashes.push(block_hash);
|
||||
previous_block_hashes.enqueue(block_hash);
|
||||
|
||||
// Send new events to execution client
|
||||
let _ = self.engine_handle.new_payload(payload).await;
|
||||
@@ -142,3 +138,60 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Looks up a block hash from the ring buffer at the given offset from the most recent entry.
|
||||
///
|
||||
/// Returns `None` if the buffer doesn't have enough entries to satisfy the offset.
|
||||
fn get_hash_at_offset(buffer: &AllocRingBuffer<B256>, offset: usize) -> Option<B256> {
|
||||
buffer.len().checked_sub(offset + 1).and_then(|index| buffer.get(index).copied())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_get_hash_at_offset() {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// Empty buffer returns None for any offset
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
|
||||
// Push hashes 0..65
|
||||
for i in 0..65u8 {
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
|
||||
// offset=0 should return the most recent (64)
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(64)));
|
||||
|
||||
// offset=32 (safe block) should return hash 32
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(32)));
|
||||
|
||||
// offset=64 (finalized block) should return hash 0 (the oldest)
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), Some(B256::with_last_byte(0)));
|
||||
|
||||
// offset=65 exceeds buffer, should return None
|
||||
assert_eq!(get_hash_at_offset(&buffer, 65), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_hash_at_offset_insufficient_entries() {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// With only 1 entry, only offset=0 works
|
||||
buffer.enqueue(B256::with_last_byte(1));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
|
||||
// With 33 entries, offset=32 works but offset=64 doesn't
|
||||
for i in 2..=33u8 {
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,3 +72,11 @@ derive_more.workspace = true
|
||||
[[test]]
|
||||
name = "e2e_testsuite"
|
||||
path = "tests/e2e-testsuite/main.rs"
|
||||
|
||||
[[test]]
|
||||
name = "rocksdb"
|
||||
path = "tests/rocksdb/main.rs"
|
||||
required-features = ["edge"]
|
||||
|
||||
[features]
|
||||
edge = ["reth-node-core/edge", "reth-provider/rocksdb", "reth-cli-commands/edge"]
|
||||
|
||||
@@ -103,7 +103,10 @@ where
|
||||
N: NodeBuilderHelper,
|
||||
{
|
||||
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
|
||||
.with_tree_config_modifier(move |_| tree_config.clone())
|
||||
.with_tree_config_modifier(move |base| {
|
||||
// Apply caller's tree_config but preserve the small cache size from base
|
||||
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
|
||||
})
|
||||
.with_node_config_modifier(move |config| config.set_dev(is_dev))
|
||||
.with_connect_nodes(connect_nodes)
|
||||
.build()
|
||||
|
||||
@@ -112,11 +112,13 @@ where
|
||||
..NetworkArgs::default()
|
||||
};
|
||||
|
||||
// Apply tree config modifier if present
|
||||
// Apply tree config modifier if present, with test-appropriate defaults
|
||||
let base_tree_config =
|
||||
reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024);
|
||||
let tree_config = if let Some(modifier) = self.tree_config_modifier {
|
||||
modifier(reth_node_api::TreeConfig::default())
|
||||
modifier(base_tree_config)
|
||||
} else {
|
||||
reth_node_api::TreeConfig::default()
|
||||
base_tree_config
|
||||
};
|
||||
|
||||
let mut nodes = (0..self.num_nodes)
|
||||
|
||||
@@ -114,19 +114,22 @@ pub async fn setup_engine_with_chain_import(
|
||||
|
||||
// Initialize the database using init_db (same as CLI import command)
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args)?;
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args)?;
|
||||
|
||||
// Create a provider factory with the initialized database (use regular DB, not
|
||||
// TempDatabase) We need to specify the node types properly for the adapter
|
||||
let provider_factory = ProviderFactory::<
|
||||
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
|
||||
>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
)?;
|
||||
let provider_factory =
|
||||
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(
|
||||
static_files_path.clone(),
|
||||
)?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
|
||||
// Initialize genesis if needed
|
||||
reth_db_common::init::init_genesis(&provider_factory)?;
|
||||
@@ -317,17 +320,17 @@ mod tests {
|
||||
// Import the chain
|
||||
{
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())
|
||||
.unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone())
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -381,17 +384,17 @@ mod tests {
|
||||
|
||||
// Now reopen the database and verify checkpoints are still there
|
||||
{
|
||||
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db,
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_only(static_files_path, false)
|
||||
.unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
@@ -490,7 +493,10 @@ mod tests {
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)
|
||||
.expect("failed to create provider factory");
|
||||
|
||||
|
||||
@@ -38,6 +38,18 @@ impl TransactionTestContext {
|
||||
signed.encoded_2718().into()
|
||||
}
|
||||
|
||||
/// Creates a transfer with a specific nonce and signs it, returning bytes.
|
||||
/// Uses high `max_fee_per_gas` (1000 gwei) to ensure tx acceptance regardless of basefee.
|
||||
pub async fn transfer_tx_bytes_with_nonce(
|
||||
chain_id: u64,
|
||||
wallet: PrivateKeySigner,
|
||||
nonce: u64,
|
||||
) -> Bytes {
|
||||
let tx = tx(chain_id, 21000, None, None, nonce, Some(1000e9 as u128));
|
||||
let signed = Self::sign_tx(wallet, tx).await;
|
||||
signed.encoded_2718().into()
|
||||
}
|
||||
|
||||
/// Creates a deployment transaction and signs it, returning an envelope.
|
||||
pub async fn deploy_tx(
|
||||
chain_id: u64,
|
||||
|
||||
591
crates/e2e-test-utils/tests/rocksdb/main.rs
Normal file
591
crates/e2e-test-utils/tests/rocksdb/main.rs
Normal file
@@ -0,0 +1,591 @@
|
||||
//! E2E tests for `RocksDB` provider functionality.
|
||||
|
||||
#![cfg(all(feature = "edge", unix))]
|
||||
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_eth::{Transaction, TransactionReceipt};
|
||||
use eyre::Result;
|
||||
use jsonrpsee::core::client::ClientT;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
use reth_db::tables;
|
||||
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
|
||||
use reth_node_core::args::RocksDbArgs;
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_payload_builder::EthPayloadBuilderAttributes;
|
||||
use reth_provider::{RocksDBProviderFactory, StorageSettings};
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const ROCKSDB_POLL_INTERVAL: Duration = Duration::from_millis(50);
|
||||
|
||||
/// Polls RPC until the given `tx_hash` is visible as pending (not yet mined).
|
||||
/// Prevents race conditions where `advance_block` is called before txs are in the pool.
|
||||
/// Returns the pending transaction.
|
||||
async fn wait_for_pending_tx<C: ClientT>(client: &C, tx_hash: B256) -> Transaction {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let tx: Option<Transaction> = client
|
||||
.request("eth_getTransactionByHash", [tx_hash])
|
||||
.await
|
||||
.expect("RPC request failed");
|
||||
if let Some(tx) = tx {
|
||||
assert!(
|
||||
tx.block_number.is_none(),
|
||||
"Expected pending tx but tx_hash={tx_hash:?} is already mined in block {:?}",
|
||||
tx.block_number
|
||||
);
|
||||
return tx;
|
||||
}
|
||||
assert!(
|
||||
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
|
||||
"Timed out after {:?} waiting for tx_hash={tx_hash:?} to appear in pending pool",
|
||||
start.elapsed()
|
||||
);
|
||||
tokio::time::sleep(ROCKSDB_POLL_INTERVAL).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers`.
|
||||
/// Returns the `tx_number` on success, or panics on timeout.
|
||||
async fn poll_tx_in_rocksdb<P: RocksDBProviderFactory>(provider: &P, tx_hash: B256) -> u64 {
|
||||
let start = std::time::Instant::now();
|
||||
let mut interval = ROCKSDB_POLL_INTERVAL;
|
||||
loop {
|
||||
// Re-acquire handle each iteration to avoid stale snapshot reads
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
let tx_number: Option<u64> =
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(tx_hash).expect("RocksDB get failed");
|
||||
if let Some(n) = tx_number {
|
||||
return n;
|
||||
}
|
||||
assert!(
|
||||
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
|
||||
"Timed out after {:?} waiting for tx_hash={tx_hash:?} in RocksDB",
|
||||
start.elapsed()
|
||||
);
|
||||
tokio::time::sleep(interval).await;
|
||||
// Simple backoff: 50ms -> 100ms -> 200ms (capped)
|
||||
interval = std::cmp::min(interval * 2, Duration::from_millis(200));
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the test chain spec for `RocksDB` tests.
|
||||
fn test_chain_spec() -> Arc<ChainSpec> {
|
||||
Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(
|
||||
serde_json::from_str(include_str!("../../src/testsuite/assets/genesis.json"))
|
||||
.expect("failed to parse genesis.json"),
|
||||
)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns test payload attributes for the given timestamp.
|
||||
fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
|
||||
let attributes = alloy_rpc_types_engine::PayloadAttributes {
|
||||
timestamp,
|
||||
prev_randao: B256::ZERO,
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
};
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
|
||||
/// Verifies that `RocksDB` CLI defaults match `StorageSettings::base()`.
|
||||
#[test]
|
||||
fn test_rocksdb_defaults_match_storage_settings() {
|
||||
let args = RocksDbArgs::default();
|
||||
let settings = StorageSettings::base();
|
||||
|
||||
assert_eq!(
|
||||
args.tx_hash, settings.transaction_hash_numbers_in_rocksdb,
|
||||
"tx_hash default should match StorageSettings::base()"
|
||||
);
|
||||
assert_eq!(
|
||||
args.storages_history, settings.storages_history_in_rocksdb,
|
||||
"storages_history default should match StorageSettings::base()"
|
||||
);
|
||||
assert_eq!(
|
||||
args.account_history, settings.account_history_in_rocksdb,
|
||||
"account_history default should match StorageSettings::base()"
|
||||
);
|
||||
}
|
||||
|
||||
/// Smoke test: node boots with `RocksDB` routing enabled.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_node_startup() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
|
||||
let (nodes, _tasks, _wallet) =
|
||||
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
// Verify RocksDB provider is functional (can query without error)
|
||||
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
|
||||
let missing_hash = B256::from([0xab; 32]);
|
||||
let result: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
|
||||
assert!(result.is_none(), "Missing hash should return None");
|
||||
|
||||
let genesis_hash = nodes[0].block_hash(0);
|
||||
assert_ne!(genesis_hash, B256::ZERO);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Block mining works with `RocksDB` storage.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_block_mining() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _wallet) =
|
||||
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
let genesis_hash = nodes[0].block_hash(0);
|
||||
assert_ne!(genesis_hash, B256::ZERO);
|
||||
|
||||
// Mine 3 blocks with transactions
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client should be available");
|
||||
|
||||
for i in 1..=3u64 {
|
||||
let raw_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), i - 1)
|
||||
.await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Wait for tx to enter pending pool before mining
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
let block = payload.block();
|
||||
assert_eq!(block.number(), i);
|
||||
assert_ne!(block.hash(), B256::ZERO);
|
||||
|
||||
// Verify tx was actually included in the block
|
||||
let receipt: Option<TransactionReceipt> =
|
||||
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
|
||||
let receipt = receipt.expect("Receipt should exist after mining");
|
||||
assert_eq!(receipt.block_number, Some(i), "Tx should be in block {i}");
|
||||
}
|
||||
|
||||
// Verify all blocks are stored
|
||||
for i in 0..=3 {
|
||||
let block_hash = nodes[0].block_hash(i);
|
||||
assert_ne!(block_hash, B256::ZERO);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tx hash lookup exercises `TransactionHashNumbers` table.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_transaction_queries() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
// Inject and mine a transaction
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client should be available");
|
||||
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Wait for tx to enter pending pool before mining
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
// Query each transaction by hash
|
||||
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
|
||||
let tx = tx.expect("Transaction should be found");
|
||||
assert_eq!(tx.block_number, Some(1));
|
||||
|
||||
let receipt: Option<TransactionReceipt> =
|
||||
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
|
||||
let receipt = receipt.expect("Receipt should be found");
|
||||
assert_eq!(receipt.block_number, Some(1));
|
||||
assert!(receipt.status());
|
||||
|
||||
// Direct RocksDB assertion - poll with timeout since persistence is async
|
||||
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
|
||||
assert_eq!(tx_number, 0, "First tx should have TxNumber 0");
|
||||
|
||||
// Verify missing hash returns None
|
||||
let missing_hash = B256::from([0xde; 32]);
|
||||
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
|
||||
let missing_tx_number: Option<u64> =
|
||||
rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
|
||||
assert!(missing_tx_number.is_none());
|
||||
|
||||
let missing_tx: Option<Transaction> =
|
||||
client.request("eth_getTransactionByHash", [missing_hash]).await?;
|
||||
assert!(missing_tx.is_none(), "expected no transaction for missing hash");
|
||||
|
||||
let missing_receipt: Option<TransactionReceipt> =
|
||||
client.request("eth_getTransactionReceipt", [missing_hash]).await?;
|
||||
assert!(missing_receipt.is_none(), "expected no receipt for missing hash");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Multiple transactions in the same block are correctly persisted to `RocksDB`.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
// Create 3 txs from the same wallet with sequential nonces
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
let mut tx_hashes = Vec::new();
|
||||
for nonce in 0..3 {
|
||||
let raw_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
|
||||
.await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
tx_hashes.push(tx_hash);
|
||||
}
|
||||
|
||||
// Wait for all txs to appear in pending pool before mining
|
||||
for tx_hash in &tx_hashes {
|
||||
wait_for_pending_tx(&client, *tx_hash).await;
|
||||
}
|
||||
|
||||
// Mine one block containing all 3 txs
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
// Verify block contains all 3 txs
|
||||
let block: Option<alloy_rpc_types_eth::Block> =
|
||||
client.request("eth_getBlockByNumber", ("0x1", true)).await?;
|
||||
let block = block.expect("Block 1 should exist");
|
||||
assert_eq!(block.transactions.len(), 3, "Block should contain 3 txs");
|
||||
|
||||
// Verify each tx via RPC
|
||||
for tx_hash in &tx_hashes {
|
||||
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
|
||||
let tx = tx.expect("Transaction should be found");
|
||||
assert_eq!(tx.block_number, Some(1), "All txs should be in block 1");
|
||||
}
|
||||
|
||||
// Poll RocksDB for all tx hashes and collect tx_numbers
|
||||
let mut tx_numbers = Vec::new();
|
||||
for tx_hash in &tx_hashes {
|
||||
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
|
||||
tx_numbers.push(n);
|
||||
}
|
||||
|
||||
// Verify tx_numbers form the set {0, 1, 2}
|
||||
tx_numbers.sort();
|
||||
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be 0, 1, 2");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Transactions across multiple blocks have globally continuous `tx_numbers`.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_txs_across_blocks() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
// Block 1: 2 transactions
|
||||
let tx_hash_0 = nodes[0]
|
||||
.rpc
|
||||
.inject_tx(
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await,
|
||||
)
|
||||
.await?;
|
||||
let tx_hash_1 = nodes[0]
|
||||
.rpc
|
||||
.inject_tx(
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
// Wait for both txs to appear in pending pool
|
||||
wait_for_pending_tx(&client, tx_hash_0).await;
|
||||
wait_for_pending_tx(&client, tx_hash_1).await;
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
|
||||
// Block 2: 1 transaction
|
||||
let tx_hash_2 = nodes[0]
|
||||
.rpc
|
||||
.inject_tx(
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await,
|
||||
)
|
||||
.await?;
|
||||
|
||||
wait_for_pending_tx(&client, tx_hash_2).await;
|
||||
|
||||
let payload2 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload2.block().number(), 2);
|
||||
|
||||
// Verify block contents via RPC
|
||||
let tx0: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
|
||||
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_1]).await?;
|
||||
let tx2: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_2]).await?;
|
||||
|
||||
assert_eq!(tx0.expect("tx0").block_number, Some(1));
|
||||
assert_eq!(tx1.expect("tx1").block_number, Some(1));
|
||||
assert_eq!(tx2.expect("tx2").block_number, Some(2));
|
||||
|
||||
// Poll RocksDB and verify global tx_number continuity
|
||||
let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2];
|
||||
let mut tx_numbers = Vec::new();
|
||||
for tx_hash in &all_tx_hashes {
|
||||
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
|
||||
tx_numbers.push(n);
|
||||
}
|
||||
|
||||
// Verify they form a continuous sequence {0, 1, 2}
|
||||
tx_numbers.sort();
|
||||
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be globally continuous: 0, 1, 2");
|
||||
|
||||
// Re-query block 1 txs after block 2 is mined (regression guard)
|
||||
let tx0_again: Option<Transaction> =
|
||||
client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
|
||||
assert!(tx0_again.is_some(), "Block 1 tx should still be queryable after block 2");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pending transactions should NOT appear in `RocksDB` until mined.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
|
||||
// Inject tx but do NOT mine
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Verify tx is in pending pool via RPC
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
|
||||
let pending_tx: Option<Transaction> =
|
||||
client.request("eth_getTransactionByHash", [tx_hash]).await?;
|
||||
assert!(pending_tx.is_some(), "Pending tx should be visible via RPC");
|
||||
assert!(pending_tx.unwrap().block_number.is_none(), "Pending tx should have no block_number");
|
||||
|
||||
// Assert tx is NOT in RocksDB before mining (single check - tx is confirmed pending)
|
||||
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
|
||||
let tx_number: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(tx_hash)?;
|
||||
assert!(
|
||||
tx_number.is_none(),
|
||||
"Pending tx should NOT be in RocksDB before mining, but found tx_number={:?}",
|
||||
tx_number
|
||||
);
|
||||
|
||||
// Now mine the block
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
// Poll until tx appears in RocksDB
|
||||
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
|
||||
assert_eq!(tx_number, 0, "First tx should have tx_number 0");
|
||||
|
||||
// Verify tx is now mined via RPC
|
||||
let mined_tx: Option<Transaction> =
|
||||
client.request("eth_getTransactionByHash", [tx_hash]).await?;
|
||||
assert_eq!(mined_tx.expect("mined tx").block_number, Some(1));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reorg with `RocksDB`: verifies that unwind correctly reads changesets from
|
||||
/// storage-aware locations (static files vs MDBX) rather than directly from MDBX.
|
||||
///
|
||||
/// This test exercises `unwind_trie_state_from` which previously failed with
|
||||
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
|
||||
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
// Use two separate wallets to avoid nonce conflicts during reorg
|
||||
let wallets = wallet::Wallet::new(2).with_chain_id(chain_id).wallet_gen();
|
||||
let signer1 = wallets[0].clone();
|
||||
let signer2 = wallets[1].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
// Mine block 1 with a transaction from signer1
|
||||
let raw_tx1 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await;
|
||||
let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?;
|
||||
wait_for_pending_tx(&client, tx_hash1).await;
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
let block1_hash = payload1.block().hash();
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
|
||||
// Poll until tx1 appears in RocksDB (ensures persistence happened)
|
||||
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
|
||||
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
|
||||
|
||||
// Mine block 2 with transaction from signer1 (nonce 1)
|
||||
let raw_tx2 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await;
|
||||
let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?;
|
||||
wait_for_pending_tx(&client, tx_hash2).await;
|
||||
|
||||
let payload2 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload2.block().number(), 2);
|
||||
|
||||
// Poll until tx2 appears in RocksDB
|
||||
let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
|
||||
assert_eq!(tx_number2, 1, "Second tx should have tx_number 1");
|
||||
|
||||
// Mine block 3 with transaction from signer1 (nonce 2)
|
||||
let raw_tx3 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await;
|
||||
let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?;
|
||||
wait_for_pending_tx(&client, tx_hash3).await;
|
||||
|
||||
let payload3 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload3.block().number(), 3);
|
||||
|
||||
// Poll until tx3 appears in RocksDB
|
||||
let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
|
||||
assert_eq!(tx_number3, 2, "Third tx should have tx_number 2");
|
||||
|
||||
// Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict)
|
||||
// Inject a tx from signer2 (nonce 0) before building the alternate block
|
||||
let raw_alt_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await;
|
||||
let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?;
|
||||
wait_for_pending_tx(&client, alt_tx_hash).await;
|
||||
|
||||
// Build an alternate payload (this builds on top of the current head, i.e., block 3)
|
||||
// But we want to reorg back to block 1, so we'll use the payload and then FCU to it
|
||||
let alt_payload = nodes[0].new_payload().await?;
|
||||
let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?;
|
||||
|
||||
// Trigger reorg: make the alternate chain canonical by sending FCU pointing to block 1's hash
|
||||
// as finalized, which should trigger an unwind of blocks 2 and 3
|
||||
// The alt block becomes the new head
|
||||
nodes[0].update_forkchoice(block1_hash, alt_block_hash).await?;
|
||||
|
||||
// Give time for the reorg to complete
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
// Verify we can still query transactions and the chain is consistent
|
||||
// If unwind_trie_state_from failed, this would have errored during reorg
|
||||
let latest: Option<alloy_rpc_types_eth::Block> =
|
||||
client.request("eth_getBlockByNumber", ("latest", false)).await?;
|
||||
let latest = latest.expect("Latest block should exist");
|
||||
// The alt block is at height 4 (on top of block 3)
|
||||
assert!(latest.header.number >= 3, "Should be at height >= 3 after operation");
|
||||
|
||||
// tx1 from block 1 should still be there
|
||||
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash1]).await?;
|
||||
assert!(tx1.is_some(), "tx1 from block 1 should still be queryable");
|
||||
assert_eq!(tx1.unwrap().block_number, Some(1));
|
||||
|
||||
// Mine another block to verify the chain can continue
|
||||
let raw_tx_final =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await;
|
||||
let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?;
|
||||
wait_for_pending_tx(&client, tx_hash_final).await;
|
||||
|
||||
let final_payload = nodes[0].advance_block().await?;
|
||||
assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg");
|
||||
|
||||
// Verify tx_final is included
|
||||
let tx_final: Option<Transaction> =
|
||||
client.request("eth_getTransactionByHash", [tx_hash_final]).await?;
|
||||
assert!(tx_final.is_some(), "final tx should be in latest block");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -32,9 +32,7 @@ futures-util.workspace = true
|
||||
# misc
|
||||
eyre.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
op-alloy-rpc-types-engine = { workspace = true, optional = true }
|
||||
reth-optimism-chainspec = { workspace = true, optional = true }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -42,7 +40,6 @@ workspace = true
|
||||
[features]
|
||||
op = [
|
||||
"dep:op-alloy-rpc-types-engine",
|
||||
"dep:reth-optimism-chainspec",
|
||||
"reth-payload-primitives/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
|
||||
@@ -72,13 +72,18 @@ where
|
||||
&self,
|
||||
parent: &SealedHeader<ChainSpec::Header>,
|
||||
) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
|
||||
/// Dummy system transaction for dev mode.
|
||||
/// OP Mainnet transaction at index 0 in block 124665056.
|
||||
///
|
||||
/// <https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1>
|
||||
const TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056: [u8; 251] = alloy_primitives::hex!(
|
||||
"7ef8f8a0683079df94aa5b9cf86687d739a60a9b4f0835e520ec4d664e2e415dca17a6df94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985"
|
||||
);
|
||||
|
||||
op_alloy_rpc_types_engine::OpPayloadAttributes {
|
||||
payload_attributes: self.build(parent),
|
||||
// Add dummy system transaction
|
||||
transactions: Some(vec![
|
||||
reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056
|
||||
.into(),
|
||||
]),
|
||||
transactions: Some(vec![TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056.into()]),
|
||||
no_tx_pool: None,
|
||||
gas_limit: None,
|
||||
eip_1559_params: None,
|
||||
|
||||
@@ -47,10 +47,31 @@ pub const DEFAULT_RESERVED_CPU_CORES: usize = 1;
|
||||
/// Default maximum concurrency for prewarm task.
|
||||
pub const DEFAULT_PREWARM_MAX_CONCURRENCY: usize = 16;
|
||||
|
||||
/// Default depth for sparse trie pruning.
|
||||
///
|
||||
/// Nodes at this depth and below are converted to hash stubs to reduce memory.
|
||||
/// Depth 4 means we keep roughly 16^4 = 65536 potential branch paths at most.
|
||||
pub const DEFAULT_SPARSE_TRIE_PRUNE_DEPTH: usize = 4;
|
||||
|
||||
/// Default maximum number of storage tries to keep after pruning.
|
||||
///
|
||||
/// Storage tries beyond this limit are cleared (but allocations preserved).
|
||||
pub const DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES: usize = 100;
|
||||
|
||||
const DEFAULT_BLOCK_BUFFER_LIMIT: u32 = EPOCH_SLOTS as u32 * 2;
|
||||
const DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH: u32 = 256;
|
||||
const DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE: usize = 4;
|
||||
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: u64 = 4 * 1024 * 1024 * 1024;
|
||||
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: usize = default_cross_block_cache_size();
|
||||
|
||||
const fn default_cross_block_cache_size() -> usize {
|
||||
if cfg!(test) {
|
||||
1024 * 1024 // 1 MB in tests
|
||||
} else if cfg!(target_pointer_width = "32") {
|
||||
usize::MAX // max possible on wasm32 / 32-bit
|
||||
} else {
|
||||
4 * 1024 * 1024 * 1024 // 4 GB on 64-bit
|
||||
}
|
||||
}
|
||||
|
||||
/// Determines if the host has enough parallelism to run the payload processor.
|
||||
///
|
||||
@@ -100,12 +121,10 @@ pub struct TreeConfig {
|
||||
disable_state_cache: bool,
|
||||
/// Whether to disable parallel prewarming.
|
||||
disable_prewarming: bool,
|
||||
/// Whether to disable the parallel sparse trie state root algorithm.
|
||||
disable_parallel_sparse_trie: bool,
|
||||
/// Whether to enable state provider metrics.
|
||||
state_provider_metrics: bool,
|
||||
/// Cross-block cache size in bytes.
|
||||
cross_block_cache_size: u64,
|
||||
cross_block_cache_size: usize,
|
||||
/// Whether the host has enough parallelism to run state root task.
|
||||
has_enough_parallelism: bool,
|
||||
/// Whether multiproof task should chunk proof targets.
|
||||
@@ -140,10 +159,16 @@ pub struct TreeConfig {
|
||||
storage_worker_count: usize,
|
||||
/// Number of account proof worker threads.
|
||||
account_worker_count: usize,
|
||||
/// Whether to enable V2 storage proofs.
|
||||
enable_proof_v2: bool,
|
||||
/// Whether to disable V2 storage proofs.
|
||||
disable_proof_v2: bool,
|
||||
/// Whether to disable cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
/// Whether to enable sparse trie as cache.
|
||||
enable_sparse_trie_as_cache: bool,
|
||||
/// Depth for sparse trie pruning after state root computation.
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum number of storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
}
|
||||
|
||||
impl Default for TreeConfig {
|
||||
@@ -158,7 +183,6 @@ impl Default for TreeConfig {
|
||||
always_compare_trie_updates: false,
|
||||
disable_state_cache: false,
|
||||
disable_prewarming: false,
|
||||
disable_parallel_sparse_trie: false,
|
||||
state_provider_metrics: false,
|
||||
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE,
|
||||
has_enough_parallelism: has_enough_parallelism(),
|
||||
@@ -172,8 +196,11 @@ impl Default for TreeConfig {
|
||||
allow_unwind_canonical_header: false,
|
||||
storage_worker_count: default_storage_worker_count(),
|
||||
account_worker_count: default_account_worker_count(),
|
||||
enable_proof_v2: false,
|
||||
disable_proof_v2: false,
|
||||
disable_cache_metrics: false,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -191,9 +218,8 @@ impl TreeConfig {
|
||||
always_compare_trie_updates: bool,
|
||||
disable_state_cache: bool,
|
||||
disable_prewarming: bool,
|
||||
disable_parallel_sparse_trie: bool,
|
||||
state_provider_metrics: bool,
|
||||
cross_block_cache_size: u64,
|
||||
cross_block_cache_size: usize,
|
||||
has_enough_parallelism: bool,
|
||||
multiproof_chunking_enabled: bool,
|
||||
multiproof_chunk_size: usize,
|
||||
@@ -205,8 +231,10 @@ impl TreeConfig {
|
||||
allow_unwind_canonical_header: bool,
|
||||
storage_worker_count: usize,
|
||||
account_worker_count: usize,
|
||||
enable_proof_v2: bool,
|
||||
disable_proof_v2: bool,
|
||||
disable_cache_metrics: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -218,7 +246,6 @@ impl TreeConfig {
|
||||
always_compare_trie_updates,
|
||||
disable_state_cache,
|
||||
disable_prewarming,
|
||||
disable_parallel_sparse_trie,
|
||||
state_provider_metrics,
|
||||
cross_block_cache_size,
|
||||
has_enough_parallelism,
|
||||
@@ -232,8 +259,11 @@ impl TreeConfig {
|
||||
allow_unwind_canonical_header,
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
enable_proof_v2,
|
||||
disable_proof_v2,
|
||||
disable_cache_metrics,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -275,7 +305,8 @@ impl TreeConfig {
|
||||
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
|
||||
/// and the chunk size is at the default value.
|
||||
pub const fn effective_multiproof_chunk_size(&self) -> usize {
|
||||
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
|
||||
if !self.disable_proof_v2 &&
|
||||
self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
|
||||
{
|
||||
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
|
||||
} else {
|
||||
@@ -299,11 +330,6 @@ impl TreeConfig {
|
||||
self.state_provider_metrics
|
||||
}
|
||||
|
||||
/// Returns whether or not the parallel sparse trie is disabled.
|
||||
pub const fn disable_parallel_sparse_trie(&self) -> bool {
|
||||
self.disable_parallel_sparse_trie
|
||||
}
|
||||
|
||||
/// Returns whether or not state cache is disabled.
|
||||
pub const fn disable_state_cache(&self) -> bool {
|
||||
self.disable_state_cache
|
||||
@@ -321,7 +347,7 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Returns the cross-block cache size.
|
||||
pub const fn cross_block_cache_size(&self) -> u64 {
|
||||
pub const fn cross_block_cache_size(&self) -> usize {
|
||||
self.cross_block_cache_size
|
||||
}
|
||||
|
||||
@@ -424,7 +450,7 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for cross block cache size.
|
||||
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: u64) -> Self {
|
||||
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: usize) -> Self {
|
||||
self.cross_block_cache_size = cross_block_cache_size;
|
||||
self
|
||||
}
|
||||
@@ -441,15 +467,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for whether to disable the parallel sparse trie
|
||||
pub const fn with_disable_parallel_sparse_trie(
|
||||
mut self,
|
||||
disable_parallel_sparse_trie: bool,
|
||||
) -> Self {
|
||||
self.disable_parallel_sparse_trie = disable_parallel_sparse_trie;
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for whether multiproof task should chunk proof targets.
|
||||
pub const fn with_multiproof_chunking_enabled(
|
||||
mut self,
|
||||
@@ -511,8 +528,12 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of storage proof worker threads.
|
||||
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
|
||||
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = storage_worker_count {
|
||||
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -522,19 +543,23 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of account proof worker threads.
|
||||
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
|
||||
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = account_worker_count {
|
||||
self.account_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Return whether V2 storage proofs are enabled.
|
||||
pub const fn enable_proof_v2(&self) -> bool {
|
||||
self.enable_proof_v2
|
||||
/// Return whether V2 storage proofs are disabled.
|
||||
pub const fn disable_proof_v2(&self) -> bool {
|
||||
self.disable_proof_v2
|
||||
}
|
||||
|
||||
/// Setter for whether to enable V2 storage proofs.
|
||||
pub const fn with_enable_proof_v2(mut self, enable_proof_v2: bool) -> Self {
|
||||
self.enable_proof_v2 = enable_proof_v2;
|
||||
/// Setter for whether to disable V2 storage proofs.
|
||||
pub const fn with_disable_proof_v2(mut self, disable_proof_v2: bool) -> Self {
|
||||
self.disable_proof_v2 = disable_proof_v2;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -548,4 +573,37 @@ impl TreeConfig {
|
||||
self.disable_cache_metrics = disable_cache_metrics;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether sparse trie as cache is enabled.
|
||||
pub const fn enable_sparse_trie_as_cache(&self) -> bool {
|
||||
self.enable_sparse_trie_as_cache
|
||||
}
|
||||
|
||||
/// Setter for whether to enable sparse trie as cache.
|
||||
pub const fn with_enable_sparse_trie_as_cache(mut self, value: bool) -> Self {
|
||||
self.enable_sparse_trie_as_cache = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the sparse trie prune depth.
|
||||
pub const fn sparse_trie_prune_depth(&self) -> usize {
|
||||
self.sparse_trie_prune_depth
|
||||
}
|
||||
|
||||
/// Setter for sparse trie prune depth.
|
||||
pub const fn with_sparse_trie_prune_depth(mut self, depth: usize) -> Self {
|
||||
self.sparse_trie_prune_depth = depth;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the maximum number of storage tries to retain after pruning.
|
||||
pub const fn sparse_trie_max_storage_tries(&self) -> usize {
|
||||
self.sparse_trie_max_storage_tries
|
||||
}
|
||||
|
||||
/// Setter for maximum storage tries to retain.
|
||||
pub const fn with_sparse_trie_max_storage_tries(mut self, max_tries: usize) -> Self {
|
||||
self.sparse_trie_max_storage_tries = max_tries;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
|
||||
@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
|
||||
chain::{ChainEvent, ChainOrchestrator},
|
||||
engine::EngineApiEvent,
|
||||
};
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_network_p2p::BlockClient;
|
||||
use reth_node_types::{BlockTy, NodeTypes};
|
||||
@@ -97,11 +96,11 @@ where
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
|
||||
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
|
||||
|
||||
let (to_tree_tx, from_tree) = EngineApiTreeHandler::<N::Primitives, _, _, _, _>::spawn_new(
|
||||
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
|
||||
blockchain_db,
|
||||
consensus,
|
||||
payload_validator,
|
||||
|
||||
@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-payload-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["rayon"] }
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
@@ -53,7 +53,7 @@ revm-primitives.workspace = true
|
||||
futures.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
|
||||
mini-moka = { workspace = true, features = ["sync"] }
|
||||
fixed-cache.workspace = true
|
||||
moka = { workspace = true, features = ["sync"] }
|
||||
smallvec.workspace = true
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
|
||||
/// How long it took for blocks to be saved
|
||||
pub(crate) save_blocks_duration_seconds: Histogram,
|
||||
/// How many blocks we persist at once.
|
||||
pub(crate) save_blocks_block_count: Histogram,
|
||||
pub(crate) save_blocks_batch_size: Histogram,
|
||||
/// How long it took for blocks to be pruned
|
||||
pub(crate) prune_before_duration_seconds: Histogram,
|
||||
}
|
||||
|
||||
@@ -12,7 +12,11 @@ use reth_provider::{
|
||||
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender};
|
||||
use std::{
|
||||
sync::mpsc::{Receiver, SendError, Sender},
|
||||
sync::{
|
||||
mpsc::{Receiver, SendError, Sender},
|
||||
Arc,
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::Instant,
|
||||
};
|
||||
use thiserror::Error;
|
||||
@@ -40,6 +44,12 @@ where
|
||||
metrics: PersistenceMetrics,
|
||||
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
/// Pending finalized block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each finalized block update.
|
||||
pending_finalized_block: Option<u64>,
|
||||
/// Pending safe block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each safe block update.
|
||||
pending_safe_block: Option<u64>,
|
||||
}
|
||||
|
||||
impl<N> PersistenceService<N>
|
||||
@@ -53,7 +63,15 @@ where
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
) -> Self {
|
||||
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
|
||||
Self {
|
||||
provider,
|
||||
incoming,
|
||||
pruner,
|
||||
metrics: PersistenceMetrics::default(),
|
||||
sync_metrics_tx,
|
||||
pending_finalized_block: None,
|
||||
pending_safe_block: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes block data before the given block number according to the configured prune
|
||||
@@ -106,14 +124,10 @@ where
|
||||
}
|
||||
}
|
||||
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_finalized_block_number(finalized_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_finalized_block = Some(finalized_block);
|
||||
}
|
||||
PersistenceAction::SaveSafeBlock(safe_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_safe_block_number(safe_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_safe_block = Some(safe_block);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,26 +152,39 @@ where
|
||||
}
|
||||
|
||||
fn on_save_blocks(
|
||||
&self,
|
||||
&mut self,
|
||||
blocks: Vec<ExecutedBlock<N::Primitives>>,
|
||||
) -> Result<Option<BlockNumHash>, PersistenceError> {
|
||||
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks.len();
|
||||
|
||||
// Take any pending finalized/safe block updates to commit together
|
||||
let pending_finalized = self.pending_finalized_block.take();
|
||||
let pending_safe = self.pending_safe_block.take();
|
||||
|
||||
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
|
||||
|
||||
let start_time = Instant::now();
|
||||
|
||||
if last_block.is_some() {
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
|
||||
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
|
||||
|
||||
// Commit pending finalized/safe block updates in the same transaction
|
||||
if let Some(finalized) = pending_finalized {
|
||||
provider_rw.save_finalized_block_number(finalized)?;
|
||||
}
|
||||
if let Some(safe) = pending_safe {
|
||||
provider_rw.save_safe_block_number(safe)?;
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
self.metrics.save_blocks_block_count.record(block_count as f64);
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
|
||||
|
||||
Ok(last_block)
|
||||
@@ -204,15 +231,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
|
||||
/// The channel used to communicate with the persistence service
|
||||
sender: Sender<PersistenceAction<N>>,
|
||||
/// Guard that joins the service thread when all handles are dropped.
|
||||
/// Uses `Arc` so the handle remains `Clone`.
|
||||
_service_guard: Arc<ServiceGuard>,
|
||||
}
|
||||
|
||||
impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
|
||||
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender }
|
||||
///
|
||||
/// This is intended for testing purposes where you want to mock the persistence service.
|
||||
/// For production use, prefer [`spawn_service`](Self::spawn_service).
|
||||
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
|
||||
}
|
||||
|
||||
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
|
||||
///
|
||||
/// The returned handle can be cloned and shared. When all clones are dropped, the service
|
||||
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
|
||||
/// released.
|
||||
pub fn spawn_service<N>(
|
||||
provider_factory: ProviderFactory<N>,
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
@@ -224,13 +261,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
// create the initial channels
|
||||
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
|
||||
|
||||
// construct persistence handle
|
||||
let persistence_handle = PersistenceHandle::new(db_service_tx);
|
||||
|
||||
// spawn the persistence service
|
||||
let db_service =
|
||||
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
|
||||
std::thread::Builder::new()
|
||||
let join_handle = std::thread::Builder::new()
|
||||
.name("Persistence Service".to_string())
|
||||
.spawn(|| {
|
||||
if let Err(err) = db_service.run() {
|
||||
@@ -239,7 +273,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
persistence_handle
|
||||
PersistenceHandle {
|
||||
sender: db_service_tx,
|
||||
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
|
||||
@@ -267,7 +304,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
|
||||
}
|
||||
|
||||
/// Persists the finalized block number on disk.
|
||||
/// Queues the finalized block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_finalized_block_number(
|
||||
&self,
|
||||
finalized_block: u64,
|
||||
@@ -275,7 +315,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
|
||||
}
|
||||
|
||||
/// Persists the safe block number on disk.
|
||||
/// Queues the safe block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_safe_block_number(
|
||||
&self,
|
||||
safe_block: u64,
|
||||
@@ -297,6 +340,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that joins the persistence service thread when dropped.
|
||||
///
|
||||
/// This ensures graceful shutdown - the service thread completes before resources like
|
||||
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
|
||||
/// can be cloned while sharing the same guard.
|
||||
struct ServiceGuard(Option<JoinHandle<()>>);
|
||||
|
||||
impl std::fmt::Debug for ServiceGuard {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ServiceGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(join_handle) = self.0.take() {
|
||||
let _ = join_handle.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -323,12 +387,12 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_empty() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let blocks = vec![];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let hash = rx.recv().unwrap();
|
||||
assert_eq!(hash, None);
|
||||
@@ -337,7 +401,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_single_block() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
let block_number = 0;
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let executed =
|
||||
@@ -347,7 +411,7 @@ mod tests {
|
||||
let blocks = vec![executed];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
@@ -360,14 +424,14 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_blocks() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
}
|
||||
@@ -375,7 +439,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_calls() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let ranges = [0..1, 1..2, 2..4, 4..5];
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
@@ -384,7 +448,7 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -13,15 +13,18 @@ use std::time::{Duration, Instant};
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct EngineApiMetrics {
|
||||
pub struct EngineApiMetrics {
|
||||
/// Engine API-specific metrics.
|
||||
pub(crate) engine: EngineMetrics,
|
||||
pub engine: EngineMetrics,
|
||||
/// Block executor metrics.
|
||||
pub(crate) executor: ExecutorMetrics,
|
||||
pub executor: ExecutorMetrics,
|
||||
/// Metrics for block validation
|
||||
pub(crate) block_validation: BlockValidationMetrics,
|
||||
pub block_validation: BlockValidationMetrics,
|
||||
/// Canonical chain and reorg related metrics
|
||||
pub tree: TreeMetrics,
|
||||
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
|
||||
#[allow(dead_code)]
|
||||
pub(crate) bal: BalMetrics,
|
||||
}
|
||||
|
||||
impl EngineApiMetrics {
|
||||
@@ -29,7 +32,7 @@ impl EngineApiMetrics {
|
||||
///
|
||||
/// This method updates metrics for execution time, gas usage, and the number
|
||||
/// of accounts, storage slots and bytecodes updated.
|
||||
pub(crate) fn record_block_execution<R>(
|
||||
pub fn record_block_execution<R>(
|
||||
&self,
|
||||
output: &BlockExecutionOutput<R>,
|
||||
execution_duration: Duration,
|
||||
@@ -56,27 +59,27 @@ impl EngineApiMetrics {
|
||||
}
|
||||
|
||||
/// Returns a reference to the executor metrics for use in state hooks.
|
||||
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
|
||||
pub const fn executor_metrics(&self) -> &ExecutorMetrics {
|
||||
&self.executor
|
||||
}
|
||||
|
||||
/// Records the duration of block pre-execution changes (e.g., beacon root update).
|
||||
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
|
||||
pub fn record_pre_execution(&self, elapsed: Duration) {
|
||||
self.executor.pre_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of block post-execution changes (e.g., finalization).
|
||||
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
|
||||
pub fn record_post_execution(&self, elapsed: Duration) {
|
||||
self.executor.post_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the time spent waiting for the next transaction from the iterator.
|
||||
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
pub fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
self.executor.transaction_wait_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records the duration of a single transaction execution.
|
||||
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
|
||||
pub fn record_transaction_execution(&self, elapsed: Duration) {
|
||||
self.executor.transaction_execution_histogram.record(elapsed);
|
||||
}
|
||||
}
|
||||
@@ -84,7 +87,7 @@ impl EngineApiMetrics {
|
||||
/// Metrics for the entire blockchain tree
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "blockchain_tree")]
|
||||
pub(crate) struct TreeMetrics {
|
||||
pub struct TreeMetrics {
|
||||
/// The highest block number in the canonical chain
|
||||
pub canonical_chain_height: Gauge,
|
||||
/// The number of reorgs
|
||||
@@ -100,7 +103,7 @@ pub(crate) struct TreeMetrics {
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
pub(crate) struct EngineMetrics {
|
||||
pub struct EngineMetrics {
|
||||
/// Engine API forkchoiceUpdated response type metrics
|
||||
#[metric(skip)]
|
||||
pub(crate) forkchoice_updated: ForkchoiceUpdatedMetrics,
|
||||
@@ -239,6 +242,8 @@ pub(crate) struct NewPayloadStatusMetrics {
|
||||
pub(crate) new_payload_error: Counter,
|
||||
/// The total gas of valid new payload messages received.
|
||||
pub(crate) new_payload_total_gas: Histogram,
|
||||
/// The gas used for the last valid new payload.
|
||||
pub(crate) new_payload_total_gas_last: Gauge,
|
||||
/// The gas per second of valid new payload messages received.
|
||||
pub(crate) new_payload_gas_per_second: Histogram,
|
||||
/// The gas per second for the last new payload call.
|
||||
@@ -251,6 +256,8 @@ pub(crate) struct NewPayloadStatusMetrics {
|
||||
pub(crate) time_between_new_payloads: Histogram,
|
||||
/// Time from previous payload start to current payload start (total interval).
|
||||
pub(crate) new_payload_interval: Histogram,
|
||||
/// Time diff between forkchoice updated call response and the next new payload call request.
|
||||
pub(crate) forkchoice_updated_new_payload_time_diff: Histogram,
|
||||
}
|
||||
|
||||
impl NewPayloadStatusMetrics {
|
||||
@@ -258,6 +265,7 @@ impl NewPayloadStatusMetrics {
|
||||
pub(crate) fn update_response_metrics(
|
||||
&mut self,
|
||||
start: Instant,
|
||||
latest_forkchoice_updated_at: &mut Option<Instant>,
|
||||
result: &Result<TreeOutcome<PayloadStatus>, InsertBlockFatalError>,
|
||||
gas_used: u64,
|
||||
) {
|
||||
@@ -277,6 +285,7 @@ impl NewPayloadStatusMetrics {
|
||||
PayloadStatusEnum::Valid => {
|
||||
self.new_payload_valid.increment(1);
|
||||
self.new_payload_total_gas.record(gas_used as f64);
|
||||
self.new_payload_total_gas_last.set(gas_used as f64);
|
||||
let gas_per_second = gas_used as f64 / elapsed.as_secs_f64();
|
||||
self.new_payload_gas_per_second.record(gas_per_second);
|
||||
self.new_payload_gas_per_second_last.set(gas_per_second);
|
||||
@@ -290,46 +299,79 @@ impl NewPayloadStatusMetrics {
|
||||
self.new_payload_messages.increment(1);
|
||||
self.new_payload_latency.record(elapsed);
|
||||
self.new_payload_last.set(elapsed);
|
||||
if let Some(latest_forkchoice_updated_at) = latest_forkchoice_updated_at.take() {
|
||||
self.forkchoice_updated_new_payload_time_diff
|
||||
.record(start - latest_forkchoice_updated_at);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
|
||||
///
|
||||
/// See also <https://github.com/ethereum/execution-metrics/issues/5>
|
||||
#[allow(dead_code)]
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "execution.block_access_list")]
|
||||
pub(crate) struct BalMetrics {
|
||||
/// Size of the BAL in bytes for the current block.
|
||||
pub(crate) size_bytes: Gauge,
|
||||
/// Total number of blocks with valid BALs.
|
||||
pub(crate) valid_total: Counter,
|
||||
/// Total number of blocks with invalid BALs.
|
||||
pub(crate) invalid_total: Counter,
|
||||
/// Time taken to validate the BAL against actual execution.
|
||||
pub(crate) validation_time_seconds: Histogram,
|
||||
/// Number of account changes in the BAL.
|
||||
pub(crate) account_changes: Gauge,
|
||||
/// Number of storage changes in the BAL.
|
||||
pub(crate) storage_changes: Gauge,
|
||||
/// Number of balance changes in the BAL.
|
||||
pub(crate) balance_changes: Gauge,
|
||||
/// Number of nonce changes in the BAL.
|
||||
pub(crate) nonce_changes: Gauge,
|
||||
/// Number of code changes in the BAL.
|
||||
pub(crate) code_changes: Gauge,
|
||||
}
|
||||
|
||||
/// Metrics for non-execution related block validation.
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "sync.block_validation")]
|
||||
pub(crate) struct BlockValidationMetrics {
|
||||
pub struct BlockValidationMetrics {
|
||||
/// Total number of storage tries updated in the state root calculation
|
||||
pub(crate) state_root_storage_tries_updated_total: Counter,
|
||||
pub state_root_storage_tries_updated_total: Counter,
|
||||
/// Total number of times the parallel state root computation fell back to regular.
|
||||
pub(crate) state_root_parallel_fallback_total: Counter,
|
||||
pub state_root_parallel_fallback_total: Counter,
|
||||
/// Total number of times the state root task failed but the fallback succeeded.
|
||||
pub state_root_task_fallback_success_total: Counter,
|
||||
/// Latest state root duration, ie the time spent blocked waiting for the state root.
|
||||
pub(crate) state_root_duration: Gauge,
|
||||
pub state_root_duration: Gauge,
|
||||
/// Histogram for state root duration ie the time spent blocked waiting for the state root
|
||||
pub(crate) state_root_histogram: Histogram,
|
||||
pub state_root_histogram: Histogram,
|
||||
/// Histogram of deferred trie computation duration.
|
||||
pub(crate) deferred_trie_compute_duration: Histogram,
|
||||
pub deferred_trie_compute_duration: Histogram,
|
||||
/// Payload conversion and validation latency
|
||||
pub(crate) payload_validation_duration: Gauge,
|
||||
pub payload_validation_duration: Gauge,
|
||||
/// Histogram of payload validation latency
|
||||
pub(crate) payload_validation_histogram: Histogram,
|
||||
pub payload_validation_histogram: Histogram,
|
||||
/// Payload processor spawning duration
|
||||
pub(crate) spawn_payload_processor: Histogram,
|
||||
pub spawn_payload_processor: Histogram,
|
||||
/// Post-execution validation duration
|
||||
pub(crate) post_execution_validation_duration: Histogram,
|
||||
pub post_execution_validation_duration: Histogram,
|
||||
/// Total duration of the new payload call
|
||||
pub(crate) total_duration: Histogram,
|
||||
pub total_duration: Histogram,
|
||||
/// Size of `HashedPostStateSorted` (`total_len`)
|
||||
pub(crate) hashed_post_state_size: Histogram,
|
||||
pub hashed_post_state_size: Histogram,
|
||||
/// Size of `TrieUpdatesSorted` (`total_len`)
|
||||
pub(crate) trie_updates_sorted_size: Histogram,
|
||||
pub trie_updates_sorted_size: Histogram,
|
||||
/// Size of `AnchoredTrieInput` overlay `TrieUpdatesSorted` (`total_len`)
|
||||
pub(crate) anchored_overlay_trie_updates_size: Histogram,
|
||||
pub anchored_overlay_trie_updates_size: Histogram,
|
||||
/// Size of `AnchoredTrieInput` overlay `HashedPostStateSorted` (`total_len`)
|
||||
pub(crate) anchored_overlay_hashed_state_size: Histogram,
|
||||
pub anchored_overlay_hashed_state_size: Histogram,
|
||||
}
|
||||
|
||||
impl BlockValidationMetrics {
|
||||
/// Records a new state root time, updating both the histogram and state root gauge
|
||||
pub(crate) fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) {
|
||||
pub fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) {
|
||||
self.state_root_storage_tries_updated_total
|
||||
.increment(trie_output.storage_tries_ref().len() as u64);
|
||||
self.state_root_duration.set(elapsed_as_secs);
|
||||
@@ -338,7 +380,7 @@ impl BlockValidationMetrics {
|
||||
|
||||
/// Records a new payload validation time, updating both the histogram and the payload
|
||||
/// validation gauge
|
||||
pub(crate) fn record_payload_validation(&self, elapsed_as_secs: f64) {
|
||||
pub fn record_payload_validation(&self, elapsed_as_secs: f64) {
|
||||
self.payload_validation_duration.set(elapsed_as_secs);
|
||||
self.payload_validation_histogram.record(elapsed_as_secs);
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use crate::{
|
||||
chain::FromOrchestrator,
|
||||
engine::{DownloadRequest, EngineApiEvent, EngineApiKind, EngineApiRequest, FromEngine},
|
||||
persistence::PersistenceHandle,
|
||||
tree::{error::InsertPayloadError, metrics::EngineApiMetrics, payload_validator::TreeCtx},
|
||||
tree::{error::InsertPayloadError, payload_validator::TreeCtx},
|
||||
};
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_eips::{eip1898::BlockWithParent, merge::EPOCH_SLOTS, BlockNumHash, NumHash};
|
||||
@@ -30,7 +30,7 @@ use reth_payload_primitives::{
|
||||
};
|
||||
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
TransactionVariant,
|
||||
@@ -55,18 +55,19 @@ pub mod error;
|
||||
pub mod instrumented_state;
|
||||
mod invalid_headers;
|
||||
mod metrics;
|
||||
mod payload_processor;
|
||||
pub mod payload_processor;
|
||||
pub mod payload_validator;
|
||||
mod persistence_state;
|
||||
pub mod precompile_cache;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[expect(unused)]
|
||||
mod trie_updates;
|
||||
|
||||
use crate::tree::error::AdvancePersistenceError;
|
||||
pub use block_buffer::BlockBuffer;
|
||||
pub use cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache};
|
||||
pub use invalid_headers::InvalidHeaderCache;
|
||||
pub use metrics::EngineApiMetrics;
|
||||
pub use payload_processor::*;
|
||||
pub use payload_validator::{BasicEngineValidator, EngineValidator};
|
||||
pub use persistence_state::PersistenceState;
|
||||
@@ -85,6 +86,12 @@ pub mod state;
|
||||
/// backfill this gap.
|
||||
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
|
||||
|
||||
/// The minimum number of blocks to retain in the changeset cache after eviction.
|
||||
///
|
||||
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
|
||||
/// even when the finalized block is not set (e.g., on L2s like Optimism).
|
||||
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
|
||||
|
||||
/// A builder for creating state providers that can be used across threads.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct StateProviderBuilder<N: NodePrimitives, P> {
|
||||
@@ -152,6 +159,16 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
|
||||
forkchoice_state_tracker: ForkchoiceStateTracker::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the tree state.
|
||||
pub const fn tree_state(&self) -> &TreeState<N> {
|
||||
&self.tree_state
|
||||
}
|
||||
|
||||
/// Returns true if the block has been marked as invalid.
|
||||
pub fn has_invalid_header(&mut self, hash: &B256) -> bool {
|
||||
self.invalid_headers.get(hash).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// The outcome of a tree operation.
|
||||
@@ -315,11 +332,10 @@ where
|
||||
+ HashedPostStateProvider
|
||||
+ Clone
|
||||
+ 'static,
|
||||
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
P::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
+ StorageChangeSetReader,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
V: EngineValidator<T>,
|
||||
@@ -1378,21 +1394,42 @@ where
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
|
||||
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
|
||||
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
|
||||
let min_threshold = last_persisted_block_number.saturating_sub(64);
|
||||
let eviction_threshold = finalized.number.min(min_threshold);
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
last_persisted = last_persisted_block_number,
|
||||
finalized_number = finalized.number,
|
||||
eviction_threshold,
|
||||
"Evicting changesets below threshold"
|
||||
);
|
||||
self.changeset_cache.evict(eviction_threshold);
|
||||
}
|
||||
// Evict trie changesets for blocks below the eviction threshold.
|
||||
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
|
||||
// the finalized block if set.
|
||||
let min_threshold =
|
||||
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
|
||||
let eviction_threshold =
|
||||
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
|
||||
// Use the minimum of finalized block and retention threshold to be conservative
|
||||
finalized.number.min(min_threshold)
|
||||
} else {
|
||||
// When finalized is not set (e.g., on L2s), use the retention threshold
|
||||
min_threshold
|
||||
};
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
last_persisted = last_persisted_block_number,
|
||||
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
|
||||
eviction_threshold,
|
||||
"Evicting changesets below threshold"
|
||||
);
|
||||
self.changeset_cache.evict(eviction_threshold);
|
||||
|
||||
// Invalidate cached overlay since the anchor has changed
|
||||
self.state.tree_state.invalidate_cached_overlay();
|
||||
|
||||
self.on_new_persisted_block()?;
|
||||
|
||||
// Re-prepare overlay for the current canonical head with the new anchor.
|
||||
// Spawn a background task to trigger computation so it's ready when the next payload
|
||||
// arrives.
|
||||
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
rayon::spawn(move || {
|
||||
let _ = overlay.get();
|
||||
});
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1478,6 +1515,10 @@ where
|
||||
self.on_maybe_tree_event(res.event.take())?;
|
||||
}
|
||||
|
||||
if let Err(ref err) = output {
|
||||
error!(target: "engine::tree", %err, ?state, "Error processing forkchoice update");
|
||||
}
|
||||
|
||||
self.metrics.engine.forkchoice_updated.update_response_metrics(
|
||||
start,
|
||||
&mut self.metrics.engine.new_payload.latest_finish_at,
|
||||
@@ -1500,10 +1541,12 @@ where
|
||||
let gas_used = payload.gas_used();
|
||||
let num_hash = payload.num_hash();
|
||||
let mut output = self.on_new_payload(payload);
|
||||
self.metrics
|
||||
.engine
|
||||
.new_payload
|
||||
.update_response_metrics(start, &output, gas_used);
|
||||
self.metrics.engine.new_payload.update_response_metrics(
|
||||
start,
|
||||
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
|
||||
&output,
|
||||
gas_used,
|
||||
);
|
||||
|
||||
let maybe_event =
|
||||
output.as_mut().ok().and_then(|out| out.event.take());
|
||||
@@ -2569,19 +2612,27 @@ where
|
||||
let block_num_hash = block_id.block;
|
||||
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
|
||||
|
||||
match self.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
// Check if block already exists - first in memory, then DB only if it could be persisted
|
||||
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
|
||||
// Only query DB if block could be persisted (number <= last persisted block).
|
||||
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
|
||||
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
|
||||
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
Ok(None) => {}
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
// We now assume that we already have this block in the tree. However, we need to
|
||||
// run the conversion to ensure that the block hash is valid.
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
}
|
||||
|
||||
// Ensure that the parent state is available.
|
||||
match self.state_provider_builder(block_id.parent) {
|
||||
|
||||
@@ -21,7 +21,7 @@ pub fn total_slots(bal: &BlockAccessList) -> usize {
|
||||
/// first, followed by read-only slots. The iterator intelligently skips accounts and slots
|
||||
/// outside the specified range for efficient traversal.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BALSlotIter<'a> {
|
||||
pub struct BALSlotIter<'a> {
|
||||
bal: &'a BlockAccessList,
|
||||
range: Range<usize>,
|
||||
current_index: usize,
|
||||
@@ -34,7 +34,7 @@ pub(crate) struct BALSlotIter<'a> {
|
||||
|
||||
impl<'a> BALSlotIter<'a> {
|
||||
/// Creates a new iterator over storage slots within the specified range.
|
||||
pub(crate) fn new(bal: &'a BlockAccessList, range: Range<usize>) -> Self {
|
||||
pub fn new(bal: &'a BlockAccessList, range: Range<usize>) -> Self {
|
||||
let mut iter = Self { bal, range, current_index: 0, account_idx: 0, slot_idx: 0 };
|
||||
iter.skip_to_range_start();
|
||||
iter
|
||||
|
||||
@@ -1,188 +0,0 @@
|
||||
//! Configured sparse trie enum for switching between serial and parallel implementations.
|
||||
|
||||
use alloy_primitives::B256;
|
||||
use reth_trie::{BranchNodeMasks, Nibbles, ProofTrieNode, TrieNode};
|
||||
use reth_trie_sparse::{
|
||||
errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError,
|
||||
SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates,
|
||||
};
|
||||
use reth_trie_sparse_parallel::ParallelSparseTrie;
|
||||
use std::borrow::Cow;
|
||||
|
||||
/// Enum for switching between serial and parallel sparse trie implementations.
|
||||
///
|
||||
/// This type allows runtime selection between different sparse trie implementations,
|
||||
/// providing flexibility in choosing the appropriate implementation based on workload
|
||||
/// characteristics.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) enum ConfiguredSparseTrie {
|
||||
/// Serial implementation of the sparse trie.
|
||||
Serial(Box<SerialSparseTrie>),
|
||||
/// Parallel implementation of the sparse trie.
|
||||
Parallel(Box<ParallelSparseTrie>),
|
||||
}
|
||||
|
||||
impl From<SerialSparseTrie> for ConfiguredSparseTrie {
|
||||
fn from(trie: SerialSparseTrie) -> Self {
|
||||
Self::Serial(Box::new(trie))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ParallelSparseTrie> for ConfiguredSparseTrie {
|
||||
fn from(trie: ParallelSparseTrie) -> Self {
|
||||
Self::Parallel(Box::new(trie))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ConfiguredSparseTrie {
|
||||
fn default() -> Self {
|
||||
Self::Serial(Default::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl SparseTrieInterface for ConfiguredSparseTrie {
|
||||
fn with_root(
|
||||
self,
|
||||
root: TrieNode,
|
||||
masks: Option<BranchNodeMasks>,
|
||||
retain_updates: bool,
|
||||
) -> SparseTrieResult<Self> {
|
||||
match self {
|
||||
Self::Serial(trie) => {
|
||||
trie.with_root(root, masks, retain_updates).map(|t| Self::Serial(Box::new(t)))
|
||||
}
|
||||
Self::Parallel(trie) => {
|
||||
trie.with_root(root, masks, retain_updates).map(|t| Self::Parallel(Box::new(t)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn with_updates(self, retain_updates: bool) -> Self {
|
||||
match self {
|
||||
Self::Serial(trie) => Self::Serial(Box::new(trie.with_updates(retain_updates))),
|
||||
Self::Parallel(trie) => Self::Parallel(Box::new(trie.with_updates(retain_updates))),
|
||||
}
|
||||
}
|
||||
|
||||
fn reserve_nodes(&mut self, additional: usize) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.reserve_nodes(additional),
|
||||
Self::Parallel(trie) => trie.reserve_nodes(additional),
|
||||
}
|
||||
}
|
||||
|
||||
fn reveal_node(
|
||||
&mut self,
|
||||
path: Nibbles,
|
||||
node: TrieNode,
|
||||
masks: Option<BranchNodeMasks>,
|
||||
) -> SparseTrieResult<()> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.reveal_node(path, node, masks),
|
||||
Self::Parallel(trie) => trie.reveal_node(path, node, masks),
|
||||
}
|
||||
}
|
||||
|
||||
fn reveal_nodes(&mut self, nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.reveal_nodes(nodes),
|
||||
Self::Parallel(trie) => trie.reveal_nodes(nodes),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_leaf<P: TrieNodeProvider>(
|
||||
&mut self,
|
||||
full_path: Nibbles,
|
||||
value: Vec<u8>,
|
||||
provider: P,
|
||||
) -> SparseTrieResult<()> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.update_leaf(full_path, value, provider),
|
||||
Self::Parallel(trie) => trie.update_leaf(full_path, value, provider),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_leaf<P: TrieNodeProvider>(
|
||||
&mut self,
|
||||
full_path: &Nibbles,
|
||||
provider: P,
|
||||
) -> SparseTrieResult<()> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.remove_leaf(full_path, provider),
|
||||
Self::Parallel(trie) => trie.remove_leaf(full_path, provider),
|
||||
}
|
||||
}
|
||||
|
||||
fn root(&mut self) -> B256 {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.root(),
|
||||
Self::Parallel(trie) => trie.root(),
|
||||
}
|
||||
}
|
||||
|
||||
fn update_subtrie_hashes(&mut self) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.update_subtrie_hashes(),
|
||||
Self::Parallel(trie) => trie.update_subtrie_hashes(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec<u8>> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.get_leaf_value(full_path),
|
||||
Self::Parallel(trie) => trie.get_leaf_value(full_path),
|
||||
}
|
||||
}
|
||||
|
||||
fn find_leaf(
|
||||
&self,
|
||||
full_path: &Nibbles,
|
||||
expected_value: Option<&Vec<u8>>,
|
||||
) -> Result<LeafLookup, LeafLookupError> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.find_leaf(full_path, expected_value),
|
||||
Self::Parallel(trie) => trie.find_leaf(full_path, expected_value),
|
||||
}
|
||||
}
|
||||
|
||||
fn take_updates(&mut self) -> SparseTrieUpdates {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.take_updates(),
|
||||
Self::Parallel(trie) => trie.take_updates(),
|
||||
}
|
||||
}
|
||||
|
||||
fn wipe(&mut self) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.wipe(),
|
||||
Self::Parallel(trie) => trie.wipe(),
|
||||
}
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.clear(),
|
||||
Self::Parallel(trie) => trie.clear(),
|
||||
}
|
||||
}
|
||||
|
||||
fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.updates_ref(),
|
||||
Self::Parallel(trie) => trie.updates_ref(),
|
||||
}
|
||||
}
|
||||
fn shrink_nodes_to(&mut self, size: usize) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.shrink_nodes_to(size),
|
||||
Self::Parallel(trie) => trie.shrink_nodes_to(size),
|
||||
}
|
||||
}
|
||||
|
||||
fn shrink_values_to(&mut self, size: usize) {
|
||||
match self {
|
||||
Self::Serial(trie) => trie.shrink_values_to(size),
|
||||
Self::Parallel(trie) => trie.shrink_values_to(size),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,22 +2,19 @@
|
||||
|
||||
use super::precompile_cache::PrecompileCacheMap;
|
||||
use crate::tree::{
|
||||
cached_state::{
|
||||
CachedStateMetrics, CachedStateProvider, ExecutionCache as StateExecutionCache,
|
||||
ExecutionCacheBuilder, SavedCache,
|
||||
},
|
||||
cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache},
|
||||
payload_processor::{
|
||||
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
},
|
||||
sparse_trie::SparseTrieTask,
|
||||
sparse_trie::{SparseTrieCacheTask, SparseTrieTask, SpawnedSparseTrieTask},
|
||||
StateProviderBuilder, TreeConfig,
|
||||
};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip1898::BlockWithParent;
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use executor::WorkloadExecutor;
|
||||
use metrics::Counter;
|
||||
use multiproof::{SparseTrieUpdate, *};
|
||||
@@ -42,10 +39,7 @@ use reth_trie_parallel::{
|
||||
proof_task::{ProofTaskCtx, ProofWorkerHandle},
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
ClearedSparseStateTrie, SparseStateTrie, SparseTrie,
|
||||
};
|
||||
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
|
||||
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
@@ -60,14 +54,14 @@ use std::{
|
||||
use tracing::{debug, debug_span, instrument, warn, Span};
|
||||
|
||||
pub mod bal;
|
||||
mod configured_sparse_trie;
|
||||
pub mod executor;
|
||||
pub mod multiproof;
|
||||
mod preserved_sparse_trie;
|
||||
pub mod prewarm;
|
||||
pub mod receipt_root_task;
|
||||
pub mod sparse_trie;
|
||||
|
||||
use configured_sparse_trie::ConfiguredSparseTrie;
|
||||
use preserved_sparse_trie::{PreservedSparseTrie, SharedPreservedSparseTrie};
|
||||
|
||||
/// Default parallelism thresholds to use with the [`ParallelSparseTrie`].
|
||||
///
|
||||
@@ -116,11 +110,11 @@ where
|
||||
/// The executor used by to spawn tasks.
|
||||
executor: WorkloadExecutor,
|
||||
/// The most recent cache used for execution.
|
||||
execution_cache: ExecutionCache,
|
||||
execution_cache: PayloadExecutionCache,
|
||||
/// Metrics for trie operations
|
||||
trie_metrics: MultiProofTaskMetrics,
|
||||
/// Cross-block cache size in bytes.
|
||||
cross_block_cache_size: u64,
|
||||
cross_block_cache_size: usize,
|
||||
/// Whether transactions should not be executed on prewarming task.
|
||||
disable_transaction_prewarming: bool,
|
||||
/// Whether state cache should be disable
|
||||
@@ -131,17 +125,16 @@ where
|
||||
precompile_cache_disabled: bool,
|
||||
/// Precompile cache map.
|
||||
precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// A cleared `SparseStateTrie`, kept around to be reused for the state root computation so
|
||||
/// that allocations can be minimized.
|
||||
sparse_state_trie: Arc<
|
||||
parking_lot::Mutex<
|
||||
Option<ClearedSparseStateTrie<ConfiguredSparseTrie, ConfiguredSparseTrie>>,
|
||||
>,
|
||||
>,
|
||||
/// Whether to disable the parallel sparse trie.
|
||||
disable_parallel_sparse_trie: bool,
|
||||
/// A pruned `SparseStateTrie`, kept around as a cache of already revealed trie nodes and to
|
||||
/// re-use allocated memory. Stored with the block hash it was computed for to enable trie
|
||||
/// preservation across sequential payload validations.
|
||||
sparse_state_trie: SharedPreservedSparseTrie,
|
||||
/// Maximum concurrency for prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Sparse trie prune depth.
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
@@ -152,7 +145,7 @@ where
|
||||
Evm: ConfigureEvm<Primitives = N>,
|
||||
{
|
||||
/// Returns a reference to the workload executor driving payload tasks.
|
||||
pub(super) const fn executor(&self) -> &WorkloadExecutor {
|
||||
pub const fn executor(&self) -> &WorkloadExecutor {
|
||||
&self.executor
|
||||
}
|
||||
|
||||
@@ -173,9 +166,10 @@ where
|
||||
disable_state_cache: config.disable_state_cache(),
|
||||
precompile_cache_disabled: config.precompile_cache_disabled(),
|
||||
precompile_cache_map,
|
||||
sparse_state_trie: Arc::default(),
|
||||
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
|
||||
sparse_state_trie: SharedPreservedSparseTrie::default(),
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
|
||||
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
@@ -241,15 +235,17 @@ where
|
||||
+ 'static,
|
||||
{
|
||||
// start preparing transactions immediately
|
||||
let (prewarm_rx, execution_rx, transaction_count_hint) =
|
||||
self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
|
||||
let span = Span::current();
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
|
||||
|
||||
// Extract V2 proofs flag early so we can pass it to prewarm
|
||||
let v2_proofs_enabled = config.enable_proof_v2();
|
||||
let v2_proofs_enabled = !config.disable_proof_v2();
|
||||
|
||||
// Capture parent_state_root before env is moved into spawn_caching_with
|
||||
let parent_state_root = env.parent_state_root;
|
||||
|
||||
// Handle BAL-based optimization if available
|
||||
let prewarm_handle = if let Some(bal) = bal {
|
||||
@@ -263,7 +259,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
Some(bal),
|
||||
@@ -274,7 +269,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
@@ -294,39 +288,46 @@ where
|
||||
v2_proofs_enabled,
|
||||
);
|
||||
|
||||
let multi_proof_task = MultiProofTask::new(
|
||||
proof_handle.clone(),
|
||||
to_sparse_trie,
|
||||
config
|
||||
.multiproof_chunking_enabled()
|
||||
.then_some(config.effective_multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof,
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
if !config.enable_sparse_trie_as_cache() {
|
||||
let multi_proof_task = MultiProofTask::new(
|
||||
proof_handle.clone(),
|
||||
to_sparse_trie,
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof.clone(),
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
let saved_cache = prewarm_handle.saved_cache.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = parent_span.entered();
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics, _) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
Box::new(provider)
|
||||
};
|
||||
multi_proof_task.run(provider);
|
||||
});
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
let saved_cache = prewarm_handle.saved_cache.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = parent_span.entered();
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics, _disable_metrics) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
Box::new(provider)
|
||||
};
|
||||
multi_proof_task.run(provider);
|
||||
});
|
||||
}
|
||||
|
||||
// wire the sparse trie to the state root response receiver
|
||||
let (state_root_tx, state_root_rx) = channel();
|
||||
|
||||
// Spawn the sparse trie task using any stored trie and parallel trie configuration.
|
||||
self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx);
|
||||
self.spawn_sparse_trie_task(
|
||||
sparse_trie_rx,
|
||||
proof_handle,
|
||||
state_root_tx,
|
||||
from_multi_proof,
|
||||
config,
|
||||
parent_state_root,
|
||||
);
|
||||
|
||||
PayloadHandle {
|
||||
to_multi_proof: Some(to_multi_proof),
|
||||
@@ -341,7 +342,7 @@ where
|
||||
///
|
||||
/// Returns a [`PayloadHandle`] to communicate with the task.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
|
||||
pub(super) fn spawn_cache_exclusive<P, I: ExecutableTxIterator<Evm>>(
|
||||
pub fn spawn_cache_exclusive<P, I: ExecutableTxIterator<Evm>>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: I,
|
||||
@@ -351,10 +352,10 @@ where
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
|
||||
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -372,19 +373,15 @@ where
|
||||
) -> (
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
|
||||
usize,
|
||||
) {
|
||||
let (transactions, convert) = transactions.into();
|
||||
let transactions = transactions.into_par_iter();
|
||||
let transaction_count_hint = transactions.len();
|
||||
|
||||
let (ooo_tx, ooo_rx) = mpsc::channel();
|
||||
let (prewarm_tx, prewarm_rx) = mpsc::channel();
|
||||
let (execute_tx, execute_rx) = mpsc::channel();
|
||||
|
||||
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
rayon::spawn(move || {
|
||||
let (transactions, convert) = transactions.into();
|
||||
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
let tx = convert(tx);
|
||||
let tx = tx.map(|tx| {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
@@ -420,16 +417,14 @@ where
|
||||
}
|
||||
});
|
||||
|
||||
(prewarm_rx, execute_rx, transaction_count_hint)
|
||||
(prewarm_rx, execute_rx)
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
transaction_count_hint: usize,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
@@ -464,7 +459,6 @@ where
|
||||
self.execution_cache.clone(),
|
||||
prewarm_ctx,
|
||||
to_multi_proof,
|
||||
transaction_count_hint,
|
||||
self.prewarm_max_concurrency,
|
||||
);
|
||||
|
||||
@@ -495,75 +489,131 @@ where
|
||||
cache
|
||||
} else {
|
||||
debug!("creating new execution cache on cache miss");
|
||||
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
|
||||
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
|
||||
let start = Instant::now();
|
||||
let cache = ExecutionCache::new(self.cross_block_cache_size);
|
||||
let metrics = CachedStateMetrics::zeroed();
|
||||
metrics.record_cache_creation(start.elapsed());
|
||||
SavedCache::new(parent_hash, cache, metrics)
|
||||
.with_disable_cache_metrics(self.disable_cache_metrics)
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns the [`SparseTrieTask`] for this payload processor.
|
||||
///
|
||||
/// The trie is preserved when the new payload is a child of the previous one.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
|
||||
fn spawn_sparse_trie_task<BPF>(
|
||||
fn spawn_sparse_trie_task(
|
||||
&self,
|
||||
sparse_trie_rx: mpsc::Receiver<SparseTrieUpdate>,
|
||||
proof_worker_handle: BPF,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
state_root_tx: mpsc::Sender<Result<StateRootComputeOutcome, ParallelStateRootError>>,
|
||||
) where
|
||||
BPF: TrieNodeProviderFactory + Clone + Send + Sync + 'static,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
{
|
||||
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
|
||||
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
|
||||
from_multi_proof: CrossbeamReceiver<MultiProofMessage>,
|
||||
config: &TreeConfig,
|
||||
parent_state_root: B256,
|
||||
) {
|
||||
let preserved_sparse_trie = self.sparse_state_trie.clone();
|
||||
let trie_metrics = self.trie_metrics.clone();
|
||||
let span = Span::current();
|
||||
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
|
||||
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
|
||||
// if there's none to reuse.
|
||||
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
|
||||
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
|
||||
ConfiguredSparseTrie::Serial(Default::default())
|
||||
} else {
|
||||
ConfiguredSparseTrie::Parallel(Box::new(
|
||||
ParallelSparseTrie::default()
|
||||
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
|
||||
))
|
||||
});
|
||||
ClearedSparseStateTrie::from_state_trie(
|
||||
// Reuse a stored SparseStateTrie if available, applying continuation logic.
|
||||
// If this payload's parent state root matches the preserved trie's anchor,
|
||||
// we can reuse the pruned trie structure. Otherwise, we clear the trie but
|
||||
// keep allocations.
|
||||
let sparse_state_trie = preserved_sparse_trie
|
||||
.take()
|
||||
.map(|preserved| preserved.into_trie_for(parent_state_root))
|
||||
.unwrap_or_else(|| {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
"Creating new sparse trie - no preserved trie available"
|
||||
);
|
||||
let default_trie = RevealableSparseTrie::blind_from(
|
||||
ParallelSparseTrie::default().with_parallelism_thresholds(
|
||||
PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS,
|
||||
),
|
||||
);
|
||||
SparseStateTrie::new()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true),
|
||||
)
|
||||
});
|
||||
.with_updates(true)
|
||||
});
|
||||
|
||||
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
|
||||
sparse_trie_rx,
|
||||
proof_worker_handle,
|
||||
trie_metrics,
|
||||
sparse_state_trie,
|
||||
);
|
||||
let mut task = if disable_sparse_trie_as_cache {
|
||||
SpawnedSparseTrieTask::Cleared(SparseTrieTask::new(
|
||||
sparse_trie_rx,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
))
|
||||
} else {
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
))
|
||||
};
|
||||
|
||||
let (result, trie) = task.run();
|
||||
// Send state root computation result
|
||||
let _ = state_root_tx.send(result);
|
||||
let result = task.run();
|
||||
// Capture the computed state_root before sending the result
|
||||
let computed_state_root = result.as_ref().ok().map(|outcome| outcome.state_root);
|
||||
|
||||
// Clear the SparseStateTrie, shrink, and replace it back into the mutex _after_ sending
|
||||
// results to the next step, so that time spent clearing doesn't block the step after
|
||||
// this one.
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor", "clear").entered();
|
||||
let mut cleared_trie = ClearedSparseStateTrie::from_state_trie(trie);
|
||||
// Acquire the guard before sending the result to prevent a race condition:
|
||||
// Without this, the next block could start after send() but before store(),
|
||||
// causing take() to return None and forcing it to create a new empty trie
|
||||
// instead of reusing the preserved one. Holding the guard ensures the next
|
||||
// block's take() blocks until we've stored the trie for reuse.
|
||||
let mut guard = preserved_sparse_trie.lock();
|
||||
|
||||
// Shrink the sparse trie so that we don't have ever increasing memory.
|
||||
cleared_trie.shrink_to(
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
// Send state root computation result - next block may start but will block on take()
|
||||
if state_root_tx.send(result).is_err() {
|
||||
// Receiver dropped - payload was likely invalid or cancelled.
|
||||
// Clear the trie instead of preserving potentially invalid state.
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
"State root receiver dropped, clearing trie"
|
||||
);
|
||||
let trie = task.into_cleared_trie(
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
guard.store(PreservedSparseTrie::cleared(trie));
|
||||
return;
|
||||
}
|
||||
|
||||
cleared_sparse_trie.lock().replace(cleared_trie);
|
||||
// Only preserve the trie as anchored if computation succeeded.
|
||||
// A failed computation may have left the trie in a partially updated state.
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor", "preserve").entered();
|
||||
if let Some(state_root) = computed_state_root {
|
||||
let start = std::time::Instant::now();
|
||||
let trie = task.into_trie_for_reuse(
|
||||
prune_depth,
|
||||
max_storage_tries,
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
trie_metrics
|
||||
.into_trie_for_reuse_duration_histogram
|
||||
.record(start.elapsed().as_secs_f64());
|
||||
guard.store(PreservedSparseTrie::anchored(trie, state_root));
|
||||
} else {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
"State root computation failed, clearing trie"
|
||||
);
|
||||
let trie = task.into_cleared_trie(
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
guard.store(PreservedSparseTrie::cleared(trie));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -574,7 +624,7 @@ where
|
||||
///
|
||||
/// The cache enables subsequent blocks to reuse account, storage, and bytecode data without
|
||||
/// hitting the database, maintaining performance consistency.
|
||||
pub(crate) fn on_inserted_executed_block(
|
||||
pub fn on_inserted_executed_block(
|
||||
&self,
|
||||
block_with_parent: BlockWithParent,
|
||||
bundle_state: &BundleState,
|
||||
@@ -587,28 +637,27 @@ where
|
||||
parent_hash = %block_with_parent.parent,
|
||||
"Cannot find cache for parent hash, skip updating cache with new state for inserted executed block",
|
||||
);
|
||||
return;
|
||||
return
|
||||
}
|
||||
|
||||
// Take existing cache (if any) or create fresh caches
|
||||
let (caches, cache_metrics) = match cached.take() {
|
||||
Some(existing) => {
|
||||
let (c, m, _) = existing.split();
|
||||
(c, m)
|
||||
}
|
||||
let (caches, cache_metrics, _) = match cached.take() {
|
||||
Some(existing) => existing.split(),
|
||||
None => (
|
||||
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
|
||||
ExecutionCache::new(self.cross_block_cache_size),
|
||||
CachedStateMetrics::zeroed(),
|
||||
false,
|
||||
),
|
||||
};
|
||||
|
||||
// Insert the block's bundle state into cache
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
let new_cache =
|
||||
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
if new_cache.cache().insert_state(bundle_state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
return;
|
||||
return
|
||||
}
|
||||
new_cache.update_metrics();
|
||||
|
||||
@@ -672,19 +721,19 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
}
|
||||
|
||||
/// Returns a clone of the caches used by prewarming
|
||||
pub(super) fn caches(&self) -> Option<StateExecutionCache> {
|
||||
pub fn caches(&self) -> Option<ExecutionCache> {
|
||||
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
|
||||
}
|
||||
|
||||
/// Returns a clone of the cache metrics used by prewarming
|
||||
pub(super) fn cache_metrics(&self) -> Option<CachedStateMetrics> {
|
||||
pub fn cache_metrics(&self) -> Option<CachedStateMetrics> {
|
||||
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.metrics().clone())
|
||||
}
|
||||
|
||||
/// Terminates the pre-warming transaction processing.
|
||||
///
|
||||
/// Note: This does not terminate the task yet.
|
||||
pub(super) fn stop_prewarming_execution(&self) {
|
||||
pub fn stop_prewarming_execution(&self) {
|
||||
self.prewarm_handle.stop_prewarming_execution()
|
||||
}
|
||||
|
||||
@@ -695,7 +744,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
/// path without cloning the expensive `BundleState`.
|
||||
///
|
||||
/// Returns a sender for the channel that should be notified on block validation success.
|
||||
pub(super) fn terminate_caching(
|
||||
pub fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
|
||||
) -> Option<mpsc::Sender<()>> {
|
||||
@@ -715,7 +764,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
|
||||
/// prewarm task without cloning the expensive `BundleState`.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct CacheTaskHandle<R> {
|
||||
pub struct CacheTaskHandle<R> {
|
||||
/// The shared cache the task operates with.
|
||||
saved_cache: Option<SavedCache>,
|
||||
/// Channel to the spawned prewarm task if any
|
||||
@@ -726,7 +775,7 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
/// Terminates the pre-warming transaction processing.
|
||||
///
|
||||
/// Note: This does not terminate the task yet.
|
||||
pub(super) fn stop_prewarming_execution(&self) {
|
||||
pub fn stop_prewarming_execution(&self) {
|
||||
self.to_prewarm_task
|
||||
.as_ref()
|
||||
.map(|tx| tx.send(PrewarmTaskEvent::TerminateTransactionExecution).ok());
|
||||
@@ -737,7 +786,7 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
|
||||
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
|
||||
#[must_use = "sender must be used and notified on block validation success"]
|
||||
pub(super) fn terminate_caching(
|
||||
pub fn terminate_caching(
|
||||
&mut self,
|
||||
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
|
||||
) -> Option<mpsc::Sender<()>> {
|
||||
@@ -776,29 +825,29 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
/// ## Cache Safety
|
||||
///
|
||||
/// **CRITICAL**: Cache update operations require exclusive access. All concurrent cache users
|
||||
/// (such as prewarming tasks) must be terminated before calling `update_with_guard`, otherwise
|
||||
/// the cache may be corrupted or cleared.
|
||||
/// (such as prewarming tasks) must be terminated before calling
|
||||
/// [`PayloadExecutionCache::update_with_guard`], otherwise the cache may be corrupted or cleared.
|
||||
///
|
||||
/// ## Cache vs Prewarming Distinction
|
||||
///
|
||||
/// **`ExecutionCache`**:
|
||||
/// **[`PayloadExecutionCache`]**:
|
||||
/// - Stores parent block's execution state after completion
|
||||
/// - Used to fetch parent data for next block's execution
|
||||
/// - Must be exclusively accessed during save operations
|
||||
///
|
||||
/// **`PrewarmCacheTask`**:
|
||||
/// **[`PrewarmCacheTask`]**:
|
||||
/// - Speculatively loads accounts/storage that might be used in transaction execution
|
||||
/// - Prepares data for state root proof computation
|
||||
/// - Runs concurrently but must not interfere with cache saves
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct ExecutionCache {
|
||||
pub struct PayloadExecutionCache {
|
||||
/// Guarded cloneable cache identified by a block hash.
|
||||
inner: Arc<RwLock<Option<SavedCache>>>,
|
||||
/// Metrics for cache operations.
|
||||
metrics: ExecutionCacheMetrics,
|
||||
}
|
||||
|
||||
impl ExecutionCache {
|
||||
impl PayloadExecutionCache {
|
||||
/// Returns the cache for `parent_hash` if it's available for use.
|
||||
///
|
||||
/// A cache is considered available when:
|
||||
@@ -834,11 +883,15 @@ impl ExecutionCache {
|
||||
"Existing cache found"
|
||||
);
|
||||
|
||||
if hash_matches && available {
|
||||
return Some(c.clone());
|
||||
}
|
||||
|
||||
if hash_matches && !available {
|
||||
if available {
|
||||
// If the has is available (no other threads are using it), but has a mismatching
|
||||
// parent hash, we can just clear it and keep using without re-creating from
|
||||
// scratch.
|
||||
if !hash_matches {
|
||||
c.clear();
|
||||
}
|
||||
return Some(c.clone())
|
||||
} else if hash_matches {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
} else {
|
||||
@@ -867,7 +920,7 @@ impl ExecutionCache {
|
||||
///
|
||||
/// Violating this requirement can result in cache corruption, incorrect state data,
|
||||
/// and potential consensus failures.
|
||||
pub(crate) fn update_with_guard<F>(&self, update_fn: F)
|
||||
pub fn update_with_guard<F>(&self, update_fn: F)
|
||||
where
|
||||
F: FnOnce(&mut Option<SavedCache>),
|
||||
{
|
||||
@@ -894,6 +947,14 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
pub hash: B256,
|
||||
/// Hash of the parent block.
|
||||
pub parent_hash: B256,
|
||||
/// State root of the parent block.
|
||||
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
|
||||
/// the trie can be reused directly.
|
||||
pub parent_state_root: B256,
|
||||
/// Number of transactions in the block.
|
||||
/// Used to determine parallel worker count for prewarming.
|
||||
/// A value of 0 indicates the count is unknown.
|
||||
pub transaction_count: usize,
|
||||
}
|
||||
|
||||
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
|
||||
@@ -905,15 +966,17 @@ where
|
||||
evm_env: Default::default(),
|
||||
hash: Default::default(),
|
||||
parent_hash: Default::default(),
|
||||
parent_state_root: Default::default(),
|
||||
transaction_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::ExecutionCache;
|
||||
use super::PayloadExecutionCache;
|
||||
use crate::tree::{
|
||||
cached_state::{CachedStateMetrics, ExecutionCacheBuilder, SavedCache},
|
||||
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
|
||||
payload_processor::{
|
||||
evm_state_to_hashed_post_state, executor::WorkloadExecutor, PayloadProcessor,
|
||||
},
|
||||
@@ -943,13 +1006,13 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
fn make_saved_cache(hash: B256) -> SavedCache {
|
||||
let execution_cache = ExecutionCacheBuilder::default().build_caches(1_000);
|
||||
let execution_cache = ExecutionCache::new(1_000);
|
||||
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_cache_allows_single_checkout() {
|
||||
let execution_cache = ExecutionCache::default();
|
||||
let execution_cache = PayloadExecutionCache::default();
|
||||
let hash = B256::from([1u8; 32]);
|
||||
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
|
||||
@@ -968,7 +1031,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn execution_cache_checkout_releases_on_drop() {
|
||||
let execution_cache = ExecutionCache::default();
|
||||
let execution_cache = PayloadExecutionCache::default();
|
||||
let hash = B256::from([2u8; 32]);
|
||||
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
|
||||
@@ -984,19 +1047,21 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_cache_mismatch_parent_returns_none() {
|
||||
let execution_cache = ExecutionCache::default();
|
||||
fn execution_cache_mismatch_parent_clears_and_returns() {
|
||||
let execution_cache = PayloadExecutionCache::default();
|
||||
let hash = B256::from([3u8; 32]);
|
||||
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
|
||||
|
||||
let miss = execution_cache.get_cache_for(B256::from([4u8; 32]));
|
||||
assert!(miss.is_none(), "checkout should fail for different parent hash");
|
||||
// When the parent hash doesn't match, the cache is cleared and returned for reuse
|
||||
let different_hash = B256::from([4u8; 32]);
|
||||
let cache = execution_cache.get_cache_for(different_hash);
|
||||
assert!(cache.is_some(), "cache should be returned for reuse after clearing")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_cache_update_after_release_succeeds() {
|
||||
let execution_cache = ExecutionCache::default();
|
||||
let execution_cache = PayloadExecutionCache::default();
|
||||
let initial = B256::from([5u8; 32]);
|
||||
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(initial)));
|
||||
|
||||
@@ -100,7 +100,7 @@ impl SparseTrieUpdate {
|
||||
|
||||
/// Messages used internally by the multi proof task.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum MultiProofMessage {
|
||||
pub enum MultiProofMessage {
|
||||
/// Prefetch proof targets
|
||||
PrefetchProofs(VersionedMultiProofTargets),
|
||||
/// New state update from transaction execution with its source
|
||||
@@ -257,7 +257,7 @@ fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiP
|
||||
|
||||
/// A set of multiproof targets which can be either in the legacy or V2 representations.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum VersionedMultiProofTargets {
|
||||
pub enum VersionedMultiProofTargets {
|
||||
/// Legacy targets
|
||||
Legacy(MultiProofTargets),
|
||||
/// V2 targets
|
||||
@@ -587,6 +587,8 @@ pub(crate) struct MultiProofTaskMetrics {
|
||||
pub first_update_wait_time_histogram: Histogram,
|
||||
/// Total time spent waiting for the last proof result.
|
||||
pub last_proof_wait_time_histogram: Histogram,
|
||||
/// Time spent preparing the sparse trie for reuse after state root computation.
|
||||
pub into_trie_for_reuse_duration_histogram: Histogram,
|
||||
}
|
||||
|
||||
/// Standalone task that receives a transaction state stream and updates relevant
|
||||
@@ -777,10 +779,21 @@ impl MultiProofTask {
|
||||
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
|
||||
// we still want to optimistically fetch extension children for the leaf addition case.
|
||||
// V2 multiproofs don't need this.
|
||||
//
|
||||
// Only clone the AddedRemovedKeys for accounts in the targets, not the entire accumulated
|
||||
// set, to avoid O(n) cloning with many buffered blocks.
|
||||
let multi_added_removed_keys =
|
||||
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
|
||||
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
|
||||
Some(Arc::new(self.multi_added_removed_keys.clone()))
|
||||
Some(Arc::new(MultiAddedRemovedKeys {
|
||||
account: self.multi_added_removed_keys.account.clone(),
|
||||
storages: legacy_targets
|
||||
.keys()
|
||||
.filter_map(|k| {
|
||||
self.multi_added_removed_keys.storages.get(k).map(|v| (*k, v.clone()))
|
||||
})
|
||||
.collect(),
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -1516,8 +1529,9 @@ where
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::tree::cached_state::CachedStateProvider;
|
||||
|
||||
use super::*;
|
||||
use crate::tree::cached_state::{CachedStateProvider, ExecutionCacheBuilder};
|
||||
use alloy_eip7928::{AccountChanges, BalanceChange};
|
||||
use alloy_primitives::Address;
|
||||
use reth_provider::{
|
||||
@@ -1577,7 +1591,7 @@ mod tests {
|
||||
{
|
||||
let db_provider = factory.database_provider_ro().unwrap();
|
||||
let state_provider: StateProviderBox = Box::new(LatestStateProvider::new(db_provider));
|
||||
let cache = ExecutionCacheBuilder::default().build_caches(1000);
|
||||
let cache = crate::tree::cached_state::ExecutionCache::new(1000);
|
||||
CachedStateProvider::new(state_provider, cache, Default::default())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
//! Preserved sparse trie for reuse across payload validations.
|
||||
|
||||
use alloy_primitives::B256;
|
||||
use parking_lot::Mutex;
|
||||
use reth_trie_sparse::SparseStateTrie;
|
||||
use reth_trie_sparse_parallel::ParallelSparseTrie;
|
||||
use std::sync::Arc;
|
||||
use tracing::debug;
|
||||
|
||||
/// Type alias for the sparse trie type used in preservation.
|
||||
pub(super) type SparseTrie = SparseStateTrie<ParallelSparseTrie, ParallelSparseTrie>;
|
||||
|
||||
/// Shared handle to a preserved sparse trie that can be reused across payload validations.
|
||||
///
|
||||
/// This is stored in [`PayloadProcessor`](super::PayloadProcessor) and cloned to pass to
|
||||
/// [`SparseTrieTask`](super::sparse_trie::SparseTrieTask) for trie reuse.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(super) struct SharedPreservedSparseTrie(Arc<Mutex<Option<PreservedSparseTrie>>>);
|
||||
|
||||
impl SharedPreservedSparseTrie {
|
||||
/// Takes the preserved trie if present, leaving `None` in its place.
|
||||
pub(super) fn take(&self) -> Option<PreservedSparseTrie> {
|
||||
self.0.lock().take()
|
||||
}
|
||||
|
||||
/// Acquires a guard that blocks `take()` until dropped.
|
||||
/// Use this before sending the state root result to ensure the next block
|
||||
/// waits for the trie to be stored.
|
||||
pub(super) fn lock(&self) -> PreservedTrieGuard<'_> {
|
||||
PreservedTrieGuard(self.0.lock())
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that holds the lock on the preserved trie.
|
||||
/// While held, `take()` will block. Call `store()` to save the trie before dropping.
|
||||
pub(super) struct PreservedTrieGuard<'a>(parking_lot::MutexGuard<'a, Option<PreservedSparseTrie>>);
|
||||
|
||||
impl PreservedTrieGuard<'_> {
|
||||
/// Stores a preserved trie for later reuse.
|
||||
pub(super) fn store(&mut self, trie: PreservedSparseTrie) {
|
||||
self.0.replace(trie);
|
||||
}
|
||||
}
|
||||
|
||||
/// A preserved sparse trie that can be reused across payload validations.
|
||||
///
|
||||
/// The trie exists in one of two states:
|
||||
/// - **Anchored**: Has a computed state root and can be reused for payloads whose parent state root
|
||||
/// matches the anchor.
|
||||
/// - **Cleared**: Trie data has been cleared but allocations are preserved for reuse.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum PreservedSparseTrie {
|
||||
/// Trie with a computed state root that can be reused for continuation payloads.
|
||||
Anchored {
|
||||
/// The sparse state trie (pruned after root computation).
|
||||
trie: SparseTrie,
|
||||
/// The state root this trie represents (computed from the previous block).
|
||||
/// Used to verify continuity: new payload's `parent_state_root` must match this.
|
||||
state_root: B256,
|
||||
},
|
||||
/// Cleared trie with preserved allocations, ready for fresh use.
|
||||
Cleared {
|
||||
/// The sparse state trie with cleared data but preserved allocations.
|
||||
trie: SparseTrie,
|
||||
},
|
||||
}
|
||||
|
||||
impl PreservedSparseTrie {
|
||||
/// Creates a new anchored preserved trie.
|
||||
///
|
||||
/// The `state_root` is the computed state root from the trie, which becomes the
|
||||
/// anchor for determining if subsequent payloads can reuse this trie.
|
||||
pub(super) const fn anchored(trie: SparseTrie, state_root: B256) -> Self {
|
||||
Self::Anchored { trie, state_root }
|
||||
}
|
||||
|
||||
/// Creates a cleared preserved trie (allocations preserved, data cleared).
|
||||
pub(super) const fn cleared(trie: SparseTrie) -> Self {
|
||||
Self::Cleared { trie }
|
||||
}
|
||||
|
||||
/// Consumes self and returns the trie for reuse.
|
||||
///
|
||||
/// If the preserved trie is anchored and the parent state root matches, the pruned
|
||||
/// trie structure is reused directly. Otherwise, the trie is cleared but allocations
|
||||
/// are preserved to reduce memory overhead.
|
||||
pub(super) fn into_trie_for(self, parent_state_root: B256) -> SparseTrie {
|
||||
match self {
|
||||
Self::Anchored { trie, state_root } if state_root == parent_state_root => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
%state_root,
|
||||
"Reusing anchored sparse trie for continuation payload"
|
||||
);
|
||||
trie
|
||||
}
|
||||
Self::Anchored { mut trie, state_root } => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
anchor_root = %state_root,
|
||||
%parent_state_root,
|
||||
"Clearing anchored sparse trie - parent state root mismatch"
|
||||
);
|
||||
trie.clear();
|
||||
trie
|
||||
}
|
||||
Self::Cleared { trie } => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
%parent_state_root,
|
||||
"Using cleared sparse trie with preserved allocations"
|
||||
);
|
||||
trie
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,17 +17,16 @@ use crate::tree::{
|
||||
bal::{total_slots, BALSlotIter},
|
||||
executor::WorkloadExecutor,
|
||||
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
|
||||
ExecutionCache as PayloadExecutionCache,
|
||||
PayloadExecutionCache,
|
||||
},
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
ExecutionEnv, StateProviderBuilder,
|
||||
};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::Typed2718;
|
||||
use alloy_evm::Database;
|
||||
use alloy_primitives::{keccak256, map::B256Set, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
|
||||
use reth_metrics::Metrics;
|
||||
@@ -50,7 +49,8 @@ use std::{
|
||||
use tracing::{debug, debug_span, instrument, trace, warn, Span};
|
||||
|
||||
/// Determines the prewarming mode: transaction-based or BAL-based.
|
||||
pub(super) enum PrewarmMode<Tx> {
|
||||
#[derive(Debug)]
|
||||
pub enum PrewarmMode<Tx> {
|
||||
/// Prewarm by executing transactions from a stream.
|
||||
Transactions(Receiver<Tx>),
|
||||
/// Prewarm by prefetching slots from a Block Access List.
|
||||
@@ -66,24 +66,12 @@ struct IndexedTransaction<Tx> {
|
||||
tx: Tx,
|
||||
}
|
||||
|
||||
/// Maximum standard Ethereum transaction type value.
|
||||
///
|
||||
/// Standard transaction types are:
|
||||
/// - Type 0: Legacy transactions (original Ethereum)
|
||||
/// - Type 1: EIP-2930 (access list transactions)
|
||||
/// - Type 2: EIP-1559 (dynamic fee transactions)
|
||||
/// - Type 3: EIP-4844 (blob transactions)
|
||||
/// - Type 4: EIP-7702 (set code authorization transactions)
|
||||
///
|
||||
/// Any transaction with a type > 4 is considered a non-standard/system transaction,
|
||||
/// typically used by L2s for special purposes (e.g., Optimism deposit transactions use type 126).
|
||||
const MAX_STANDARD_TX_TYPE: u8 = 4;
|
||||
|
||||
/// A task that is responsible for caching and prewarming the cache by executing transactions
|
||||
/// individually in parallel.
|
||||
///
|
||||
/// Note: This task runs until cancelled externally.
|
||||
pub(super) struct PrewarmCacheTask<N, P, Evm>
|
||||
#[derive(Debug)]
|
||||
pub struct PrewarmCacheTask<N, P, Evm>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
Evm: ConfigureEvm<Primitives = N>,
|
||||
@@ -96,8 +84,6 @@ where
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
/// How many transactions should be executed in parallel
|
||||
max_concurrency: usize,
|
||||
/// The number of transactions to be processed
|
||||
transaction_count_hint: usize,
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
@@ -113,12 +99,11 @@ where
|
||||
Evm: ConfigureEvm<Primitives = N> + 'static,
|
||||
{
|
||||
/// Initializes the task with the given transactions pending execution
|
||||
pub(super) fn new(
|
||||
pub fn new(
|
||||
executor: WorkloadExecutor,
|
||||
execution_cache: PayloadExecutionCache,
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
transaction_count_hint: usize,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
@@ -126,7 +111,7 @@ where
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
transaction_count = ctx.env.transaction_count,
|
||||
"Initialized prewarm task"
|
||||
);
|
||||
|
||||
@@ -136,7 +121,6 @@ where
|
||||
execution_cache,
|
||||
ctx,
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
to_multi_proof,
|
||||
actions_rx,
|
||||
parent_span: Span::current(),
|
||||
@@ -160,7 +144,6 @@ where
|
||||
let executor = self.executor.clone();
|
||||
let ctx = self.ctx.clone();
|
||||
let max_concurrency = self.max_concurrency;
|
||||
let transaction_count_hint = self.transaction_count_hint;
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
@@ -168,17 +151,18 @@ where
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
|
||||
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
|
||||
// When transaction_count is 0, it means the count is unknown. In this case, spawn
|
||||
// max workers to handle potentially many transactions in parallel rather
|
||||
// than bottlenecking on a single worker.
|
||||
let workers_needed = if transaction_count_hint == 0 {
|
||||
let transaction_count = ctx.env.transaction_count;
|
||||
let workers_needed = if transaction_count == 0 {
|
||||
max_concurrency
|
||||
} else {
|
||||
transaction_count_hint.min(max_concurrency)
|
||||
transaction_count.min(max_concurrency)
|
||||
};
|
||||
|
||||
// Initialize worker handles container
|
||||
let handles = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
|
||||
// Spawn workers
|
||||
let tx_sender = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
|
||||
|
||||
// Distribute transactions to workers
|
||||
let mut tx_index = 0usize;
|
||||
@@ -193,37 +177,18 @@ where
|
||||
}
|
||||
|
||||
let indexed_tx = IndexedTransaction { index: tx_index, tx };
|
||||
let is_system_tx = indexed_tx.tx.tx().ty() > MAX_STANDARD_TX_TYPE;
|
||||
|
||||
// System transactions (type > 4) in the first position set critical metadata
|
||||
// that affects all subsequent transactions (e.g., L1 block info on L2s).
|
||||
// Broadcast the first system transaction to all workers to ensure they have
|
||||
// the critical state. This is particularly important for L2s like Optimism
|
||||
// where the first deposit transaction (type 126) contains essential block metadata.
|
||||
if tx_index == 0 && is_system_tx {
|
||||
for handle in &handles {
|
||||
// Ignore send errors: workers listen to terminate_execution and may
|
||||
// exit early when signaled. Sending to a disconnected worker is
|
||||
// possible and harmless and should happen at most once due to
|
||||
// the terminate_execution check above.
|
||||
let _ = handle.send(indexed_tx.clone());
|
||||
}
|
||||
} else {
|
||||
// Round-robin distribution for all other transactions
|
||||
let worker_idx = tx_index % workers_needed;
|
||||
// Ignore send errors: workers listen to terminate_execution and may
|
||||
// exit early when signaled. Sending to a disconnected worker is
|
||||
// possible and harmless and should happen at most once due to
|
||||
// the terminate_execution check above.
|
||||
let _ = handles[worker_idx].send(indexed_tx);
|
||||
}
|
||||
// Send transaction to the workers
|
||||
// Ignore send errors: workers listen to terminate_execution and may
|
||||
// exit early when signaled.
|
||||
let _ = tx_sender.send(indexed_tx);
|
||||
|
||||
tx_index += 1;
|
||||
}
|
||||
|
||||
// drop handle and wait for all tasks to finish and drop theirs
|
||||
// drop sender and wait for all tasks to finish
|
||||
drop(done_tx);
|
||||
drop(handles);
|
||||
drop(tx_sender);
|
||||
while done_rx.recv().is_ok() {}
|
||||
|
||||
let _ = actions_tx
|
||||
@@ -403,11 +368,8 @@ where
|
||||
name = "prewarm and caching",
|
||||
skip_all
|
||||
)]
|
||||
pub(super) fn run<Tx>(
|
||||
self,
|
||||
mode: PrewarmMode<Tx>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) where
|
||||
pub fn run<Tx>(self, mode: PrewarmMode<Tx>, actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>)
|
||||
where
|
||||
Tx: ExecutableTxFor<Evm> + Clone + Send + 'static,
|
||||
{
|
||||
// Spawn execution tasks based on mode
|
||||
@@ -469,23 +431,29 @@ where
|
||||
|
||||
/// Context required by tx execution tasks.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct PrewarmContext<N, P, Evm>
|
||||
pub struct PrewarmContext<N, P, Evm>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
Evm: ConfigureEvm<Primitives = N>,
|
||||
{
|
||||
pub(super) env: ExecutionEnv<Evm>,
|
||||
pub(super) evm_config: Evm,
|
||||
pub(super) saved_cache: Option<SavedCache>,
|
||||
/// The execution environment.
|
||||
pub env: ExecutionEnv<Evm>,
|
||||
/// The EVM configuration.
|
||||
pub evm_config: Evm,
|
||||
/// The saved cache.
|
||||
pub saved_cache: Option<SavedCache>,
|
||||
/// Provider to obtain the state
|
||||
pub(super) provider: StateProviderBuilder<N, P>,
|
||||
pub(super) metrics: PrewarmMetrics,
|
||||
pub provider: StateProviderBuilder<N, P>,
|
||||
/// The metrics for the prewarm task.
|
||||
pub metrics: PrewarmMetrics,
|
||||
/// An atomic bool that tells prewarm tasks to not start any more execution.
|
||||
pub(super) terminate_execution: Arc<AtomicBool>,
|
||||
pub(super) precompile_cache_disabled: bool,
|
||||
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
pub terminate_execution: Arc<AtomicBool>,
|
||||
/// Whether the precompile cache is disabled.
|
||||
pub precompile_cache_disabled: bool,
|
||||
/// The precompile cache map.
|
||||
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether V2 proof calculation is enabled.
|
||||
pub(super) v2_proofs_enabled: bool,
|
||||
pub v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl<N, P, Evm> PrewarmContext<N, P, Evm>
|
||||
@@ -562,7 +530,7 @@ where
|
||||
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
|
||||
}
|
||||
|
||||
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
|
||||
/// Accepts a [`CrossbeamReceiver`] of transactions and a handle to prewarm task. Executes
|
||||
/// transactions and streams [`PrewarmTaskEvent::Outcome`] messages for each transaction.
|
||||
///
|
||||
/// This function processes transactions sequentially from the receiver and emits outcome events
|
||||
@@ -574,7 +542,7 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn transact_batch<Tx>(
|
||||
self,
|
||||
txs: mpsc::Receiver<IndexedTransaction<Tx>>,
|
||||
txs: CrossbeamReceiver<IndexedTransaction<Tx>>,
|
||||
sender: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
done_tx: Sender<()>,
|
||||
) where
|
||||
@@ -596,6 +564,7 @@ where
|
||||
index,
|
||||
tx_hash = %tx.tx().tx_hash(),
|
||||
is_success = tracing::field::Empty,
|
||||
gas_used = tracing::field::Empty,
|
||||
)
|
||||
.entered();
|
||||
|
||||
@@ -661,35 +630,31 @@ where
|
||||
let _ = done_tx.send(());
|
||||
}
|
||||
|
||||
/// Spawns a worker task for transaction execution and returns its sender channel.
|
||||
/// Spawns worker tasks that pull transactions from a shared channel.
|
||||
///
|
||||
/// Returns the sender for distributing transactions to workers.
|
||||
fn spawn_workers<Tx>(
|
||||
self,
|
||||
workers_needed: usize,
|
||||
task_executor: &WorkloadExecutor,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
done_tx: Sender<()>,
|
||||
) -> Vec<mpsc::Sender<IndexedTransaction<Tx>>>
|
||||
) -> CrossbeamSender<IndexedTransaction<Tx>>
|
||||
where
|
||||
Tx: ExecutableTxFor<Evm> + Send + 'static,
|
||||
{
|
||||
let mut handles = Vec::with_capacity(workers_needed);
|
||||
let mut receivers = Vec::with_capacity(workers_needed);
|
||||
let (tx_sender, tx_receiver) = crossbeam_channel::unbounded();
|
||||
|
||||
for _ in 0..workers_needed {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
handles.push(tx);
|
||||
receivers.push(rx);
|
||||
}
|
||||
|
||||
// Spawn a separate task spawning workers in parallel.
|
||||
// Spawn workers that all pull from the shared receiver
|
||||
let executor = task_executor.clone();
|
||||
let span = Span::current();
|
||||
task_executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
for (idx, rx) in receivers.into_iter().enumerate() {
|
||||
for idx in 0..workers_needed {
|
||||
let ctx = self.clone();
|
||||
let actions_tx = actions_tx.clone();
|
||||
let done_tx = done_tx.clone();
|
||||
let rx = tx_receiver.clone();
|
||||
let span = debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx);
|
||||
executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
@@ -698,7 +663,7 @@ where
|
||||
}
|
||||
});
|
||||
|
||||
handles
|
||||
tx_sender
|
||||
}
|
||||
|
||||
/// Spawns a worker task for BAL slot prefetching.
|
||||
@@ -888,7 +853,8 @@ fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTarg
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the main
|
||||
/// execution path without cloning the expensive `BundleState`.
|
||||
pub(super) enum PrewarmTaskEvent<R> {
|
||||
#[derive(Debug)]
|
||||
pub enum PrewarmTaskEvent<R> {
|
||||
/// Forcefully terminate all remaining transaction execution.
|
||||
TerminateTransactionExecution,
|
||||
/// Forcefully terminate the task on demand and update the shared cache with the given output
|
||||
@@ -918,7 +884,7 @@ pub(super) enum PrewarmTaskEvent<R> {
|
||||
/// Metrics for transactions prewarming.
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "sync.prewarm")]
|
||||
pub(crate) struct PrewarmMetrics {
|
||||
pub struct PrewarmMetrics {
|
||||
/// The number of transactions to prewarm
|
||||
pub(crate) transactions: Gauge,
|
||||
/// A histogram of the number of transactions to prewarm
|
||||
|
||||
@@ -77,8 +77,22 @@ impl<R: Receipt> ReceiptRootTaskHandle<R> {
|
||||
receipt_with_bloom.encode_2718(&mut encode_buf);
|
||||
|
||||
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
|
||||
builder.push_unchecked(indexed_receipt.index, &encode_buf);
|
||||
received_count += 1;
|
||||
match builder.push(indexed_receipt.index, &encode_buf) {
|
||||
Ok(()) => {
|
||||
received_count += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
// If a duplicate or out-of-bounds index is streamed, skip it and
|
||||
// fall back to computing the receipt root from the full receipts
|
||||
// vector later.
|
||||
tracing::error!(
|
||||
target: "engine::tree::payload_processor",
|
||||
index = indexed_receipt.index,
|
||||
?err,
|
||||
"Receipt root task received invalid receipt index, skipping"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let Ok(root) = builder.finalize() else {
|
||||
|
||||
@@ -1,15 +1,34 @@
|
||||
//! Sparse Trie task related functionality.
|
||||
|
||||
use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate};
|
||||
use crate::tree::{
|
||||
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
|
||||
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use reth_trie::{updates::TrieUpdates, Nibbles};
|
||||
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use rayon::iter::ParallelIterator;
|
||||
use reth_errors::ProviderError;
|
||||
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
|
||||
TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
},
|
||||
root::ParallelStateRootError,
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrieInterface,
|
||||
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::mpsc,
|
||||
@@ -17,6 +36,64 @@ use std::{
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, trace};
|
||||
|
||||
#[expect(clippy::large_enum_variant)]
|
||||
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + SparseTrieExt + Send + Sync + Default,
|
||||
S: SparseTrie + SparseTrieExt + Send + Sync + Default + Clone,
|
||||
{
|
||||
Cleared(SparseTrieTask<BPF, A, S>),
|
||||
Cached(SparseTrieCacheTask<A, S>),
|
||||
}
|
||||
|
||||
impl<BPF, A, S> SpawnedSparseTrieTask<BPF, A, S>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + SparseTrieExt + Send + Sync + Default,
|
||||
S: SparseTrie + SparseTrieExt + Send + Sync + Default + Clone,
|
||||
{
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
match self {
|
||||
Self::Cleared(task) => task.run(),
|
||||
Self::Cached(task) => task.run(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_trie_for_reuse(
|
||||
self,
|
||||
prune_depth: usize,
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> SparseStateTrie<A, S> {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
Self::Cached(task) => task.into_trie_for_reuse(
|
||||
prune_depth,
|
||||
max_storage_tries,
|
||||
max_nodes_capacity,
|
||||
max_values_capacity,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_cleared_trie(
|
||||
self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> SparseStateTrie<A, S> {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
Self::Cached(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A task responsible for populating the sparse trie.
|
||||
pub(super) struct SparseTrieTask<BPF, A = SerialSparseTrie, S = SerialSparseTrie>
|
||||
where
|
||||
@@ -38,46 +115,29 @@ where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrieInterface + Send + Sync + Default,
|
||||
S: SparseTrieInterface + Send + Sync + Default + Clone,
|
||||
A: SparseTrie + SparseTrieExt + Send + Sync + Default,
|
||||
S: SparseTrie + SparseTrieExt + Send + Sync + Default + Clone,
|
||||
{
|
||||
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
|
||||
pub(super) fn new_with_cleared_trie(
|
||||
/// Creates a new sparse trie task with the given trie.
|
||||
pub(super) const fn new(
|
||||
updates: mpsc::Receiver<SparseTrieUpdate>,
|
||||
blinded_provider_factory: BPF,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
sparse_state_trie: ClearedSparseStateTrie<A, S>,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
) -> Self {
|
||||
Self { updates, metrics, trie: sparse_state_trie.into_inner(), blinded_provider_factory }
|
||||
Self { updates, metrics, trie, blinded_provider_factory }
|
||||
}
|
||||
|
||||
/// Runs the sparse trie task to completion.
|
||||
/// Runs the sparse trie task to completion, computing the state root.
|
||||
///
|
||||
/// This waits for new incoming [`SparseTrieUpdate`].
|
||||
///
|
||||
/// This concludes once the last trie update has been received.
|
||||
///
|
||||
/// # Returns
|
||||
///
|
||||
/// - State root computation outcome.
|
||||
/// - `SparseStateTrie` that needs to be cleared and reused to avoid reallocations.
|
||||
/// Receives [`SparseTrieUpdate`]s until the channel is closed, applying each update
|
||||
/// to the trie. Once all updates are processed, computes and returns the final state root.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
pub(super) fn run(
|
||||
mut self,
|
||||
) -> (Result<StateRootComputeOutcome, ParallelStateRootError>, SparseStateTrie<A, S>) {
|
||||
// run the main loop to completion
|
||||
let result = self.run_inner();
|
||||
(result, self.trie)
|
||||
}
|
||||
|
||||
/// Inner function to run the sparse trie task to completion.
|
||||
///
|
||||
/// See [`Self::run`] for more information.
|
||||
fn run_inner(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
let now = Instant::now();
|
||||
|
||||
let mut num_iterations = 0;
|
||||
@@ -127,6 +187,433 @@ where
|
||||
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
|
||||
/// Clears and shrinks the trie, discarding all state.
|
||||
///
|
||||
/// Use this when the payload was invalid or cancelled - we don't want to preserve
|
||||
/// potentially invalid trie state, but we keep the allocations for reuse.
|
||||
pub(super) fn into_cleared_trie(
|
||||
mut self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> SparseStateTrie<A, S> {
|
||||
self.trie.clear();
|
||||
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
self.trie
|
||||
}
|
||||
}
|
||||
|
||||
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
|
||||
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
|
||||
/// Sender for proof results.
|
||||
proof_result_tx: CrossbeamSender<ProofResultMessage>,
|
||||
/// Receiver for proof results directly from workers.
|
||||
proof_result_rx: CrossbeamReceiver<ProofResultMessage>,
|
||||
/// Receives updates from execution and prewarming.
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
/// `SparseStateTrie` used for computing the state root.
|
||||
trie: SparseStateTrie<A, S>,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
/// Account trie updates.
|
||||
account_updates: B256Map<LeafUpdate>,
|
||||
/// Storage trie updates. hashed address -> slot -> update.
|
||||
storage_updates: B256Map<B256Map<LeafUpdate>>,
|
||||
/// Account updates that are blocked by storage root calculation or account reveal.
|
||||
///
|
||||
/// Those are being moved into `account_updates` once storage roots
|
||||
/// are revealed and/or calculated.
|
||||
///
|
||||
/// Invariant: for each entry in `pending_account_updates` account must either be already
|
||||
/// revealed in the trie or have an entry in `account_updates`.
|
||||
///
|
||||
/// Values can be either of:
|
||||
/// - None: account had a storage update and is awaiting storage root calculation and/or
|
||||
/// account node reveal to complete.
|
||||
/// - Some(_): account was changed/destroyed and is awaiting storage root calculation/reveal
|
||||
/// to complete.
|
||||
pending_account_updates: B256Map<Option<Option<Account>>>,
|
||||
/// Cache of account proof targets that were already fetched/requested from the proof workers.
|
||||
/// account -> lowest `min_len` requested.
|
||||
fetched_account_targets: B256Map<u8>,
|
||||
/// Cache of storage proof targets that have already been fetched/requested from the proof
|
||||
/// workers. account -> slot -> lowest `min_len` requested.
|
||||
fetched_storage_targets: B256Map<B256Map<u8>>,
|
||||
/// Reusable buffer for RLP encoding of accounts.
|
||||
account_rlp_buf: Vec<u8>,
|
||||
/// Metrics for the sparse trie.
|
||||
metrics: MultiProofTaskMetrics,
|
||||
}
|
||||
|
||||
impl<A, S> SparseTrieCacheTask<A, S>
|
||||
where
|
||||
A: SparseTrieExt + Default,
|
||||
S: SparseTrieExt + Default + Clone,
|
||||
{
|
||||
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
|
||||
pub(super) fn new_with_trie(
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
sparse_state_trie: SparseStateTrie<A, S>,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
Self {
|
||||
proof_result_tx,
|
||||
proof_result_rx,
|
||||
updates,
|
||||
proof_worker_handle,
|
||||
trie: sparse_state_trie,
|
||||
account_updates: Default::default(),
|
||||
storage_updates: Default::default(),
|
||||
pending_account_updates: Default::default(),
|
||||
fetched_account_targets: Default::default(),
|
||||
fetched_storage_targets: Default::default(),
|
||||
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes and shrinks the trie for reuse in the next payload built on top of this one.
|
||||
///
|
||||
/// Should be called after the state root result has been sent.
|
||||
pub(super) fn into_trie_for_reuse(
|
||||
mut self,
|
||||
prune_depth: usize,
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> SparseStateTrie<A, S> {
|
||||
self.trie.prune(prune_depth, max_storage_tries);
|
||||
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
self.trie
|
||||
}
|
||||
|
||||
/// Clears and shrinks the trie, discarding all state.
|
||||
///
|
||||
/// Use this when the payload was invalid or cancelled - we don't want to preserve
|
||||
/// potentially invalid trie state, but we keep the allocations for reuse.
|
||||
pub(super) fn into_cleared_trie(
|
||||
mut self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> SparseStateTrie<A, S> {
|
||||
self.trie.clear();
|
||||
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
self.trie
|
||||
}
|
||||
|
||||
/// Runs the sparse trie task to completion.
|
||||
///
|
||||
/// This waits for new incoming [`MultiProofMessage`]s, applies updates to the trie and
|
||||
/// schedules proof fetching when needed.
|
||||
///
|
||||
/// This concludes once the last state update has been received and processed.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
let now = Instant::now();
|
||||
|
||||
let mut finished_state_updates = false;
|
||||
loop {
|
||||
crossbeam_channel::select_biased! {
|
||||
recv(self.proof_result_rx) -> message => {
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
self.on_proof_result(result)?;
|
||||
},
|
||||
recv(self.updates) -> message => {
|
||||
let update = match message {
|
||||
Ok(m) => m,
|
||||
Err(_) => {
|
||||
break
|
||||
}
|
||||
};
|
||||
|
||||
match update {
|
||||
MultiProofMessage::PrefetchProofs(targets) => {
|
||||
self.on_prewarm_targets(targets);
|
||||
}
|
||||
MultiProofMessage::StateUpdate(_, state) => {
|
||||
self.on_state_update(state);
|
||||
}
|
||||
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
|
||||
self.on_hashed_state_update(state);
|
||||
}
|
||||
MultiProofMessage::BlockAccessList(_) => todo!(),
|
||||
MultiProofMessage::FinishedStateUpdates => {
|
||||
finished_state_updates = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
self.process_updates()?;
|
||||
|
||||
if finished_state_updates &&
|
||||
self.account_updates.is_empty() &&
|
||||
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
debug!(target: "engine::root", "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
let (state_root, trie_updates) =
|
||||
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
|
||||
let end = Instant::now();
|
||||
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
|
||||
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
|
||||
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
|
||||
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
|
||||
let VersionedMultiProofTargets::V2(targets) = targets else {
|
||||
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
|
||||
};
|
||||
|
||||
for target in targets.account_targets {
|
||||
// Only touch accounts that are not yet present in the updates set.
|
||||
self.account_updates.entry(target.key()).or_insert(LeafUpdate::Touched);
|
||||
}
|
||||
|
||||
for (address, slots) in targets.storage_targets {
|
||||
for slot in slots {
|
||||
// Only touch storages that are not yet present in the updates set.
|
||||
self.storage_updates
|
||||
.entry(address)
|
||||
.or_default()
|
||||
.entry(slot.key())
|
||||
.or_insert(LeafUpdate::Touched);
|
||||
}
|
||||
|
||||
// Touch corresponding account leaf to make sure its revealed in accounts trie for
|
||||
// storage root update.
|
||||
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
|
||||
}
|
||||
}
|
||||
|
||||
/// Processes a state update and encodes all state changes as trie updates.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all,
|
||||
fields(accounts = update.len())
|
||||
)]
|
||||
fn on_state_update(&mut self, update: EvmState) {
|
||||
let hashed_state_update = evm_state_to_hashed_post_state(update);
|
||||
self.on_hashed_state_update(hashed_state_update)
|
||||
}
|
||||
|
||||
/// Processes a hashed state update and encodes all state changes as trie updates.
|
||||
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
|
||||
for (address, storage) in hashed_state_update.storages {
|
||||
for (slot, value) in storage.storage {
|
||||
let encoded = if value.is_zero() {
|
||||
Vec::new()
|
||||
} else {
|
||||
alloy_rlp::encode_fixed_size(&value).to_vec()
|
||||
};
|
||||
self.storage_updates
|
||||
.entry(address)
|
||||
.or_default()
|
||||
.insert(slot, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
|
||||
// Make sure account is tracked in `account_updates` so that it is revealed in accounts
|
||||
// trie for storage root update.
|
||||
self.account_updates.entry(address).or_insert(LeafUpdate::Touched);
|
||||
|
||||
// Make sure account is tracked in `pending_account_updates` so that once storage root
|
||||
// is computed, it will be updated in the accounts trie.
|
||||
self.pending_account_updates.entry(address).or_insert(None);
|
||||
}
|
||||
|
||||
for (address, account) in hashed_state_update.accounts {
|
||||
// Track account as touched.
|
||||
//
|
||||
// This might overwrite an existing update, which is fine, because storage root from it
|
||||
// is already tracked in the trie and can be easily fetched again.
|
||||
self.account_updates.insert(address, LeafUpdate::Touched);
|
||||
|
||||
// Track account in `pending_account_updates` so that once storage root is computed,
|
||||
// it will be updated in the accounts trie.
|
||||
self.pending_account_updates.insert(address, Some(account));
|
||||
}
|
||||
}
|
||||
|
||||
fn on_proof_result(
|
||||
&mut self,
|
||||
result: ProofResultMessage,
|
||||
) -> Result<(), ParallelStateRootError> {
|
||||
let ProofResult::V2(result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
|
||||
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
|
||||
fn process_updates(&mut self) -> Result<(), ProviderError> {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
|
||||
for (addr, updates) in &mut self.storage_updates {
|
||||
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
|
||||
let fetched_storage = self.fetched_storage_targets.entry(*addr).or_default();
|
||||
|
||||
trie.update_leaves(updates, |path, min_len| match fetched_storage.entry(path) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
|
||||
// If all storage updates were processed, we can now compute the new storage root.
|
||||
if updates.is_empty() {
|
||||
let storage_root =
|
||||
trie.root().expect("updates are drained, trie should be revealed by now");
|
||||
|
||||
// If there is a pending account update for this address with known info, we can
|
||||
// encode it into proper update right away.
|
||||
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
|
||||
entry.get().is_some()
|
||||
{
|
||||
let account = entry.remove().expect("just checked, should be Some");
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) &&
|
||||
storage_root == EMPTY_ROOT_HASH
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
self.account_rlp_buf.clear();
|
||||
account
|
||||
.unwrap_or_default()
|
||||
.into_trie_account(storage_root)
|
||||
.encode(&mut self.account_rlp_buf);
|
||||
self.account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
// Now handle pending account updates that can be upgraded to a proper update.
|
||||
let account_rlp_buf = &mut self.account_rlp_buf;
|
||||
self.pending_account_updates.retain(|addr, account| {
|
||||
// If account has pending storage updates, it is still pending.
|
||||
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get the current account state either from the trie or from latest account update.
|
||||
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
|
||||
Some(encoded).filter(|encoded| !encoded.is_empty())
|
||||
} else if !self.account_updates.contains_key(addr) {
|
||||
self.trie.get_account_value(addr)
|
||||
} else {
|
||||
// Needs to be revealed first
|
||||
return true;
|
||||
};
|
||||
|
||||
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
|
||||
|
||||
let (account, storage_root) = if let Some(account) = account.take() {
|
||||
// If account is Some(_) here it means it didn't have any storage updates
|
||||
// and we can fetch the storage root directly from the account trie.
|
||||
//
|
||||
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
|
||||
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
|
||||
|
||||
(account, storage_root)
|
||||
} else {
|
||||
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
|
||||
};
|
||||
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
|
||||
Vec::new()
|
||||
} else {
|
||||
account_rlp_buf.clear();
|
||||
account.unwrap_or_default().into_trie_account(storage_root).encode(account_rlp_buf);
|
||||
account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
|
||||
false
|
||||
});
|
||||
|
||||
let updates_len_before = self.account_updates.len();
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.trie
|
||||
.trie_mut()
|
||||
.update_leaves(&mut self.account_updates, |target, min_len| {
|
||||
match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
|
||||
if updates_len_before == self.account_updates.len() {
|
||||
// Only exit when no new updates are processed.
|
||||
//
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
|
||||
targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
0,
|
||||
HashedPostState::default(),
|
||||
Instant::now(),
|
||||
),
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of the state root computation, including the state root itself with
|
||||
@@ -150,8 +637,8 @@ where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrieInterface + Send + Sync + Default,
|
||||
S: SparseTrieInterface + Send + Sync + Default + Clone,
|
||||
A: SparseTrie + Send + Sync + Default,
|
||||
S: SparseTrie + Send + Sync + Default + Clone,
|
||||
{
|
||||
trace!(target: "engine::root::sparse", "Updating sparse trie");
|
||||
let started_at = Instant::now();
|
||||
@@ -178,7 +665,7 @@ where
|
||||
.storages
|
||||
.into_iter()
|
||||
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
|
||||
.par_bridge()
|
||||
.par_bridge_buffered()
|
||||
.map(|(address, storage, storage_trie)| {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)
|
||||
|
||||
@@ -402,7 +402,13 @@ where
|
||||
.in_scope(|| self.evm_env_for(&input))
|
||||
.map_err(NewPayloadError::other)?;
|
||||
|
||||
let env = ExecutionEnv { evm_env, hash: input.hash(), parent_hash: input.parent_hash() };
|
||||
let env = ExecutionEnv {
|
||||
evm_env,
|
||||
hash: input.hash(),
|
||||
parent_hash: input.parent_hash(),
|
||||
parent_state_root: parent_block.state_root(),
|
||||
transaction_count: input.transaction_count(),
|
||||
};
|
||||
|
||||
// Plan the strategy used for state root computation.
|
||||
let strategy = self.plan_state_root_computation();
|
||||
@@ -503,6 +509,7 @@ where
|
||||
|
||||
let root_time = Instant::now();
|
||||
let mut maybe_state_root = None;
|
||||
let mut state_root_task_failed = false;
|
||||
|
||||
match strategy {
|
||||
StateRootStrategy::StateRootTask => {
|
||||
@@ -513,6 +520,14 @@ where
|
||||
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
|
||||
// we double check the state root here for good measure
|
||||
if state_root == block.header().state_root() {
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.clone(),
|
||||
);
|
||||
}
|
||||
maybe_state_root = Some((state_root, trie_updates, elapsed))
|
||||
} else {
|
||||
warn!(
|
||||
@@ -521,10 +536,12 @@ where
|
||||
block_state_root = ?block.header().state_root(),
|
||||
"State root task returned incorrect state root"
|
||||
);
|
||||
state_root_task_failed = true;
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
debug!(target: "engine::tree::payload_validator", %error, "State root task failed");
|
||||
state_root_task_failed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -569,6 +586,11 @@ where
|
||||
self.compute_state_root_serial(overlay_factory.clone(), &hashed_state),
|
||||
block
|
||||
);
|
||||
|
||||
if state_root_task_failed {
|
||||
self.metrics.block_validation.state_root_task_fallback_success_total.increment(1);
|
||||
}
|
||||
|
||||
(root, updates, root_time.elapsed())
|
||||
};
|
||||
|
||||
@@ -784,6 +806,11 @@ where
|
||||
// Execute transactions
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
let mut transactions = transactions.into_iter();
|
||||
// Some executors may execute transactions that do not append receipts during the
|
||||
// main loop (e.g., system transactions whose receipts are added during finalization).
|
||||
// In that case, invoking the callback on every transaction would resend the previous
|
||||
// receipt with the same index and can panic the ordered root builder.
|
||||
let mut last_sent_len = 0usize;
|
||||
loop {
|
||||
// Measure time spent waiting for next transaction from iterator
|
||||
// (e.g., parallel signature recovery)
|
||||
@@ -810,10 +837,14 @@ where
|
||||
let gas_used = executor.execute_transaction(tx)?;
|
||||
self.metrics.record_transaction_execution(tx_start.elapsed());
|
||||
|
||||
// Send the latest receipt to the background task for incremental root computation
|
||||
if let Some(receipt) = executor.receipts().last() {
|
||||
let tx_index = executor.receipts().len() - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
let current_len = executor.receipts().len();
|
||||
if current_len > last_sent_len {
|
||||
last_sent_len = current_len;
|
||||
// Send the latest receipt to the background task for incremental root computation.
|
||||
if let Some(receipt) = executor.receipts().last() {
|
||||
let tx_index = current_len - 1;
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
}
|
||||
|
||||
enter.record("gas_used", gas_used);
|
||||
@@ -872,6 +903,62 @@ where
|
||||
.root_with_updates()?)
|
||||
}
|
||||
|
||||
/// Compares trie updates from the state root task with serial state root computation.
|
||||
///
|
||||
/// This is used for debugging and validating the correctness of the parallel state root
|
||||
/// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this
|
||||
/// method runs a separate serial state root computation and compares the resulting trie
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &HashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match self.compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
?serial_root,
|
||||
"Serial state root computation finished for comparison"
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
if let Err(err) = super::trie_updates::compare_trie_updates(
|
||||
&provider,
|
||||
task_trie_updates,
|
||||
serial_trie_updates,
|
||||
) {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Error comparing trie updates"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to get database provider for trie update comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to compute serial state root for comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the block after execution.
|
||||
///
|
||||
/// This performs:
|
||||
@@ -1095,10 +1182,13 @@ where
|
||||
/// while the trie input computation is deferred until the overlay is actually needed.
|
||||
///
|
||||
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
|
||||
///
|
||||
/// Uses a cached overlay if available for the canonical head (the common case).
|
||||
fn get_parent_lazy_overlay(
|
||||
parent_hash: B256,
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> (Option<LazyOverlay>, B256) {
|
||||
// Get blocks leading to the parent to determine the anchor
|
||||
let (anchor_hash, blocks) =
|
||||
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
|
||||
|
||||
@@ -1107,6 +1197,17 @@ where
|
||||
return (None, anchor_hash);
|
||||
}
|
||||
|
||||
// Try to use the cached overlay if it matches both parent hash and anchor
|
||||
if let Some(cached) = state.tree_state.get_cached_overlay(parent_hash, anchor_hash) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%parent_hash,
|
||||
%anchor_hash,
|
||||
"Using cached canonical overlay"
|
||||
);
|
||||
return (Some(cached.overlay.clone()), cached.anchor_hash);
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%anchor_hash,
|
||||
|
||||
@@ -21,7 +21,8 @@ impl<S> PrecompileCacheMap<S>
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
pub(crate) fn cache_for_address(&self, address: Address) -> PrecompileCache<S> {
|
||||
/// Get the precompile cache for the given address.
|
||||
pub fn cache_for_address(&self, address: Address) -> PrecompileCache<S> {
|
||||
// Try just using `.get` first to avoid acquiring a write lock.
|
||||
if let Some(cache) = self.0.get(&address) {
|
||||
return cache.clone();
|
||||
@@ -90,7 +91,7 @@ impl<S> CacheEntry<S> {
|
||||
|
||||
/// A cache for precompile inputs / outputs.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct CachedPrecompile<S>
|
||||
pub struct CachedPrecompile<S>
|
||||
where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
@@ -109,7 +110,7 @@ where
|
||||
S: Eq + Hash + std::fmt::Debug + Send + Sync + Clone + 'static,
|
||||
{
|
||||
/// `CachedPrecompile` constructor.
|
||||
pub(crate) const fn new(
|
||||
pub const fn new(
|
||||
precompile: DynPrecompile,
|
||||
cache: PrecompileCache<S>,
|
||||
spec_id: S,
|
||||
@@ -118,7 +119,8 @@ where
|
||||
Self { precompile, cache, spec_id, metrics }
|
||||
}
|
||||
|
||||
pub(crate) fn wrap(
|
||||
/// Wrap the given precompile in a cached precompile.
|
||||
pub fn wrap(
|
||||
precompile: DynPrecompile,
|
||||
cache: PrecompileCache<S>,
|
||||
spec_id: S,
|
||||
@@ -196,18 +198,18 @@ where
|
||||
/// Metrics for the cached precompile.
|
||||
#[derive(reth_metrics::Metrics, Clone)]
|
||||
#[metrics(scope = "sync.caching")]
|
||||
pub(crate) struct CachedPrecompileMetrics {
|
||||
pub struct CachedPrecompileMetrics {
|
||||
/// Precompile cache hits
|
||||
precompile_cache_hits: metrics::Counter,
|
||||
pub precompile_cache_hits: metrics::Counter,
|
||||
|
||||
/// Precompile cache misses
|
||||
precompile_cache_misses: metrics::Counter,
|
||||
pub precompile_cache_misses: metrics::Counter,
|
||||
|
||||
/// Precompile cache size. Uses the LRU cache length as the size metric.
|
||||
precompile_cache_size: metrics::Gauge,
|
||||
pub precompile_cache_size: metrics::Gauge,
|
||||
|
||||
/// Precompile execution errors.
|
||||
precompile_errors: metrics::Counter,
|
||||
pub precompile_errors: metrics::Counter,
|
||||
}
|
||||
|
||||
impl CachedPrecompileMetrics {
|
||||
@@ -215,7 +217,7 @@ impl CachedPrecompileMetrics {
|
||||
///
|
||||
/// Adds address as an `address` label padded with zeros to at least two hex symbols, prefixed
|
||||
/// by `0x`.
|
||||
pub(crate) fn new_with_address(address: Address) -> Self {
|
||||
pub fn new_with_address(address: Address) -> Self {
|
||||
Self::new_with_labels(&[("address", format!("0x{address:02x}"))])
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user