mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
1 Commits
mediocrego
...
klkvr/debu
| Author | SHA1 | Date | |
|---|---|---|---|
|
ae9042a5fb |
2
.github/scripts/bench-reth-build.sh
vendored
2
.github/scripts/bench-reth-build.sh
vendored
@@ -53,7 +53,7 @@ build_node_binary() {
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg
|
||||
cargo build --profile profiling $NODE_PKG $workspace_arg $features_arg
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
|
||||
25
.github/scripts/hive/build_simulators.sh
vendored
25
.github/scripts/hive/build_simulators.sh
vendored
@@ -1,23 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
fixture_variant="${1:-osaka}"
|
||||
|
||||
case "${fixture_variant}" in
|
||||
amsterdam)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.7.0/fixtures_bal.tar.gz"
|
||||
eels_branch="devnets/bal/4"
|
||||
;;
|
||||
osaka)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz"
|
||||
eels_branch="forks/osaka"
|
||||
;;
|
||||
*)
|
||||
echo "unknown hive fixture variant: ${fixture_variant}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Create the hive_assets directory
|
||||
mkdir hive_assets/
|
||||
|
||||
@@ -29,12 +12,12 @@ go build .
|
||||
# Run each hive command in the background for each simulator and wait
|
||||
echo "Building images"
|
||||
./hive -client reth --sim "ethereum/eels/consume-engine" \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/eels/consume-rlp" \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
|
||||
|
||||
6
.github/scripts/hive/run_simulator.sh
vendored
6
.github/scripts/hive/run_simulator.sh
vendored
@@ -5,12 +5,6 @@ cd hivetests/
|
||||
|
||||
sim="${1}"
|
||||
limit="${2}"
|
||||
fixture_variant="${3:-}"
|
||||
|
||||
if [[ "${fixture_variant}" == "osaka" && "${sim}" == *"eels"* && "${limit}" == *"tests/amsterdam"* ]]; then
|
||||
echo "osaka fixtures do not support amsterdam tests"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use lower parallelism for eels tests to avoid OOM-killing the runner
|
||||
parallelism=16
|
||||
|
||||
29
.github/workflows/bench-scheduled.yml
vendored
29
.github/workflows/bench-scheduled.yml
vendored
@@ -366,24 +366,19 @@ jobs:
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
|
||||
29
.github/workflows/bench.yml
vendored
29
.github/workflows/bench.yml
vendored
@@ -802,26 +802,21 @@ jobs:
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
|
||||
204
.github/workflows/hive.yml
vendored
204
.github/workflows/hive.yml
vendored
@@ -25,14 +25,7 @@ jobs:
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant:
|
||||
- amsterdam
|
||||
- osaka
|
||||
name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Checkout hive tests
|
||||
@@ -55,11 +48,11 @@ jobs:
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: ./hive_assets
|
||||
key: hive-assets-${{ matrix.variant }}-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
|
||||
- name: Build hive assets
|
||||
if: steps.cache-hive.outputs.cache-hit != 'true'
|
||||
run: .github/scripts/hive/build_simulators.sh ${{ matrix.variant }}
|
||||
run: .github/scripts/hive/build_simulators.sh
|
||||
|
||||
- name: Load cached Docker images
|
||||
if: steps.cache-hive.outputs.cache-hit == 'true'
|
||||
@@ -77,186 +70,9 @@ jobs:
|
||||
- name: Upload hive assets
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: hive_assets_${{ matrix.variant }}
|
||||
name: hive_assets
|
||||
path: ./hive_assets
|
||||
test-amsterdam:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# ethereum/rpc to be deprecated:
|
||||
# https://github.com/ethereum/hive/pull/1117
|
||||
scenario:
|
||||
- sim: smoke/genesis
|
||||
- sim: smoke/network
|
||||
- sim: ethereum/sync
|
||||
- sim: devp2p
|
||||
limit: discv4
|
||||
# started failing after https://github.com/ethereum/go-ethereum/pull/31843, no
|
||||
# action on our side, remove from here when we get unexpected passes on these tests
|
||||
# - sim: devp2p
|
||||
# limit: eth
|
||||
# include:
|
||||
# - MaliciousHandshake
|
||||
# # failures tracked in https://github.com/paradigmxyz/reth/issues/14825
|
||||
# - Status
|
||||
# - GetBlockHeaders
|
||||
# - ZeroRequestID
|
||||
# - GetBlockBodies
|
||||
# - Transaction
|
||||
# - NewPooledTxs
|
||||
- sim: devp2p
|
||||
limit: discv5
|
||||
include:
|
||||
# failures tracked at https://github.com/paradigmxyz/reth/issues/14825
|
||||
- PingLargeRequestID
|
||||
- sim: ethereum/engine
|
||||
limit: engine-exchange-capabilities
|
||||
- sim: ethereum/engine
|
||||
limit: engine-withdrawals
|
||||
- sim: ethereum/engine
|
||||
limit: engine-auth
|
||||
- sim: ethereum/engine
|
||||
limit: engine-api
|
||||
- sim: ethereum/engine
|
||||
limit: cancun
|
||||
# eth_ rpc methods
|
||||
- sim: ethereum/rpc-compat
|
||||
include:
|
||||
- eth_blockNumber
|
||||
- eth_call
|
||||
- eth_chainId
|
||||
- eth_createAccessList
|
||||
- eth_estimateGas
|
||||
- eth_feeHistory
|
||||
- eth_getBalance
|
||||
- eth_getBlockBy
|
||||
- eth_getBlockTransactionCountBy
|
||||
- eth_getCode
|
||||
- eth_getProof
|
||||
- eth_getStorage
|
||||
- eth_getTransactionBy
|
||||
- eth_getTransactionCount
|
||||
- eth_getTransactionReceipt
|
||||
- eth_sendRawTransaction
|
||||
- eth_syncing
|
||||
# debug_ rpc methods
|
||||
- debug_
|
||||
|
||||
# consume-engine
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/paris.*
|
||||
|
||||
# consume-rlp
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/paris.*
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: Hive-Amsterdam / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: hive_assets_amsterdam
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: reth
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
run: .github/scripts/hive/load_images.sh
|
||||
|
||||
- name: Move hive binary
|
||||
run: |
|
||||
mv /tmp/hive /usr/local/bin
|
||||
chmod +x /usr/local/bin/hive
|
||||
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
|
||||
- name: Run simulator
|
||||
run: |
|
||||
LIMIT="${{ matrix.scenario.limit }}"
|
||||
TESTS="${{ join(matrix.scenario.include, '|') }}"
|
||||
if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
|
||||
FILTER="$LIMIT/$TESTS"
|
||||
elif [ -n "$LIMIT" ]; then
|
||||
FILTER="$LIMIT"
|
||||
elif [ -n "$TESTS" ]; then
|
||||
FILTER="/$TESTS"
|
||||
else
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER" "amsterdam"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
|
||||
|
||||
- name: Print simulator output
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/*simulator*.log
|
||||
|
||||
- name: Print reth client logs
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
|
||||
test-osaka:
|
||||
test:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -362,7 +178,7 @@ jobs:
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: Hive-Osaka / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
name: ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
@@ -375,7 +191,7 @@ jobs:
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: hive_assets_osaka
|
||||
name: hive_assets
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
@@ -413,7 +229,7 @@ jobs:
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER" "osaka"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
@@ -429,9 +245,7 @@ jobs:
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
notify-on-error:
|
||||
needs:
|
||||
- test-amsterdam
|
||||
- test-osaka
|
||||
needs: test
|
||||
if: failure()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
29
Cargo.lock
generated
29
Cargo.lock
generated
@@ -2950,7 +2950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de"
|
||||
dependencies = [
|
||||
"data-encoding",
|
||||
"syn 1.0.109",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7712,9 +7712,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reth-codecs"
|
||||
version = "0.3.1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce542a96bf888f31854803e80b3340bc233927743aa580838014e8a88fe0d66"
|
||||
checksum = "a79b3247ae4fbb1d4d35ce83a11fc596428a4c6ea836c98a75a55340192578a4"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips 2.0.1",
|
||||
@@ -7733,9 +7733,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reth-codecs-derive"
|
||||
version = "0.3.1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "634c90f1cc0f9887680ca785b0b21aa961070b9465917bf65afaec56a6d005bb"
|
||||
checksum = "df5dbae40c272b8a1b4fcc08ee2d4e77d3b0ccdb187c1313f412a73ff54ff2a2"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -9418,9 +9418,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reth-primitives-traits"
|
||||
version = "0.3.1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ee12e304adbacbb32248c9806ebafbe1e2811fbfefe53c5e5b710a8438b7ec0"
|
||||
checksum = "cc759fd87c3f65440e5d3bfa3107fe8a13a61a6807cd485c62c49d63c7bf6717"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips 2.0.1",
|
||||
@@ -9963,9 +9963,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reth-rpc-traits"
|
||||
version = "0.3.1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "860fe223501a76ff14aa3bf164f739f31008c2a2905ac85708bfd88f042e6151"
|
||||
checksum = "b766da61ec7c46596386b4bc88d9b57d1939d3da2bc9e927567a8a23650e5ce9"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-network",
|
||||
@@ -10442,8 +10442,6 @@ dependencies = [
|
||||
"proptest-arbitrary-interop",
|
||||
"rand 0.9.4",
|
||||
"rayon",
|
||||
"reth-chainspec",
|
||||
"reth-ethereum-primitives",
|
||||
"reth-execution-errors",
|
||||
"reth-metrics",
|
||||
"reth-primitives-traits",
|
||||
@@ -10494,9 +10492,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "reth-zstd-compressors"
|
||||
version = "0.3.1"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c12fafa33d2f420a9d39249a3e0357b1928d09429f30758b85280409092873b2"
|
||||
checksum = "82fa54c341d926ec9b0f7fc0c87f831aa4959de699e69caab1a0bfd914326c09"
|
||||
dependencies = [
|
||||
"zstd",
|
||||
]
|
||||
@@ -10676,7 +10674,6 @@ dependencies = [
|
||||
"ark-serialize 0.5.0",
|
||||
"arrayref",
|
||||
"aurora-engine-modexp",
|
||||
"aws-lc-rs",
|
||||
"blst",
|
||||
"c-kzg",
|
||||
"cfg-if",
|
||||
@@ -10793,9 +10790,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "roaring"
|
||||
version = "0.11.4"
|
||||
version = "0.11.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1dedc5658c6ecb3bdb5ef5f3295bb9253f42dcf3fd1402c03f6b1f7659c3c4a9"
|
||||
checksum = "8ba9ce64a8f45d7fc86358410bb1a82e8c987504c0d4900e9141d69a9f26c885"
|
||||
dependencies = [
|
||||
"bytemuck",
|
||||
"byteorder",
|
||||
|
||||
10
Cargo.toml
10
Cargo.toml
@@ -326,8 +326,8 @@ reth-cli = { path = "crates/cli/cli" }
|
||||
reth-cli-commands = { path = "crates/cli/commands" }
|
||||
reth-cli-runner = { path = "crates/cli/runner" }
|
||||
reth-cli-util = { path = "crates/cli/util" }
|
||||
reth-codecs = { version = "0.3.1", default-features = false }
|
||||
reth-codecs-derive = "0.3.1"
|
||||
reth-codecs = { version = "0.3.0", default-features = false }
|
||||
reth-codecs-derive = "0.3.0"
|
||||
reth-config = { path = "crates/config", default-features = false }
|
||||
reth-consensus = { path = "crates/consensus/consensus", default-features = false }
|
||||
reth-consensus-common = { path = "crates/consensus/common", default-features = false }
|
||||
@@ -395,7 +395,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives-traits = { version = "0.3.1", default-features = false }
|
||||
reth-primitives-traits = { version = "0.3.0", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
reth-prune-types = { path = "crates/prune/types", default-features = false }
|
||||
@@ -411,7 +411,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
|
||||
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
|
||||
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
|
||||
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
|
||||
reth-rpc-traits = { version = "0.3.1", default-features = false }
|
||||
reth-rpc-traits = { version = "0.3.0", default-features = false }
|
||||
reth-stages = { path = "crates/stages/stages" }
|
||||
reth-stages-api = { path = "crates/stages/api" }
|
||||
reth-stages-types = { path = "crates/stages/types", default-features = false }
|
||||
@@ -430,7 +430,7 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
|
||||
reth-trie-db = { path = "crates/trie/db" }
|
||||
reth-trie-parallel = { path = "crates/trie/parallel" }
|
||||
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.1", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.0", default-features = false }
|
||||
|
||||
# revm
|
||||
revm = { version = "38.0.0", default-features = false }
|
||||
|
||||
11
Dockerfile
11
Dockerfile
@@ -33,17 +33,8 @@ ENV FEATURES=$FEATURES
|
||||
RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json
|
||||
|
||||
# Build application
|
||||
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
|
||||
#
|
||||
# TARGETPLATFORM is set by BuildKit: https://docs.docker.com/reference/dockerfile#automatic-platform-args-in-the-global-scope
|
||||
ARG TARGETPLATFORM
|
||||
COPY --exclude=dist . .
|
||||
RUN if [ -n "$RUSTFLAGS" ]; then \
|
||||
export RUSTFLAGS="$RUSTFLAGS"; \
|
||||
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
|
||||
fi && \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
|
||||
# ARG is not resolved in COPY so we have to hack around it by copying the
|
||||
# binary to a temporary location
|
||||
|
||||
@@ -69,7 +69,6 @@ default = [
|
||||
"jemalloc",
|
||||
"reth-cli-util/jemalloc",
|
||||
"asm-keccak",
|
||||
"keccak-cache-global",
|
||||
"min-debug-logs",
|
||||
]
|
||||
|
||||
@@ -90,12 +89,6 @@ asm-keccak = [
|
||||
"revm-primitives/asm-keccak",
|
||||
]
|
||||
|
||||
keccak-cache-global = [
|
||||
"reth-node-core/keccak-cache-global",
|
||||
"reth-node-ethereum/keccak-cache-global",
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
]
|
||||
|
||||
min-debug-logs = [
|
||||
"tracing/release_max_level_debug",
|
||||
"reth-ethereum-cli/min-debug-logs",
|
||||
|
||||
@@ -39,7 +39,7 @@ Both `new-payload-fcu` and `new-payload-only` support `--rpc-block-fetch-retries
|
||||
to control how many times block fetches are retried after an RPC failure. The default is `10`.
|
||||
Use `--rpc-block-fetch-retries forever` to keep retrying indefinitely.
|
||||
|
||||
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold. This can be customized with `--persistence-threshold <N>`.
|
||||
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
|
||||
|
||||
By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) auth_provider: RootProvider<AnyNetwork>,
|
||||
/// The block provider is used for block queries.
|
||||
pub(crate) block_provider: RootProvider<AnyNetwork>,
|
||||
/// The local regular RPC provider is used for non-authenticated node RPCs like `testing_*`.
|
||||
pub(crate) local_rpc_provider: RootProvider<AnyNetwork>,
|
||||
/// The benchmark mode, which defines whether the benchmark should run for a closed or open
|
||||
/// range of blocks.
|
||||
pub(crate) benchmark_mode: BenchMode,
|
||||
@@ -85,11 +83,6 @@ impl BenchContext {
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
let local_rpc_url = Url::parse(&bench_args.local_rpc_url)?;
|
||||
info!(target: "reth-bench", "Connecting to local regular RPC at {} for testing namespace calls", local_rpc_url);
|
||||
let local_rpc_provider =
|
||||
RootProvider::<AnyNetwork>::new(ClientBuilder::default().http(local_rpc_url));
|
||||
|
||||
// Computes the block range for the benchmark.
|
||||
//
|
||||
// - If `--advance` is provided, fetches the latest block from the engine and sets:
|
||||
@@ -166,7 +159,6 @@ impl BenchContext {
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
local_rpc_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
|
||||
@@ -14,24 +14,14 @@ use crate::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
},
|
||||
};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{
|
||||
ext::DebugApi,
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
Provider, RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionData, ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes,
|
||||
};
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use clap::Parser;
|
||||
use eyre::{bail, ensure, Context, OptionExt};
|
||||
use eyre::{Context, OptionExt};
|
||||
use futures::{stream, StreamExt, TryStreamExt};
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use reth_rpc_api::{RethNewPayloadInput, TestingBuildBlockRequestV1};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -42,22 +32,6 @@ pub struct Command {
|
||||
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
|
||||
rpc_url: String,
|
||||
|
||||
/// Build a separate fork with `testing_buildBlockV1` and alternate forkchoice updates between
|
||||
/// the canonical chain and that fork on every block while the fork grows up to the configured
|
||||
/// depth.
|
||||
///
|
||||
/// This requires enabling the hidden `testing` RPC module on the target node,
|
||||
/// for example with `reth node --http --http.api eth,testing`.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "DEPTH",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "8",
|
||||
value_parser = parse_reorg_depth,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
reorg: Option<usize>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
///
|
||||
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
|
||||
@@ -67,8 +41,9 @@ pub struct Command {
|
||||
|
||||
/// Engine persistence threshold used for deciding when to wait for persistence.
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks.
|
||||
/// By default this matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD`.
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
|
||||
/// at blocks 3, 6, 9, etc.
|
||||
#[arg(
|
||||
long = "persistence-threshold",
|
||||
value_name = "PERSISTENCE_THRESHOLD",
|
||||
@@ -108,79 +83,18 @@ pub struct Command {
|
||||
benchmark: BenchmarkArgs,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PreparedBuiltBlock {
|
||||
block_hash: B256,
|
||||
params: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QueuedForkBlock {
|
||||
block_number: u64,
|
||||
prepared: PreparedBuiltBlock,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ReorgState {
|
||||
depth: usize,
|
||||
fork_length: usize,
|
||||
branch_point_hash: Option<B256>,
|
||||
fork_parent_hash: Option<B256>,
|
||||
}
|
||||
|
||||
impl ReorgState {
|
||||
const fn new(depth: usize) -> Self {
|
||||
Self { depth, fork_length: 0, branch_point_hash: None, fork_parent_hash: None }
|
||||
}
|
||||
|
||||
const fn push_fork_head(&mut self, canonical_parent_hash: B256, fork_head_hash: B256) {
|
||||
if self.fork_length == 0 {
|
||||
self.branch_point_hash = Some(canonical_parent_hash);
|
||||
}
|
||||
self.fork_length += 1;
|
||||
self.fork_parent_hash = Some(fork_head_hash);
|
||||
}
|
||||
|
||||
fn forkchoice_state(&self, fork_head_hash: B256) -> eyre::Result<ForkchoiceState> {
|
||||
let branch_point_hash = self.branch_point_hash.ok_or_eyre("missing reorg branch point")?;
|
||||
|
||||
Ok(ForkchoiceState {
|
||||
head_block_hash: fork_head_hash,
|
||||
safe_block_hash: branch_point_hash,
|
||||
finalized_block_hash: branch_point_hash,
|
||||
})
|
||||
}
|
||||
|
||||
const fn reset(&mut self) {
|
||||
self.fork_length = 0;
|
||||
self.branch_point_hash = None;
|
||||
self.fork_parent_hash = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark new-payload-fcu` command
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.reorg.is_some() && self.benchmark.rlp_blocks {
|
||||
bail!("--reorg cannot be combined with --rlp-blocks")
|
||||
}
|
||||
if self.reorg.is_some() && self.enable_bal {
|
||||
bail!("--reorg cannot be combined with --enable-bal")
|
||||
}
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms minimum interval between blocks", duration.as_millis());
|
||||
}
|
||||
if let Some(depth) = self.reorg {
|
||||
info!(target: "reth-bench", depth, "Using testing_buildBlockV1 reorg mode");
|
||||
}
|
||||
|
||||
let BenchContext {
|
||||
benchmark_mode,
|
||||
block_provider,
|
||||
auth_provider,
|
||||
local_rpc_provider,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
@@ -268,8 +182,7 @@ impl Command {
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
let mut reorg_state = self.reorg.map(ReorgState::new);
|
||||
let mut queued_fork_block = None;
|
||||
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = blocks.try_next().await?;
|
||||
@@ -279,13 +192,11 @@ impl Command {
|
||||
let gas_used = block.header.gas_used;
|
||||
let gas_limit = block.header.gas_limit;
|
||||
let block_number = block.header.number;
|
||||
let canonical_parent_hash = block.header.parent_hash;
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
let deferred_branch_start_block = reorg_state
|
||||
.as_ref()
|
||||
.filter(|state| state.fork_length == 0 && queued_fork_block.is_none())
|
||||
.map(|_| block.clone());
|
||||
let canonical_forkchoice_state = ForkchoiceState {
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: head,
|
||||
safe_block_hash: safe,
|
||||
finalized_block_hash: finalized,
|
||||
@@ -307,11 +218,10 @@ impl Command {
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
@@ -332,12 +242,17 @@ impl Command {
|
||||
};
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, canonical_forkchoice_state)
|
||||
.await?;
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
@@ -347,88 +262,6 @@ impl Command {
|
||||
total_latency,
|
||||
};
|
||||
|
||||
if let Some(reorg_state) = reorg_state.as_mut() {
|
||||
if queued_fork_block.is_none() && reorg_state.fork_length == 0 {
|
||||
// A branch start uses a canonical parent, so it can be built lazily here
|
||||
// instead of being queued ahead of time.
|
||||
let block = deferred_branch_start_block
|
||||
.as_ref()
|
||||
.ok_or_eyre("missing deferred fork block for reorg branch start")?;
|
||||
queued_fork_block = Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
&local_rpc_provider,
|
||||
block,
|
||||
canonical_parent_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
});
|
||||
}
|
||||
|
||||
let queued = queued_fork_block
|
||||
.take()
|
||||
.ok_or_eyre("missing queued fork block for reorg replay")?;
|
||||
ensure!(
|
||||
queued.block_number == block_number,
|
||||
"queued fork block {} does not match source block {}",
|
||||
queued.block_number,
|
||||
block_number
|
||||
);
|
||||
let prepared = queued.prepared;
|
||||
|
||||
call_new_payload_with_reth(&auth_provider, None, prepared.params).await?;
|
||||
|
||||
reorg_state.push_fork_head(canonical_parent_hash, prepared.block_hash);
|
||||
let forkchoice_state = reorg_state.forkchoice_state(prepared.block_hash)?;
|
||||
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
branch_point = %forkchoice_state.safe_block_hash,
|
||||
fork_head = %prepared.block_hash,
|
||||
fork_depth = reorg_state.fork_length,
|
||||
max_reorg_depth = reorg_state.depth,
|
||||
"Switching forkchoice to reorg branch"
|
||||
);
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, None, forkchoice_state).await?;
|
||||
let _fork_fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let next_fork_block_number = block_number + 1;
|
||||
if reorg_state.fork_length < reorg_state.depth {
|
||||
queued_fork_block = queue_fork_block(
|
||||
&block_provider,
|
||||
&local_rpc_provider,
|
||||
&benchmark_mode,
|
||||
next_fork_block_number,
|
||||
Some(prepared.block_hash),
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
reorg_depth = reorg_state.depth,
|
||||
"Resetting reorg branch after reaching max depth"
|
||||
);
|
||||
|
||||
// `testing_buildBlockV1` resolves the parent from canonical state, so switch
|
||||
// back to the source chain before reseeding the next queued fork block.
|
||||
call_forkchoice_updated_with_reth(
|
||||
&auth_provider,
|
||||
version,
|
||||
canonical_forkchoice_state,
|
||||
)
|
||||
.await?;
|
||||
|
||||
reorg_state.reset();
|
||||
queued_fork_block = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
@@ -485,155 +318,3 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_built_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<PreparedBuiltBlock> {
|
||||
const MAX_BUILD_ATTEMPTS: usize = 10;
|
||||
const BUILD_RETRY_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
let request = build_block_request(block, parent_block_hash)?;
|
||||
let built_payload: ExecutionPayloadEnvelopeV5 = {
|
||||
let mut attempts_remaining = MAX_BUILD_ATTEMPTS;
|
||||
|
||||
loop {
|
||||
match block_provider.client().request("testing_buildBlockV1", [request.clone()]).await {
|
||||
Ok(payload) => break payload,
|
||||
Err(err) if attempts_remaining > 1 && is_retryable_build_block_error(&err) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number = block.header.number,
|
||||
%parent_block_hash,
|
||||
attempts_remaining,
|
||||
error = %err,
|
||||
"Retrying testing_buildBlockV1 after transient fork build failure"
|
||||
);
|
||||
attempts_remaining -= 1;
|
||||
tokio::time::sleep(BUILD_RETRY_INTERVAL).await;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(err).wrap_err_with(|| {
|
||||
format!(
|
||||
"Failed to build block {} via testing_buildBlockV1",
|
||||
block.header.number
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let payload = &built_payload.execution_payload.payload_inner.payload_inner;
|
||||
let block_hash = payload.block_hash;
|
||||
let (payload, sidecar) = built_payload
|
||||
.into_payload_and_sidecar(block.header.parent_beacon_block_root.unwrap_or_default());
|
||||
// Fork payloads are built immediately before the next `testing_buildBlockV1` call. Leaving
|
||||
// reth's default persistence wait enabled here gives the regular RPC side a consistent base
|
||||
// state for the next synthetic fork block build.
|
||||
let params = serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(ExecutionData { payload, sidecar }),
|
||||
None::<bool>,
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?;
|
||||
|
||||
Ok(PreparedBuiltBlock { block_hash, params })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn queue_fork_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
local_rpc_provider: &RootProvider<AnyNetwork>,
|
||||
benchmark_mode: &crate::bench_mode::BenchMode,
|
||||
block_number: u64,
|
||||
parent_block_hash: Option<B256>,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<Option<QueuedForkBlock>> {
|
||||
if !benchmark_mode.contains(block_number) {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
let future_block = block_provider
|
||||
.get_block_by_number(alloy_eips::BlockNumberOrTag::Number(block_number))
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch block by number {block_number}"))?
|
||||
.ok_or_eyre("Block not found")?;
|
||||
let parent_block_hash = parent_block_hash.unwrap_or(future_block.header.parent_hash);
|
||||
|
||||
Ok(Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
local_rpc_provider,
|
||||
&future_block,
|
||||
parent_block_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
}))
|
||||
}
|
||||
|
||||
fn is_retryable_build_block_error(err: &alloy_transport::TransportError) -> bool {
|
||||
let message = err.to_string();
|
||||
message.contains("block not found: hash") ||
|
||||
message.contains("block hash not found for block number")
|
||||
}
|
||||
|
||||
fn build_block_request(
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
) -> eyre::Result<TestingBuildBlockRequestV1> {
|
||||
let mut transactions = block
|
||||
.clone()
|
||||
.try_into_transactions()
|
||||
.map_err(|_| eyre::eyre!("Block transactions must be fetched in full for --reorg"))?
|
||||
.into_iter()
|
||||
.map(|tx| {
|
||||
let tx: TxEnvelope =
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type in RPC block"))?;
|
||||
if tx.is_eip4844() {
|
||||
return Ok(None)
|
||||
}
|
||||
Ok(Some(tx.encoded_2718().into()))
|
||||
})
|
||||
.filter_map(|tx| tx.transpose())
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
// `testing_buildBlockV1` only takes raw transaction bytes, so we exclude blob transactions
|
||||
// from the synthetic fork blocks rather than trying to reconstruct their sidecars.
|
||||
// Keep only 90% of the remaining transactions so the alternate branch produces a materially
|
||||
// different post-state instead of only differing by header data.
|
||||
let keep = transactions.len().saturating_mul(9) / 10;
|
||||
transactions.truncate(keep);
|
||||
|
||||
let rpc_block = block.clone().into_inner();
|
||||
|
||||
Ok(TestingBuildBlockRequestV1 {
|
||||
parent_block_hash,
|
||||
payload_attributes: PayloadAttributes {
|
||||
timestamp: block.header.timestamp,
|
||||
prev_randao: block.header.mix_hash.unwrap_or_default(),
|
||||
suggested_fee_recipient: block.header.beneficiary,
|
||||
withdrawals: rpc_block.withdrawals.map(|withdrawals| withdrawals.into_inner()),
|
||||
parent_beacon_block_root: block.header.parent_beacon_block_root,
|
||||
slot_number: block.header.slot_number,
|
||||
},
|
||||
transactions,
|
||||
extra_data: Some(block.header.extra_data.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_reorg_depth(value: &str) -> Result<usize, String> {
|
||||
let depth = value
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("invalid reorg depth {value:?}, expected a positive integer"))?;
|
||||
|
||||
if depth == 0 {
|
||||
return Err("reorg depth must be greater than 0".to_string())
|
||||
}
|
||||
|
||||
Ok(depth)
|
||||
}
|
||||
|
||||
@@ -54,7 +54,6 @@ impl Command {
|
||||
rlp_blocks,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
@@ -18,7 +18,7 @@ reth-errors.workspace = true
|
||||
reth-execution-types.workspace = true
|
||||
reth-metrics.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["dashmap"] }
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie.workspace = true
|
||||
|
||||
|
||||
@@ -320,19 +320,6 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
|
||||
/// This will update the links between blocks and remove all blocks that are [..
|
||||
/// `persisted_height`].
|
||||
pub fn remove_persisted_blocks(&self, persisted_num_hash: BlockNumHash) {
|
||||
self.remove_persisted_blocks_until(persisted_num_hash, persisted_num_hash.number);
|
||||
}
|
||||
|
||||
/// Removes blocks from the in-memory state through `remove_until` while still reporting the
|
||||
/// provided block as the persisted tip.
|
||||
///
|
||||
/// This is used when block bodies/plain state have been persisted further than trie data, so a
|
||||
/// suffix still needs to remain in memory for trie-backed operations.
|
||||
pub fn remove_persisted_blocks_until(
|
||||
&self,
|
||||
persisted_num_hash: BlockNumHash,
|
||||
remove_until: BlockNumber,
|
||||
) {
|
||||
self.set_persisted(persisted_num_hash);
|
||||
// if the persisted hash is not in the canonical in memory state, do nothing, because it
|
||||
// means canonical blocks were not actually persisted.
|
||||
@@ -350,15 +337,16 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
|
||||
let mut numbers = self.inner.in_memory_state.numbers.write();
|
||||
let mut blocks = self.inner.in_memory_state.blocks.write();
|
||||
|
||||
let remove_until = remove_until.min(persisted_num_hash.number);
|
||||
let BlockNumHash { number: persisted_height, hash: _ } = persisted_num_hash;
|
||||
|
||||
// clear all numbers
|
||||
numbers.clear();
|
||||
|
||||
// Drain all blocks and keep only the suffix that still has to stay in memory.
|
||||
// drain all blocks and only keep the ones that are not persisted (below the persisted
|
||||
// height)
|
||||
let mut old_blocks = blocks
|
||||
.drain()
|
||||
.filter(|(_, b)| b.block_ref().recovered_block().number() > remove_until)
|
||||
.filter(|(_, b)| b.block_ref().recovered_block().number() > persisted_height)
|
||||
.map(|(_, b)| b.block.clone())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
|
||||
@@ -4,32 +4,26 @@
|
||||
//! lazily on first access. This allows execution to start before the trie overlay
|
||||
//! is fully computed.
|
||||
|
||||
use crate::{EthPrimitives, ExecutedBlock};
|
||||
use crate::DeferredTrieData;
|
||||
use alloy_primitives::B256;
|
||||
use reth_primitives_traits::{
|
||||
dashmap::{self, DashMap},
|
||||
AlloyBlockHeader, NodePrimitives,
|
||||
};
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Inputs captured for lazy overlay computation.
|
||||
#[derive(Clone)]
|
||||
struct LazyOverlayInputs<N: NodePrimitives = EthPrimitives> {
|
||||
/// In-memory blocks from tip to anchor child.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest).
|
||||
blocks: Vec<ExecutedBlock<N>>,
|
||||
struct LazyOverlayInputs {
|
||||
/// The persisted ancestor hash (anchor) this overlay should be built on.
|
||||
anchor_hash: B256,
|
||||
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
|
||||
blocks: Vec<DeferredTrieData>,
|
||||
}
|
||||
|
||||
/// Lazily computed trie overlay.
|
||||
///
|
||||
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
|
||||
/// computation until first access.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest), so the first block is the
|
||||
/// chain tip and the last block is the oldest in-memory block in the chain segment.
|
||||
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
|
||||
/// but for overlay computation.
|
||||
///
|
||||
/// # Fast Path vs Slow Path
|
||||
///
|
||||
@@ -37,41 +31,37 @@ struct LazyOverlayInputs<N: NodePrimitives = EthPrimitives> {
|
||||
/// matches our expected anchor, we can reuse it directly (O(1)).
|
||||
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
|
||||
#[derive(Clone)]
|
||||
pub struct LazyOverlay<N: NodePrimitives = EthPrimitives> {
|
||||
/// Computed results, cached by requested anchor hash.
|
||||
inner: Arc<DashMap<B256, Arc<TrieInputSorted>>>,
|
||||
pub struct LazyOverlay {
|
||||
/// Computed result, cached after first access.
|
||||
inner: Arc<OnceLock<TrieInputSorted>>,
|
||||
/// Inputs for lazy computation.
|
||||
inputs: LazyOverlayInputs<N>,
|
||||
inputs: LazyOverlayInputs,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> std::fmt::Debug for LazyOverlay<N> {
|
||||
impl std::fmt::Debug for LazyOverlay {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LazyOverlay")
|
||||
.field(
|
||||
"oldest_block_parent_hash",
|
||||
&self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash()),
|
||||
)
|
||||
.field("anchor_hash", &self.inputs.anchor_hash)
|
||||
.field("num_blocks", &self.inputs.blocks.len())
|
||||
.field("cached_anchors", &self.inner.len())
|
||||
.field("computed", &self.inner.get().is_some())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
/// Create a new lazy overlay from in-memory blocks.
|
||||
impl LazyOverlay {
|
||||
/// Create a new lazy overlay with the given anchor hash and block handles.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `blocks` - Executed blocks in reverse chain order (newest to oldest)
|
||||
pub fn new(blocks: Vec<ExecutedBlock<N>>) -> Self {
|
||||
debug_assert!(
|
||||
blocks.windows(2).all(|window| {
|
||||
window[0].recovered_block().parent_hash() == window[1].recovered_block().hash()
|
||||
}),
|
||||
"LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
);
|
||||
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
|
||||
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
|
||||
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
|
||||
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
|
||||
}
|
||||
|
||||
Self { inner: Default::default(), inputs: LazyOverlayInputs { blocks } }
|
||||
/// Returns the anchor hash this overlay is built on.
|
||||
pub const fn anchor_hash(&self) -> B256 {
|
||||
self.inputs.anchor_hash
|
||||
}
|
||||
|
||||
/// Returns the number of in-memory blocks this overlay covers.
|
||||
@@ -79,75 +69,43 @@ impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
self.inputs.blocks.len()
|
||||
}
|
||||
|
||||
/// Returns the oldest anchor hash this overlay can serve.
|
||||
///
|
||||
/// This is the parent hash of the oldest block in the stored newest-to-oldest chain segment.
|
||||
pub fn anchor_hash(&self) -> Option<B256> {
|
||||
self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash())
|
||||
/// Returns true if the overlay has already been computed.
|
||||
pub fn is_computed(&self) -> bool {
|
||||
self.inner.get().is_some()
|
||||
}
|
||||
|
||||
/// Returns true if there are no blocks in the overlay, or if one of the blocks has the given
|
||||
/// hash as a parent hash.
|
||||
pub fn has_anchor_hash(&self, hash: B256) -> bool {
|
||||
self.inputs.blocks.is_empty() ||
|
||||
self.inputs.blocks.iter().any(|b| b.recovered_block().parent_hash() == hash)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Returns true if the overlay has already been computed for the requested anchor.
|
||||
pub fn is_computed(&self, anchor_hash: B256) -> bool {
|
||||
self.inner.contains_key(&anchor_hash)
|
||||
}
|
||||
|
||||
/// Returns the computed trie input for the requested anchor, computing it if necessary.
|
||||
/// Returns the computed trie input, computing it if necessary.
|
||||
///
|
||||
/// The first call triggers computation (which may block waiting for deferred data).
|
||||
/// Subsequent calls for the same anchor return the cached result immediately.
|
||||
pub fn get(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
match self.inner.entry(anchor_hash) {
|
||||
dashmap::Entry::Occupied(entry) => Arc::clone(entry.get()),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
let input = self.compute(anchor_hash);
|
||||
entry.insert(Arc::clone(&input));
|
||||
input
|
||||
}
|
||||
}
|
||||
/// Subsequent calls return the cached result immediately.
|
||||
pub fn get(&self) -> &TrieInputSorted {
|
||||
self.inner.get_or_init(|| self.compute())
|
||||
}
|
||||
|
||||
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
|
||||
pub fn as_overlay(
|
||||
&self,
|
||||
anchor_hash: B256,
|
||||
) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get(anchor_hash);
|
||||
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get();
|
||||
(Arc::clone(&input.nodes), Arc::clone(&input.state))
|
||||
}
|
||||
|
||||
/// Compute the trie input overlay.
|
||||
fn compute(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
fn compute(&self) -> TrieInputSorted {
|
||||
let anchor_hash = self.inputs.anchor_hash;
|
||||
let blocks = &self.inputs.blocks;
|
||||
|
||||
if blocks.is_empty() {
|
||||
return Default::default()
|
||||
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let Some(last_index) =
|
||||
blocks.iter().position(|block| block.recovered_block().parent_hash() == anchor_hash)
|
||||
else {
|
||||
panic!(
|
||||
"LazyOverlay does not contain a block whose parent hash matches requested anchor {anchor_hash}"
|
||||
);
|
||||
};
|
||||
let blocks = &blocks[..=last_index];
|
||||
|
||||
// Fast path: Check if tip block's overlay is ready and anchor matches.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors up to the
|
||||
// requested anchor.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors.
|
||||
if let Some(tip) = blocks.first() {
|
||||
let data = tip.trie_data();
|
||||
let data = tip.wait_cloned();
|
||||
if let Some(anchored) = &data.anchored_trie_input {
|
||||
if anchored.anchor_hash == anchor_hash {
|
||||
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
|
||||
return Arc::clone(&anchored.trie_input);
|
||||
return (*anchored.trie_input).clone();
|
||||
}
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
@@ -158,30 +116,23 @@ impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: Merge the prefix of blocks from the tip back to the requested anchor.
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
%anchor_hash,
|
||||
num_blocks = blocks.len(),
|
||||
"Merging blocks (slow path)"
|
||||
);
|
||||
Arc::new(Self::merge_blocks(blocks))
|
||||
// Slow path: Merge all blocks' trie data into a new overlay.
|
||||
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
|
||||
Self::merge_blocks(blocks)
|
||||
}
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let state = HashedPostStateSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().hashed_state),
|
||||
);
|
||||
let nodes = TrieUpdatesSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().trie_updates),
|
||||
);
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
@@ -190,138 +141,46 @@ impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{test_utils::TestBlockBuilder, ComputedTrieData, EthPrimitives, ExecutedBlock};
|
||||
use alloy_primitives::U256;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostState, HashedStorage};
|
||||
use std::sync::Arc;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState};
|
||||
|
||||
fn with_unique_state(
|
||||
block: &ExecutedBlock<EthPrimitives>,
|
||||
id: u8,
|
||||
) -> ExecutedBlock<EthPrimitives> {
|
||||
let hashed_address = B256::with_last_byte(id);
|
||||
let hashed_slot = B256::with_last_byte(id.saturating_add(32));
|
||||
let hashed_state = HashedPostState::default()
|
||||
.with_accounts([(hashed_address, Some(Account::default()))])
|
||||
.with_storages([(
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]),
|
||||
)])
|
||||
.into_sorted();
|
||||
|
||||
ExecutedBlock::new(
|
||||
Arc::clone(&block.recovered_block),
|
||||
Arc::clone(&block.execution_output),
|
||||
ComputedTrieData::without_trie_input(
|
||||
Arc::new(hashed_state),
|
||||
Arc::new(TrieUpdatesSorted::default()),
|
||||
),
|
||||
fn empty_deferred(anchor: B256) -> DeferredTrieData {
|
||||
DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
Vec::new(),
|
||||
)
|
||||
}
|
||||
|
||||
fn test_blocks() -> Vec<ExecutedBlock<EthPrimitives>> {
|
||||
TestBlockBuilder::eth()
|
||||
.get_executed_blocks(1..4)
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.enumerate()
|
||||
.map(|(index, block)| with_unique_state(&block, index as u8 + 1))
|
||||
.collect()
|
||||
#[test]
|
||||
fn empty_blocks_returns_default() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
let result = overlay.get();
|
||||
assert!(result.state.is_empty());
|
||||
assert!(result.nodes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_uses_data_directly() {
|
||||
let block = TestBlockBuilder::eth().get_executed_block_with_number(1, B256::random());
|
||||
let anchor_hash = block.recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(vec![block]);
|
||||
let anchor = B256::random();
|
||||
let deferred = empty_deferred(anchor);
|
||||
let overlay = LazyOverlay::new(anchor, vec![deferred]);
|
||||
|
||||
assert!(!overlay.is_computed(anchor_hash));
|
||||
let _ = overlay.get(anchor_hash);
|
||||
assert!(overlay.is_computed(anchor_hash));
|
||||
assert!(!overlay.is_computed());
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn caches_results_per_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let full_anchor = blocks[2].recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
fn cached_after_first_access() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
|
||||
let prefix = overlay.get(prefix_anchor);
|
||||
let full = overlay.get(full_anchor);
|
||||
// First access computes
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
|
||||
assert!(overlay.is_computed(prefix_anchor));
|
||||
assert!(overlay.is_computed(full_anchor));
|
||||
assert!(!Arc::ptr_eq(&prefix, &full));
|
||||
assert!(Arc::ptr_eq(&prefix, &overlay.get(prefix_anchor)));
|
||||
assert!(Arc::ptr_eq(&full, &overlay.get(full_anchor)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_anchor_limits_the_merged_prefix() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let expected = LazyOverlay::merge_blocks(&blocks[..2]);
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert_eq!(actual.nodes.as_ref(), expected.nodes.as_ref());
|
||||
assert_eq!(actual.state.as_ref(), expected.state.as_ref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_hash_returns_oldest_served_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let expected_anchor = blocks.last().unwrap().recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
assert_eq!(overlay.anchor_hash(), Some(expected_anchor));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reuses_tip_overlay_when_anchor_matches() {
|
||||
let mut blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let tip_overlay = Arc::new(LazyOverlay::merge_blocks(&blocks[..2]));
|
||||
let tip_data = blocks[0].trie_data();
|
||||
|
||||
blocks[0] = ExecutedBlock::new(
|
||||
Arc::clone(&blocks[0].recovered_block),
|
||||
Arc::clone(&blocks[0].execution_output),
|
||||
ComputedTrieData::with_trie_input(
|
||||
tip_data.hashed_state,
|
||||
tip_data.trie_updates,
|
||||
prefix_anchor,
|
||||
Arc::clone(&tip_overlay),
|
||||
),
|
||||
);
|
||||
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert!(Arc::ptr_eq(&actual, &tip_overlay));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay does not contain a block whose parent hash matches requested anchor"
|
||||
)]
|
||||
fn missing_anchor_panics() {
|
||||
let blocks = test_blocks();
|
||||
let missing_anchor = blocks[0].recovered_block().hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
let _ = overlay.get(missing_anchor);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
)]
|
||||
fn misordered_blocks_panic() {
|
||||
let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect();
|
||||
let _ = LazyOverlay::new(blocks);
|
||||
// Second access uses cache
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,22 +150,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
// commands can proceed.
|
||||
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
|
||||
reth_fs_util::create_dir_all(&rocksdb_path)?;
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level);
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
} else {
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write());
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
.build()?
|
||||
} else {
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?
|
||||
};
|
||||
|
||||
let provider_factory =
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
//! state), compacts MDBX, then runs the pipeline to rebuild them.
|
||||
|
||||
use crate::common::CliNodeTypes;
|
||||
use alloy_primitives::Address;
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
mdbx::{self, ffi},
|
||||
@@ -133,12 +132,8 @@ impl Command {
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::AccountChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(first_block))?.peekable();
|
||||
@@ -179,15 +174,11 @@ impl Command {
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::StorageChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some((first_block, Address::ZERO).into()))?.peekable();
|
||||
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
@@ -247,18 +238,6 @@ impl Command {
|
||||
.map_or(0, |b| b + 1);
|
||||
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
if first_block > 0 {
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::Receipts)?;
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
writer.commit()?;
|
||||
}
|
||||
|
||||
let before = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
|
||||
let block_range = first_block..=tip;
|
||||
|
||||
let segment = reth_static_file::segments::Receipts;
|
||||
@@ -266,11 +245,7 @@ impl Command {
|
||||
|
||||
sf_provider.commit()?;
|
||||
|
||||
let after = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
let count = after - before;
|
||||
info!(target: "reth::cli", count, "Receipts migrated");
|
||||
info!(target: "reth::cli", "Receipts migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -50,13 +50,8 @@ where
|
||||
info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state.");
|
||||
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
// Write EVM dummy data up to `header - 1` block. Skip when the supplied
|
||||
// header is at block 0: `header.number() - 1` would underflow in u64 to
|
||||
// `u64::MAX`, sending `append_dummy_chain` into a 1..=u64::MAX loop that
|
||||
// exhausts memory before failing.
|
||||
if header.number() > 0 {
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
}
|
||||
// Write EVM dummy data up to `header - 1` block
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
|
||||
info!(target: "reth::cli", "Appending first valid block.");
|
||||
|
||||
@@ -196,13 +191,7 @@ mod tests {
|
||||
use alloy_primitives::{address, b256};
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory};
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
@@ -275,45 +264,4 @@ mod tests {
|
||||
|
||||
assert_eq!(actual_next_height, expected_next_height);
|
||||
}
|
||||
|
||||
/// Regression: a header at block 0 used to send `append_dummy_chain` into
|
||||
/// a `1..=u64::MAX` loop because `header.number() - 1` underflowed in
|
||||
/// u64. The guard `if header.number() > 0` skips the dummy-chain step
|
||||
/// when there is no pre-genesis range to backfill, so `header_factory`
|
||||
/// is never invoked.
|
||||
#[test]
|
||||
fn test_setup_without_evm_skips_dummy_chain_for_genesis_header() {
|
||||
let header = Header { number: 0, ..Default::default() };
|
||||
let header_hash = header.hash_slow();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
init_genesis(&provider_factory).unwrap();
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
|
||||
let factory_calls = Arc::new(AtomicU64::new(0));
|
||||
let factory_calls_inner = Arc::clone(&factory_calls);
|
||||
|
||||
// The Result of `setup_without_evm` itself is not asserted: with
|
||||
// `number == 0` plus a genesis already written by `init_genesis`,
|
||||
// the subsequent `append_first_block` may legitimately fail. The
|
||||
// bug under test is the OOM in the dummy-chain loop, observable
|
||||
// through the factory-call counter below.
|
||||
let _ = setup_without_evm(
|
||||
&provider_rw,
|
||||
SealedHeader::new(header, header_hash),
|
||||
move |number| {
|
||||
// Bound calls so a regression cannot exhaust the test
|
||||
// runner's memory; the only correct value here is 0.
|
||||
let n = factory_calls_inner.fetch_add(1, Ordering::Relaxed);
|
||||
assert!(n < 8, "header_factory must not be invoked for a genesis-block header");
|
||||
Header { number, ..Default::default() }
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
factory_calls.load(Ordering::Relaxed),
|
||||
0,
|
||||
"append_dummy_chain must be skipped when header.number() == 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
)
|
||||
}
|
||||
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
|
||||
let default_secret_key_path = data_dir.p2p_secret();
|
||||
let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;
|
||||
|
||||
@@ -5,7 +5,6 @@ use crate::common::{
|
||||
EnvironmentArgs,
|
||||
};
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt};
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use clap::Parser;
|
||||
use eyre::WrapErr;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
|
||||
@@ -13,19 +12,15 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_evm::{execute::Executor, ConfigureEvm};
|
||||
use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected};
|
||||
use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected};
|
||||
use reth_provider::{
|
||||
BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider,
|
||||
StaticFileProviderFactory, TransactionVariant,
|
||||
};
|
||||
use reth_revm::{
|
||||
database::StateProviderDatabase,
|
||||
db::{states::reverts::AccountInfoRevert, BundleState},
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages::stages::calculate_gas_used_from_headers;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use reth_storage_api::{DBProvider, TryIntoHistoricalStateProvider};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
@@ -74,18 +69,13 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>> Command<C> {
|
||||
/// Execute `re-execute` command
|
||||
pub async fn execute<N>(
|
||||
mut self,
|
||||
self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
// Default to 4GB RocksDB block cache for re-execute unless explicitly set.
|
||||
if self.env.db.rocksdb_block_cache_size.is_none() {
|
||||
self.env.db.rocksdb_block_cache_size = Some(4 << 30);
|
||||
}
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
|
||||
|
||||
let components = components(provider_factory.chain_spec());
|
||||
@@ -119,6 +109,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
move |block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider_factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.disable_long_read_transaction_safety()
|
||||
.try_into_history_at_block(block_number)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let skip_invalid_blocks = self.skip_invalid_blocks;
|
||||
let blocks_per_chunk = self.blocks_per_chunk;
|
||||
let (stats_tx, mut stats_rx) = mpsc::unbounded_channel();
|
||||
@@ -134,23 +138,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let provider_factory = provider_factory.clone();
|
||||
let evm_config = components.evm_config().clone();
|
||||
let consensus = components.consensus().clone();
|
||||
let db_at = db_at.clone();
|
||||
let stats_tx = stats_tx.clone();
|
||||
let info_tx = info_tx.clone();
|
||||
let cancellation = cancellation.clone();
|
||||
let next_block = Arc::clone(&next_block);
|
||||
tasks.spawn_blocking(move || {
|
||||
let executor_lifetime = Duration::from_secs(600);
|
||||
let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
|
||||
|
||||
let db_at = {
|
||||
|block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider
|
||||
.history_by_block_number(block_number)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -260,28 +254,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
if executor.size_hint() > 5_000_000 ||
|
||||
executor_created.elapsed() > executor_lifetime
|
||||
{
|
||||
let last_block = block.number();
|
||||
let old_executor = std::mem::replace(
|
||||
&mut executor,
|
||||
evm_config.batch_executor(db_at(last_block)),
|
||||
);
|
||||
let bundle = old_executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
last_block,
|
||||
)?;
|
||||
executor =
|
||||
evm_config.batch_executor(db_at(block.number()));
|
||||
executor_created = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Full verification at chunk end for remaining unverified blocks
|
||||
let bundle = executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
chunk_end - 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
eyre::Ok(())
|
||||
@@ -362,98 +339,3 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies reverts against database changesets.
|
||||
///
|
||||
/// For each block, reverts must match changeset entries exactly. No extra slots/accounts
|
||||
/// in reverts for non-destroyed accounts. Destroyed accounts may have extra changeset slots
|
||||
/// (from DB storage wipe) absent from reverts.
|
||||
fn verify_bundle_against_changesets<P>(
|
||||
provider: &P,
|
||||
bundle: &BundleState,
|
||||
last_block: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
P: ChangeSetReader + StorageChangeSetReader,
|
||||
{
|
||||
// Verify reverts against changesets per block
|
||||
for (i, block_reverts) in bundle.reverts.iter().rev().enumerate() {
|
||||
let block_number = last_block - i as u64;
|
||||
|
||||
let mut cs_accounts: HashMap<Address, Option<Account>> = provider
|
||||
.account_block_changeset(block_number)?
|
||||
.into_iter()
|
||||
.map(|cs| (cs.address, cs.info))
|
||||
.collect();
|
||||
|
||||
let mut cs_storage: HashMap<Address, HashMap<B256, U256>> = HashMap::new();
|
||||
for (bna, entry) in provider.storage_changeset(block_number)? {
|
||||
cs_storage.entry(bna.address()).or_default().insert(entry.key, entry.value);
|
||||
}
|
||||
|
||||
for (addr, revert) in block_reverts {
|
||||
// Verify account info
|
||||
match &revert.account {
|
||||
AccountInfoRevert::DoNothing => {
|
||||
eyre::ensure!(
|
||||
!cs_accounts.contains_key(addr),
|
||||
"Block {block_number}: account {addr} in changeset but revert is DoNothing",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::DeleteIt => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is DeleteIt but not in changeset")
|
||||
})?;
|
||||
eyre::ensure!(
|
||||
cs_info.is_none(),
|
||||
"Block {block_number}: account {addr} revert is DeleteIt but changeset has {cs_info:?}",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::RevertTo(info) => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is RevertTo but not in changeset")
|
||||
})?;
|
||||
let revert_acct = Some(Account::from(info));
|
||||
eyre::ensure!(
|
||||
revert_acct == cs_info,
|
||||
"Block {block_number}: account {addr} info mismatch: revert={revert_acct:?} cs={cs_info:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify storage slots — remove matched changeset entries as we go
|
||||
let mut cs_slots = cs_storage.get_mut(addr);
|
||||
for (slot_key, revert_slot) in &revert.storage {
|
||||
let b256_key = B256::from(*slot_key);
|
||||
match cs_slots.as_mut().and_then(|s| s.remove(&b256_key)) {
|
||||
Some(cs_value) => eyre::ensure!(
|
||||
revert_slot.to_previous_value() == cs_value,
|
||||
"Block {block_number}: {addr} slot {b256_key} mismatch: \
|
||||
revert={} cs={cs_value}",
|
||||
revert_slot.to_previous_value(),
|
||||
),
|
||||
None => eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} slot {b256_key} in reverts but not in changeset",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_storage slots for this address must be from a destroyed account
|
||||
if let Some(remaining) = cs_slots.filter(|s| !s.is_empty()) {
|
||||
eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} has {} unmatched storage slots in changeset",
|
||||
remaining.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_accounts entries had no corresponding revert
|
||||
if let Some(addr) = cs_accounts.keys().next() {
|
||||
eyre::bail!("Block {block_number}: account {addr} in changeset but not in reverts");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_node_api::{HeaderTy, TxTy};
|
||||
use reth_node_api::HeaderTy;
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
|
||||
@@ -88,7 +88,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
)
|
||||
})??;
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::BlockOmmers<HeaderTy<N>>, _>(
|
||||
tx.import_table_with_range::<tables::BlockOmmers, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from),
|
||||
to,
|
||||
@@ -110,7 +110,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
})??;
|
||||
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::Transactions<TxTy<N>>, _>(
|
||||
tx.import_table_with_range::<tables::Transactions, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from_tx),
|
||||
to_tx,
|
||||
|
||||
@@ -210,7 +210,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let consensus = Arc::new(components.consensus().clone());
|
||||
|
||||
let mut config = config;
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
config.peers.trusted_nodes.extend(self.network.trusted_peers.clone());
|
||||
|
||||
let network_secret_path = self
|
||||
|
||||
@@ -374,7 +374,7 @@ async fn test_setup_builder_with_custom_tree_config() -> Result<()> {
|
||||
PayloadAttributes::default()
|
||||
})
|
||||
.with_tree_config_modifier(|config| {
|
||||
config.with_persistence_threshold(6).with_memory_block_buffer_target(5)
|
||||
config.with_persistence_threshold(0).with_memory_block_buffer_target(5)
|
||||
})
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -189,7 +189,7 @@ async fn test_rocksdb_transaction_queries() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -200,7 +200,7 @@ async fn test_rocksdb_transaction_queries() -> Result<()> {
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client should be available");
|
||||
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer.clone()).await;
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Wait for tx to enter pending pool before mining
|
||||
@@ -209,14 +209,6 @@ async fn test_rocksdb_transaction_queries() -> Result<()> {
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
let flush_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await;
|
||||
let flush_tx_hash = nodes[0].rpc.inject_tx(flush_tx).await?;
|
||||
wait_for_pending_tx(&client, flush_tx_hash).await;
|
||||
|
||||
let flush_payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(flush_payload.block().number(), 2);
|
||||
|
||||
// Query each transaction by hash
|
||||
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
|
||||
let tx = tx.expect("Transaction should be found");
|
||||
@@ -264,7 +256,7 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -291,14 +283,6 @@ async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
let flush_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 3).await;
|
||||
let flush_tx_hash = nodes[0].rpc.inject_tx(flush_tx).await?;
|
||||
wait_for_pending_tx(&client, flush_tx_hash).await;
|
||||
|
||||
let flush_payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(flush_payload.block().number(), 2);
|
||||
|
||||
// Verify block contains all 3 txs
|
||||
let block: Option<alloy_rpc_types_eth::Block> =
|
||||
client.request("eth_getBlockByNumber", ("0x1", true)).await?;
|
||||
@@ -340,7 +324,7 @@ async fn test_rocksdb_txs_across_blocks() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -425,7 +409,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -433,7 +417,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
let signer = wallets[0].clone();
|
||||
|
||||
// Inject tx but do NOT mine
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer.clone()).await;
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
|
||||
// Verify tx is in pending pool via RPC
|
||||
@@ -458,14 +442,6 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), 1);
|
||||
|
||||
let flush_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await;
|
||||
let flush_tx_hash = nodes[0].rpc.inject_tx(flush_tx).await?;
|
||||
wait_for_pending_tx(&client, flush_tx_hash).await;
|
||||
|
||||
let flush_payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(flush_payload.block().number(), 2);
|
||||
|
||||
// Poll until tx appears in RocksDB
|
||||
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
|
||||
assert_eq!(tx_number, 0, "First tx should have tx_number 0");
|
||||
@@ -497,7 +473,7 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -519,6 +495,10 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
let block1_hash = payload1.block().hash();
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
|
||||
// Poll until tx1 appears in RocksDB (ensures persistence happened)
|
||||
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
|
||||
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
|
||||
|
||||
// Mine block 2 with transaction from signer1 (nonce 1)
|
||||
let raw_tx2 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await;
|
||||
@@ -528,10 +508,6 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
let payload2 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload2.block().number(), 2);
|
||||
|
||||
// The second block triggers the first persistence cycle, which flushes both block 1 and 2.
|
||||
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
|
||||
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
|
||||
|
||||
// Poll until tx2 appears in RocksDB
|
||||
let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
|
||||
assert_eq!(tx_number2, 1, "Second tx should have tx_number 1");
|
||||
@@ -545,14 +521,6 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
let payload3 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload3.block().number(), 3);
|
||||
|
||||
let flush_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 3).await;
|
||||
let flush_tx_hash = nodes[0].rpc.inject_tx(flush_tx).await?;
|
||||
wait_for_pending_tx(&client, flush_tx_hash).await;
|
||||
|
||||
let flush_payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(flush_payload.block().number(), 4);
|
||||
|
||||
// Poll until tx3 appears in RocksDB
|
||||
let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
|
||||
assert_eq!(tx_number3, 2, "Third tx should have tx_number 2");
|
||||
@@ -564,7 +532,7 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?;
|
||||
wait_for_pending_tx(&client, alt_tx_hash).await;
|
||||
|
||||
// Build an alternate payload on top of the current flushed head.
|
||||
// Build an alternate payload (this builds on top of the current head, i.e., block 3)
|
||||
// But we want to reorg back to block 1, so we'll use the payload and then FCU to it
|
||||
let alt_payload = nodes[0].new_payload().await?;
|
||||
let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?;
|
||||
@@ -582,8 +550,8 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
let latest: Option<alloy_rpc_types_eth::Block> =
|
||||
client.request("eth_getBlockByNumber", ("latest", false)).await?;
|
||||
let latest = latest.expect("Latest block should exist");
|
||||
// The alt block is built on top of the flushed canonical head.
|
||||
assert!(latest.header.number >= 4, "Should be at height >= 4 after operation");
|
||||
// The alt block is at height 4 (on top of block 3)
|
||||
assert!(latest.header.number >= 3, "Should be at height >= 3 after operation");
|
||||
|
||||
// tx1 from block 1 should still be there
|
||||
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash1]).await?;
|
||||
@@ -628,7 +596,7 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
@@ -653,6 +621,8 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
|
||||
|
||||
// Record state after block 1
|
||||
let balance_at_1: U256 = client.request("eth_getBalance", (sender, "0x1")).await?;
|
||||
let nonce_at_1: U256 = client.request("eth_getTransactionCount", (sender, "0x1")).await?;
|
||||
@@ -667,6 +637,8 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
|
||||
let payload2 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload2.block().number(), 2);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
|
||||
|
||||
let balance_at_2: U256 = client.request("eth_getBalance", (sender, "0x2")).await?;
|
||||
let nonce_at_2: U256 = client.request("eth_getTransactionCount", (sender, "0x2")).await?;
|
||||
assert!(balance_at_2 < balance_at_1, "Balance should decrease further after second tx");
|
||||
@@ -680,14 +652,18 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
|
||||
let payload3 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload3.block().number(), 3);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
|
||||
|
||||
let balance_at_3: U256 = client.request("eth_getBalance", (sender, "0x3")).await?;
|
||||
let nonce_at_3: U256 = client.request("eth_getTransactionCount", (sender, "0x3")).await?;
|
||||
assert!(balance_at_3 < balance_at_2, "Balance should decrease further after third tx");
|
||||
assert_eq!(nonce_at_3, U256::from(3), "Nonce should be 3 after third tx");
|
||||
|
||||
// Mine additional blocks to push blocks 1-3 out of the in-memory overlay.
|
||||
// With a persistence threshold of 1, every second block triggers a flush, so a few extra
|
||||
// blocks are enough to durably persist and evict the earlier history we want to query.
|
||||
// With persistence_threshold=0 and memory_block_buffer_target=0, each new block
|
||||
// triggers persistence up to `head` followed by in-memory eviction. Mining several
|
||||
// more blocks ensures the engine loop has completed at least one full
|
||||
// persist-then-evict cycle covering blocks 1-3.
|
||||
// Each block needs a transaction because the payload builder requires non-empty payloads.
|
||||
for nonce in 3..8u64 {
|
||||
let raw_tx =
|
||||
@@ -697,7 +673,6 @@ async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
nodes[0].advance_block().await?;
|
||||
}
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
|
||||
// Allow the engine loop to process the persistence completions
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
@@ -768,7 +743,7 @@ async fn test_rocksdb_account_history_pruning() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.with_node_config_modifier(|mut config| {
|
||||
config.pruning.account_history_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.minimum_distance = Some(PRUNE_DISTANCE);
|
||||
@@ -865,7 +840,7 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> {
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(1))
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.with_node_config_modifier(|mut config| {
|
||||
config.pruning.storage_history_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.minimum_distance = Some(PRUNE_DISTANCE);
|
||||
@@ -937,6 +912,10 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> {
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await;
|
||||
|
||||
// Let the persistence cycle complete before the next block (same cadence as the loop below)
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// Get the deployed contract address from the receipt
|
||||
let receipt: Option<TransactionReceipt> =
|
||||
@@ -986,10 +965,6 @@ async fn test_rocksdb_storage_history_pruning() -> Result<()> {
|
||||
assert_eq!(payload.block().number(), block_num);
|
||||
last_tx_hash = tx_hash;
|
||||
|
||||
if nonce == 1 {
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await;
|
||||
}
|
||||
|
||||
// Let the persistence cycle complete before the next block
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
|
||||
@@ -37,9 +37,6 @@ auto_impl.workspace = true
|
||||
serde.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
alloy-primitives = { workspace = true, features = ["getrandom"] }
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
trie-debug = []
|
||||
|
||||
@@ -6,33 +6,12 @@ use core::time::Duration;
|
||||
/// Triggers persistence when the number of canonical blocks in memory exceeds this threshold.
|
||||
pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2;
|
||||
|
||||
/// Maximum number of consecutive canonical blocks whose non-trie outputs may be persisted ahead
|
||||
/// of trie persistence.
|
||||
pub const DEFAULT_DEFERRED_TRIE_BLOCKS: u64 = 0;
|
||||
/// Maximum canonical-minus-persisted gap before engine API processing is stalled.
|
||||
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16;
|
||||
|
||||
/// How close to the canonical head we persist blocks.
|
||||
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
|
||||
|
||||
/// Derives the default canonical-minus-persisted gap that triggers backpressure.
|
||||
pub const fn default_persistence_backpressure_threshold(
|
||||
persistence_threshold: u64,
|
||||
memory_block_buffer_target: u64,
|
||||
) -> u64 {
|
||||
let threshold = 2 * (persistence_threshold + memory_block_buffer_target);
|
||||
if threshold < 16 {
|
||||
16
|
||||
} else {
|
||||
threshold
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum canonical-minus-persisted gap before engine API processing is stalled.
|
||||
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 =
|
||||
default_persistence_backpressure_threshold(
|
||||
DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
);
|
||||
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5;
|
||||
|
||||
@@ -81,17 +60,6 @@ const fn assert_backpressure_threshold_invariant(
|
||||
);
|
||||
}
|
||||
|
||||
const fn assert_state_masking_invariant(
|
||||
persistence_threshold: u64,
|
||||
num_state_masking_blocks: u64,
|
||||
memory_block_buffer_target: u64,
|
||||
) {
|
||||
debug_assert!(
|
||||
num_state_masking_blocks + memory_block_buffer_target < persistence_threshold,
|
||||
"num_state_masking_blocks + memory_block_buffer_target must be less than persistence_threshold",
|
||||
);
|
||||
}
|
||||
|
||||
const fn default_cross_block_cache_size() -> usize {
|
||||
if cfg!(test) {
|
||||
1024 * 1024 // 1 MB in tests
|
||||
@@ -125,9 +93,6 @@ pub struct TreeConfig {
|
||||
/// Maximum number of blocks to be kept only in memory without triggering
|
||||
/// persistence.
|
||||
persistence_threshold: u64,
|
||||
/// Number of persisted blocks whose state/trie writes are masked instead of being durably
|
||||
/// written in the current cycle.
|
||||
num_state_masking_blocks: u64,
|
||||
/// How close to the canonical head we persist blocks. Represents the ideal
|
||||
/// number of most recent blocks to keep in memory for quick access and reorgs.
|
||||
///
|
||||
@@ -239,24 +204,14 @@ pub struct TreeConfig {
|
||||
|
||||
impl Default for TreeConfig {
|
||||
fn default() -> Self {
|
||||
let persistence_backpressure_threshold = default_persistence_backpressure_threshold(
|
||||
DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
);
|
||||
assert_backpressure_threshold_invariant(
|
||||
DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
persistence_backpressure_threshold,
|
||||
);
|
||||
assert_state_masking_invariant(
|
||||
DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
DEFAULT_DEFERRED_TRIE_BLOCKS,
|
||||
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
|
||||
);
|
||||
Self {
|
||||
persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
num_state_masking_blocks: DEFAULT_DEFERRED_TRIE_BLOCKS,
|
||||
memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
persistence_backpressure_threshold,
|
||||
persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
|
||||
block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT,
|
||||
max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH,
|
||||
invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
|
||||
@@ -298,7 +253,6 @@ impl TreeConfig {
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub const fn new(
|
||||
persistence_threshold: u64,
|
||||
num_state_masking_blocks: u64,
|
||||
memory_block_buffer_target: u64,
|
||||
persistence_backpressure_threshold: u64,
|
||||
block_buffer_limit: u32,
|
||||
@@ -331,14 +285,8 @@ impl TreeConfig {
|
||||
persistence_threshold,
|
||||
persistence_backpressure_threshold,
|
||||
);
|
||||
assert_state_masking_invariant(
|
||||
persistence_threshold,
|
||||
num_state_masking_blocks,
|
||||
memory_block_buffer_target,
|
||||
);
|
||||
Self {
|
||||
persistence_threshold,
|
||||
num_state_masking_blocks,
|
||||
memory_block_buffer_target,
|
||||
persistence_backpressure_threshold,
|
||||
block_buffer_limit,
|
||||
@@ -381,11 +329,6 @@ impl TreeConfig {
|
||||
self.persistence_threshold
|
||||
}
|
||||
|
||||
/// Return the number of persisted blocks whose state/trie writes are masked.
|
||||
pub const fn num_state_masking_blocks(&self) -> u64 {
|
||||
self.num_state_masking_blocks
|
||||
}
|
||||
|
||||
/// Return the memory block buffer target.
|
||||
pub const fn memory_block_buffer_target(&self) -> u64 {
|
||||
self.memory_block_buffer_target
|
||||
@@ -504,22 +447,6 @@ impl TreeConfig {
|
||||
self.persistence_threshold,
|
||||
self.persistence_backpressure_threshold,
|
||||
);
|
||||
assert_state_masking_invariant(
|
||||
self.persistence_threshold,
|
||||
self.num_state_masking_blocks,
|
||||
self.memory_block_buffer_target,
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for the number of persisted blocks whose state/trie writes are masked.
|
||||
pub const fn with_num_state_masking_blocks(mut self, num_state_masking_blocks: u64) -> Self {
|
||||
self.num_state_masking_blocks = num_state_masking_blocks;
|
||||
assert_state_masking_invariant(
|
||||
self.persistence_threshold,
|
||||
self.num_state_masking_blocks,
|
||||
self.memory_block_buffer_target,
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
@@ -529,11 +456,6 @@ impl TreeConfig {
|
||||
memory_block_buffer_target: u64,
|
||||
) -> Self {
|
||||
self.memory_block_buffer_target = memory_block_buffer_target;
|
||||
assert_state_masking_invariant(
|
||||
self.persistence_threshold,
|
||||
self.num_state_masking_blocks,
|
||||
self.memory_block_buffer_target,
|
||||
);
|
||||
self
|
||||
}
|
||||
|
||||
@@ -843,26 +765,7 @@ impl TreeConfig {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
default_persistence_backpressure_threshold, TreeConfig, DEFAULT_DEFERRED_TRIE_BLOCKS,
|
||||
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn default_thresholds_use_derived_backpressure_threshold() {
|
||||
let config = TreeConfig::default();
|
||||
|
||||
assert_eq!(config.persistence_threshold(), DEFAULT_PERSISTENCE_THRESHOLD);
|
||||
assert_eq!(config.num_state_masking_blocks(), DEFAULT_DEFERRED_TRIE_BLOCKS);
|
||||
assert_eq!(config.memory_block_buffer_target(), DEFAULT_MEMORY_BLOCK_BUFFER_TARGET);
|
||||
assert_eq!(
|
||||
config.persistence_backpressure_threshold(),
|
||||
default_persistence_backpressure_threshold(
|
||||
DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
)
|
||||
);
|
||||
}
|
||||
use super::TreeConfig;
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
@@ -873,15 +776,4 @@ mod tests {
|
||||
.with_persistence_threshold(4)
|
||||
.with_persistence_backpressure_threshold(4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "num_state_masking_blocks + memory_block_buffer_target must be less than persistence_threshold"
|
||||
)]
|
||||
fn rejects_state_masking_window_at_or_above_persistence_threshold() {
|
||||
let _ = TreeConfig::default()
|
||||
.with_persistence_threshold(4)
|
||||
.with_num_state_masking_blocks(2)
|
||||
.with_memory_block_buffer_target(2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,16 +1,16 @@
|
||||
use crate::metrics::PersistenceMetrics;
|
||||
use alloy_eips::BlockNumHash;
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use reth_chain_state::ExecutedBlock;
|
||||
use reth_errors::ProviderError;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_primitives_traits::{FastInstant as Instant, NodePrimitives};
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, BlockExecutionWriter, BlockHashReader, ChainStateBlockWriter,
|
||||
DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode, SaveBlocksPlan,
|
||||
StageCheckpointReader,
|
||||
DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode,
|
||||
};
|
||||
use reth_prune::{PrunerError, PrunerWithFactory};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender, StageId};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender};
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use std::{
|
||||
sync::{
|
||||
@@ -26,13 +26,8 @@ use tracing::{debug, error, instrument};
|
||||
/// Unified result of any persistence operation.
|
||||
#[derive(Debug)]
|
||||
pub struct PersistenceResult {
|
||||
/// The highest block whose non-state/trie outputs are persisted, if any.
|
||||
/// The last block that was persisted, if any.
|
||||
pub last_block: Option<BlockNumHash>,
|
||||
/// The highest block whose state/trie data is fully persisted, if known.
|
||||
///
|
||||
/// When this lags behind [`Self::last_block`], callers must retain the suffix
|
||||
/// above it in memory so trie-backed operations can still unwind from that point.
|
||||
pub last_state_trie_block: Option<u64>,
|
||||
/// The commit duration, only available for save-blocks operations.
|
||||
pub commit_duration: Option<Duration>,
|
||||
}
|
||||
@@ -101,14 +96,14 @@ where
|
||||
while let Ok(action) = self.incoming.recv() {
|
||||
match action {
|
||||
PersistenceAction::RemoveBlocksAbove(new_tip_num, sender) => {
|
||||
let result = self.on_remove_blocks_above(new_tip_num)?;
|
||||
let last_block = self.on_remove_blocks_above(new_tip_num)?;
|
||||
// send new sync metrics based on removed blocks
|
||||
let _ =
|
||||
self.sync_metrics_tx.send(MetricEvent::SyncHeight { height: new_tip_num });
|
||||
let _ = sender.send(result);
|
||||
let _ = sender.send(PersistenceResult { last_block, commit_duration: None });
|
||||
}
|
||||
PersistenceAction::SaveBlocks(plan, sender) => {
|
||||
let result = self.on_save_blocks(plan)?;
|
||||
PersistenceAction::SaveBlocks(blocks, sender) => {
|
||||
let result = self.on_save_blocks(blocks)?;
|
||||
let result_number = result.last_block.map(|b| b.number);
|
||||
|
||||
let _ = sender.send(result);
|
||||
@@ -135,40 +130,28 @@ where
|
||||
fn on_remove_blocks_above(
|
||||
&self,
|
||||
new_tip_num: u64,
|
||||
) -> Result<PersistenceResult, PersistenceError> {
|
||||
) -> Result<Option<BlockNumHash>, PersistenceError> {
|
||||
debug!(target: "engine::persistence", ?new_tip_num, "Removing blocks");
|
||||
let start_time = Instant::now();
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
|
||||
let new_tip_hash = provider_rw.block_hash(new_tip_num)?;
|
||||
provider_rw.remove_block_and_execution_above(new_tip_num)?;
|
||||
let last_state_trie_block =
|
||||
provider_rw.get_stage_checkpoint(StageId::Finish)?.map(|checkpoint| {
|
||||
checkpoint
|
||||
.finish_stage_checkpoint()
|
||||
.and_then(|finish| finish.partial_state_trie)
|
||||
.unwrap_or(checkpoint.block_number)
|
||||
});
|
||||
provider_rw.commit()?;
|
||||
|
||||
debug!(target: "engine::persistence", ?new_tip_num, ?new_tip_hash, "Removed blocks from disk");
|
||||
self.metrics.remove_blocks_above_duration_seconds.record(start_time.elapsed());
|
||||
Ok(PersistenceResult {
|
||||
last_block: new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }),
|
||||
last_state_trie_block,
|
||||
commit_duration: None,
|
||||
})
|
||||
Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = plan.blocks.len()))]
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = blocks.len()))]
|
||||
fn on_save_blocks(
|
||||
&mut self,
|
||||
plan: SaveBlocksPlan<N::Primitives>,
|
||||
blocks: Vec<ExecutedBlock<N::Primitives>>,
|
||||
) -> Result<PersistenceResult, PersistenceError> {
|
||||
let first_block = plan.blocks.first().map(|block| block.recovered_block().num_hash());
|
||||
let last_block = plan.last_block();
|
||||
let block_count = plan.blocks.len();
|
||||
let mut last_state_trie_block = None;
|
||||
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks.len();
|
||||
|
||||
let pending_finalized = self.pending_finalized_block.take();
|
||||
let pending_safe = self.pending_safe_block.take();
|
||||
@@ -177,27 +160,19 @@ where
|
||||
|
||||
let start_time = Instant::now();
|
||||
|
||||
if let Some(last_block) = last_block {
|
||||
if let Some(last) = last_block {
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
provider_rw.save_blocks(&plan, SaveBlocksMode::Full)?;
|
||||
last_state_trie_block = provider_rw
|
||||
.get_stage_checkpoint(StageId::Finish)?
|
||||
.and_then(|checkpoint| {
|
||||
checkpoint
|
||||
.finish_stage_checkpoint()
|
||||
.and_then(|finish| finish.partial_state_trie)
|
||||
})
|
||||
.or(Some(last_block.number));
|
||||
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
|
||||
|
||||
if let Some(finalized) = pending_finalized {
|
||||
provider_rw.save_finalized_block_number(finalized.min(last_block.number))?;
|
||||
if finalized > last_block.number {
|
||||
provider_rw.save_finalized_block_number(finalized.min(last.number))?;
|
||||
if finalized > last.number {
|
||||
self.pending_finalized_block = Some(finalized);
|
||||
}
|
||||
}
|
||||
if let Some(safe) = pending_safe {
|
||||
provider_rw.save_safe_block_number(safe.min(last_block.number))?;
|
||||
if safe > last_block.number {
|
||||
provider_rw.save_safe_block_number(safe.min(last.number))?;
|
||||
if safe > last.number {
|
||||
self.pending_safe_block = Some(safe);
|
||||
}
|
||||
}
|
||||
@@ -210,13 +185,13 @@ where
|
||||
//
|
||||
// The pruner reads the indices from rocksdb, filters it, and writes to indices, so it
|
||||
// must be able to read anything written by save_blocks.
|
||||
if self.pruner.is_pruning_needed(last_block.number) {
|
||||
debug!(target: "engine::persistence", block_num=?last_block.number, "Running pruner");
|
||||
if self.pruner.is_pruning_needed(last.number) {
|
||||
debug!(target: "engine::persistence", block_num=?last.number, "Running pruner");
|
||||
let prune_start = Instant::now();
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
let _ = self.pruner.run_with_provider(&provider_rw, last_block.number)?;
|
||||
let _ = self.pruner.run_with_provider(&provider_rw, last.number)?;
|
||||
provider_rw.commit()?;
|
||||
debug!(target: "engine::persistence", tip=?last_block.number, "Finished pruning after saving blocks");
|
||||
debug!(target: "engine::persistence", tip=?last.number, "Finished pruning after saving blocks");
|
||||
self.metrics.prune_before_duration_seconds.record(prune_start.elapsed());
|
||||
}
|
||||
}
|
||||
@@ -225,7 +200,7 @@ where
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(elapsed);
|
||||
|
||||
Ok(PersistenceResult { last_block, last_state_trie_block, commit_duration: Some(elapsed) })
|
||||
Ok(PersistenceResult { last_block, commit_duration: Some(elapsed) })
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,10 +222,9 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
/// The section of tree state that should be persisted. These blocks are expected in order of
|
||||
/// increasing block number.
|
||||
///
|
||||
/// First, header, transaction, and receipt-related data should be written to static files for
|
||||
/// the deferred trie region. Then the execution history-related data will be written to the
|
||||
/// database, while trie catchup is persisted for the prefix.
|
||||
SaveBlocks(SaveBlocksPlan<N>, CrossbeamSender<PersistenceResult>),
|
||||
/// First, header, transaction, and receipt-related data should be written to static files.
|
||||
/// Then the execution history-related data will be written to the database.
|
||||
SaveBlocks(Vec<ExecutedBlock<N>>, CrossbeamSender<PersistenceResult>),
|
||||
|
||||
/// Removes block data above the given block number from the database.
|
||||
///
|
||||
@@ -334,10 +308,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
/// If there are no blocks to persist, then `None` is sent in the sender.
|
||||
pub fn save_blocks(
|
||||
&self,
|
||||
plan: SaveBlocksPlan<T>,
|
||||
blocks: Vec<ExecutedBlock<T>>,
|
||||
tx: CrossbeamSender<PersistenceResult>,
|
||||
) -> Result<(), SendError<PersistenceAction<T>>> {
|
||||
self.send_action(PersistenceAction::SaveBlocks(plan, tx))
|
||||
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
|
||||
}
|
||||
|
||||
/// Queues the finalized block number to be persisted on disk.
|
||||
@@ -401,12 +375,12 @@ impl Drop for ServiceGuard {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{B256, U256};
|
||||
use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock};
|
||||
use reth_chain_state::test_utils::TestBlockBuilder;
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_provider::{
|
||||
providers::{ProviderFactoryBuilder, ReadOnlyConfig},
|
||||
test_utils::{create_test_provider_factory, MockNodeTypes},
|
||||
AccountReader, ChainSpecProvider, HeaderProvider, SaveBlocksPlanStep, StorageSettingsCache,
|
||||
AccountReader, ChainSpecProvider, HeaderProvider, StorageSettingsCache,
|
||||
TryIntoHistoricalStateProvider,
|
||||
};
|
||||
use reth_prune::Pruner;
|
||||
@@ -415,13 +389,6 @@ mod tests {
|
||||
fn default_persistence_handle() -> PersistenceHandle<EthPrimitives> {
|
||||
let provider = create_test_provider_factory();
|
||||
|
||||
persistence_handle(provider)
|
||||
}
|
||||
|
||||
fn persistence_handle<N>(provider: ProviderFactory<N>) -> PersistenceHandle<EthPrimitives>
|
||||
where
|
||||
N: ProviderNodeTypes<Primitives = EthPrimitives>,
|
||||
{
|
||||
let (_finished_exex_height_tx, finished_exex_height_rx) =
|
||||
tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
@@ -432,31 +399,18 @@ mod tests {
|
||||
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx)
|
||||
}
|
||||
|
||||
fn full_save_plan(blocks: Vec<ExecutedBlock<EthPrimitives>>) -> SaveBlocksPlan<EthPrimitives> {
|
||||
let full_range = 0..blocks.len();
|
||||
SaveBlocksPlan::new(
|
||||
blocks,
|
||||
vec![SaveBlocksPlanStep::new(
|
||||
full_range.clone(),
|
||||
Some(full_range.end..full_range.end),
|
||||
true,
|
||||
)],
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_save_blocks_empty() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let blocks = full_save_plan(vec![]);
|
||||
let blocks = vec![];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let result = rx.recv().unwrap();
|
||||
assert!(result.last_block.is_none());
|
||||
assert!(result.last_state_trie_block.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -469,16 +423,14 @@ mod tests {
|
||||
test_block_builder.get_executed_block_with_number(block_number, B256::random());
|
||||
let block_hash = executed.recovered_block().hash();
|
||||
|
||||
let blocks = full_save_plan(vec![executed]);
|
||||
let blocks = vec![executed];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out");
|
||||
|
||||
let last_block = result.last_block.unwrap();
|
||||
assert_eq!(block_hash, last_block.hash);
|
||||
assert_eq!(result.last_state_trie_block, Some(last_block.number));
|
||||
assert_eq!(block_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -491,11 +443,9 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(full_save_plan(blocks), tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
let result = rx.recv().unwrap();
|
||||
let last_block = result.last_block.unwrap();
|
||||
assert_eq!(last_hash, last_block.hash);
|
||||
assert_eq!(result.last_state_trie_block, Some(last_block.number));
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -510,57 +460,13 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(full_save_plan(blocks), tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let result = rx.recv().unwrap();
|
||||
let last_block = result.last_block.unwrap();
|
||||
assert_eq!(last_hash, last_block.hash);
|
||||
assert_eq!(result.last_state_trie_block, Some(last_block.number));
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_blocks_above_preserves_partial_state_trie() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let provider = create_test_provider_factory();
|
||||
let mut test_block_builder = TestBlockBuilder::eth().with_state();
|
||||
let blocks = test_block_builder.get_executed_blocks(0..4).collect::<Vec<_>>();
|
||||
|
||||
let provider_rw = provider.database_provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&SaveBlocksPlan::new(
|
||||
blocks,
|
||||
vec![
|
||||
SaveBlocksPlanStep::new(0..2, Some(2..4), true),
|
||||
SaveBlocksPlanStep::new(2..4, None, true),
|
||||
],
|
||||
),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let handle = persistence_handle(provider.clone());
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.remove_blocks_above(2, tx).unwrap();
|
||||
|
||||
let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out");
|
||||
let last_block = result.last_block.unwrap();
|
||||
assert_eq!(last_block.number, 2);
|
||||
assert_eq!(result.last_state_trie_block, Some(1));
|
||||
|
||||
let finish_checkpoint =
|
||||
provider.provider().unwrap().get_stage_checkpoint(StageId::Finish).unwrap().unwrap();
|
||||
assert_eq!(finish_checkpoint.block_number, 2);
|
||||
assert_eq!(
|
||||
finish_checkpoint.finish_stage_checkpoint().unwrap().partial_state_trie,
|
||||
Some(1)
|
||||
);
|
||||
}
|
||||
|
||||
/// Verifies that committing `save_blocks` history before running the pruner
|
||||
/// prevents the pruner from overwriting new entries.
|
||||
///
|
||||
@@ -649,7 +555,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
provider_rw.save_blocks(&full_save_plan(blocks_a), SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
}
|
||||
|
||||
@@ -706,12 +612,7 @@ mod tests {
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider_rw = pf.database_provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&full_save_plan(std::slice::from_ref(&block_b2).to_vec()),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
});
|
||||
|
||||
|
||||
@@ -30,9 +30,9 @@ use reth_primitives_traits::{
|
||||
};
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, SaveBlocksPlan,
|
||||
SaveBlocksPlanStep, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader,
|
||||
StorageChangeSetReader, StorageSettingsCache, TransactionVariant,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
StorageSettingsCache, TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
@@ -433,7 +433,6 @@ where
|
||||
|
||||
let persistence_state = PersistenceState {
|
||||
last_persisted_block: BlockNumHash::new(best_block_number, header.hash()),
|
||||
last_state_trie_persisted_block: BlockNumHash::new(best_block_number, header.hash()),
|
||||
rx: None,
|
||||
};
|
||||
|
||||
@@ -1351,7 +1350,7 @@ where
|
||||
/// Helper method to remove blocks and set the persistence state. This ensures we keep track of
|
||||
/// the current persistence action while we're removing blocks.
|
||||
fn remove_blocks(&mut self, new_tip_num: u64) {
|
||||
debug!(target: "engine::tree", ?new_tip_num, last_persisted_block=?self.persistence_state.last_persisted_block.number, "Removing blocks using persistence task");
|
||||
debug!(target: "engine::tree", ?new_tip_num, last_persisted_block_number=?self.persistence_state.last_persisted_block.number, "Removing blocks using persistence task");
|
||||
if new_tip_num < self.persistence_state.last_persisted_block.number {
|
||||
debug!(target: "engine::tree", ?new_tip_num, "Starting remove blocks job");
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
@@ -1362,25 +1361,24 @@ where
|
||||
|
||||
/// Helper method to save blocks and set the persistence state. This ensures we keep track of
|
||||
/// the current persistence action while we're saving blocks.
|
||||
fn persist_blocks(&mut self, plan: SaveBlocksPlan<N>) {
|
||||
if plan.is_empty() {
|
||||
fn persist_blocks(&mut self, blocks_to_persist: Vec<ExecutedBlock<N>>) {
|
||||
if blocks_to_persist.is_empty() {
|
||||
debug!(target: "engine::tree", "Returned empty set of blocks to persist");
|
||||
return
|
||||
}
|
||||
|
||||
let last_block = plan.last_block().expect("checked non-empty persisting blocks");
|
||||
// NOTE: checked non-empty above
|
||||
let highest_num_hash = blocks_to_persist
|
||||
.iter()
|
||||
.max_by_key(|block| block.recovered_block().number())
|
||||
.map(|b| b.recovered_block().num_hash())
|
||||
.expect("Checked non-empty persisting blocks");
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
count = plan.blocks.len(),
|
||||
steps = ?plan.steps,
|
||||
blocks = ?plan.blocks.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(),
|
||||
"Persisting blocks"
|
||||
);
|
||||
debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
let _ = self.persistence.save_blocks(plan, tx);
|
||||
let _ = self.persistence.save_blocks(blocks_to_persist, tx);
|
||||
|
||||
self.persistence_state.start_save(last_block, rx);
|
||||
self.persistence_state.start_save(highest_num_hash, rx);
|
||||
}
|
||||
|
||||
/// Triggers new persistence actions if no persistence task is currently in progress.
|
||||
@@ -1392,8 +1390,9 @@ where
|
||||
if let Some(new_tip_num) = self.find_disk_reorg()? {
|
||||
self.remove_blocks(new_tip_num)
|
||||
} else if self.should_persist() {
|
||||
let plan = self.get_save_blocks_plan(PersistTarget::Threshold)?;
|
||||
self.persist_blocks(plan);
|
||||
let blocks_to_persist =
|
||||
self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1424,15 +1423,15 @@ where
|
||||
self.on_persistence_complete(result, start_time)?;
|
||||
}
|
||||
|
||||
let plan = self.get_save_blocks_plan(PersistTarget::Head)?;
|
||||
let blocks_to_persist = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
|
||||
|
||||
if plan.is_empty() {
|
||||
if blocks_to_persist.is_empty() {
|
||||
debug!(target: "engine::tree", "persistence complete, signaling termination");
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
debug!(target: "engine::tree", count = plan.blocks.len(), "persisting remaining blocks before shutdown");
|
||||
self.persist_blocks(plan);
|
||||
debug!(target: "engine::tree", count = blocks_to_persist.len(), "persisting remaining blocks before shutdown");
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1468,25 +1467,25 @@ where
|
||||
) -> Result<(), AdvancePersistenceError> {
|
||||
self.metrics.engine.persistence_duration.record(start_time.elapsed());
|
||||
|
||||
let PersistenceResult { last_block, last_state_trie_block, commit_duration } = result;
|
||||
let Some(BlockNumHash { hash: last_block_hash, number: last_block_number }) = last_block
|
||||
let commit_duration = result.commit_duration;
|
||||
let Some(BlockNumHash {
|
||||
hash: last_persisted_block_hash,
|
||||
number: last_persisted_block_number,
|
||||
}) = result.last_block
|
||||
else {
|
||||
// if this happened, then we persisted no blocks because we sent an empty vec of blocks
|
||||
warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks");
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let last_block = BlockNumHash::new(last_block_number, last_block_hash);
|
||||
let last_state_trie_persisted_block =
|
||||
self.last_state_trie_persisted_block(last_block, last_state_trie_block)?;
|
||||
|
||||
debug!(target: "engine::tree", ?last_block_hash, ?last_block_number, last_state_trie_persisted_block = last_state_trie_persisted_block.number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_block, last_state_trie_persisted_block);
|
||||
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
|
||||
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
|
||||
|
||||
// Evict trie changesets for blocks below the eviction threshold.
|
||||
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
|
||||
// the finalized block if set.
|
||||
let min_threshold = last_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
|
||||
let min_threshold =
|
||||
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
|
||||
let eviction_threshold =
|
||||
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
|
||||
// Use the minimum of finalized block and retention threshold to be conservative
|
||||
@@ -1497,7 +1496,7 @@ where
|
||||
};
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
last_persisted_block = last_block_number,
|
||||
last_persisted = last_persisted_block_number,
|
||||
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
|
||||
eviction_threshold,
|
||||
"Evicting changesets below threshold"
|
||||
@@ -1507,50 +1506,22 @@ where
|
||||
// Invalidate cached overlay since the anchor has changed
|
||||
self.state.tree_state.invalidate_cached_overlay();
|
||||
|
||||
self.on_new_persisted_block(last_state_trie_persisted_block)?;
|
||||
self.on_new_persisted_block()?;
|
||||
|
||||
// Re-prepare overlay for the current canonical head with the new anchor.
|
||||
// Spawn a background task to trigger computation so it's ready when the next payload
|
||||
// arrives.
|
||||
if let Some(prepared) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
self.runtime.spawn_blocking_named("prepare-overlay", move || {
|
||||
let _ = prepared.overlay.get(prepared.anchor_hash);
|
||||
let _ = overlay.get();
|
||||
});
|
||||
}
|
||||
|
||||
self.purge_timing_stats(last_block_number, commit_duration);
|
||||
self.purge_timing_stats(last_persisted_block_number, commit_duration);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Returns the highest block that can be dropped from memory after persistence completes.
|
||||
fn last_state_trie_persisted_block(
|
||||
&self,
|
||||
last_block: BlockNumHash,
|
||||
last_state_trie_block: Option<u64>,
|
||||
) -> ProviderResult<BlockNumHash> {
|
||||
let Some(last_state_trie_block) = last_state_trie_block else { return Ok(last_block) };
|
||||
debug_assert!(
|
||||
last_state_trie_block <= last_block.number,
|
||||
"state/trie frontier cannot exceed the last persisted block"
|
||||
);
|
||||
if last_state_trie_block >= last_block.number {
|
||||
return Ok(last_block)
|
||||
}
|
||||
|
||||
let hash = self
|
||||
.canonical_in_memory_state
|
||||
.hash_by_number(last_state_trie_block)
|
||||
.map(Ok)
|
||||
.unwrap_or_else(|| {
|
||||
self.provider
|
||||
.block_hash(last_state_trie_block)?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(last_state_trie_block.into()))
|
||||
})?;
|
||||
|
||||
Ok(BlockNumHash::new(last_state_trie_block, hash))
|
||||
}
|
||||
|
||||
/// Handles a message from the engine.
|
||||
///
|
||||
/// Returns `ControlFlow::Break(())` if the engine should terminate.
|
||||
@@ -1854,7 +1825,7 @@ where
|
||||
// update the tracked chain height, after backfill sync both the canonical height and
|
||||
// persisted height are the same
|
||||
self.state.tree_state.set_canonical_head(new_head.num_hash());
|
||||
self.persistence_state.finish(new_head.num_hash(), new_head.num_hash());
|
||||
self.persistence_state.finish(new_head.hash(), new_head.number());
|
||||
|
||||
// update the tracked canonical head
|
||||
self.canonical_in_memory_state.set_canonical_head(new_head);
|
||||
@@ -2062,96 +2033,62 @@ where
|
||||
self.config.persistence_threshold()
|
||||
}
|
||||
|
||||
/// Returns the save plan for the next persistence cycle.
|
||||
fn get_save_blocks_plan(
|
||||
/// Returns a batch of consecutive canonical blocks to persist in the range
|
||||
/// `(last_persisted_number .. target]`. The expected order is oldest -> newest.
|
||||
fn get_canonical_blocks_to_persist(
|
||||
&self,
|
||||
target: PersistTarget,
|
||||
) -> Result<SaveBlocksPlan<N>, AdvancePersistenceError> {
|
||||
) -> Result<Vec<ExecutedBlock<N>>, AdvancePersistenceError> {
|
||||
// We will calculate the state root using the database, so we need to be sure there are no
|
||||
// changes
|
||||
debug_assert!(!self.persistence_state.in_progress());
|
||||
|
||||
let mut blocks = Vec::new();
|
||||
let mut blocks_to_persist = Vec::new();
|
||||
let mut current_hash = self.state.tree_state.canonical_block_hash();
|
||||
let last_state_trie_persisted_block_number =
|
||||
self.persistence_state.last_state_trie_persisted_block.number;
|
||||
let last_persisted_block_number = self.persistence_state.last_persisted_block.number;
|
||||
let last_persisted_number = self.persistence_state.last_persisted_block.number;
|
||||
let canonical_head_number = self.state.tree_state.canonical_block_number();
|
||||
let last_block_target_number = match target {
|
||||
|
||||
let target_number = match target {
|
||||
PersistTarget::Head => canonical_head_number,
|
||||
PersistTarget::Threshold => {
|
||||
canonical_head_number.saturating_sub(self.config.memory_block_buffer_target())
|
||||
}
|
||||
PersistTarget::Head => canonical_head_number,
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
?current_hash,
|
||||
?last_state_trie_persisted_block_number,
|
||||
?last_persisted_block_number,
|
||||
?last_persisted_number,
|
||||
?canonical_head_number,
|
||||
target = ?target,
|
||||
"Returning save plan"
|
||||
?target_number,
|
||||
"Returning canonical blocks to persist"
|
||||
);
|
||||
while let Some(block) = self.state.tree_state.blocks_by_hash.get(¤t_hash) {
|
||||
if block.recovered_block().number() <= last_state_trie_persisted_block_number {
|
||||
if block.recovered_block().number() <= last_persisted_number {
|
||||
break;
|
||||
}
|
||||
|
||||
if block.recovered_block().number() <= last_block_target_number {
|
||||
blocks.push(block.clone());
|
||||
if block.recovered_block().number() <= target_number {
|
||||
blocks_to_persist.push(block.clone());
|
||||
}
|
||||
|
||||
current_hash = block.recovered_block().parent_hash();
|
||||
}
|
||||
|
||||
// Reverse the order so that the oldest block comes first
|
||||
blocks.reverse();
|
||||
blocks_to_persist.reverse();
|
||||
|
||||
let trie_catchup_block_count = last_persisted_block_number
|
||||
.saturating_sub(last_state_trie_persisted_block_number)
|
||||
.min(blocks.len() as u64) as usize;
|
||||
let persist_rest_block_count = blocks.len().saturating_sub(trie_catchup_block_count);
|
||||
let state_masking_block_count =
|
||||
persist_rest_block_count.min(self.config.num_state_masking_blocks() as usize);
|
||||
let full_persist_block_count = persist_rest_block_count - state_masking_block_count;
|
||||
let full_persist_start = trie_catchup_block_count;
|
||||
let state_masking_start = full_persist_start + full_persist_block_count;
|
||||
let state_masking_range = state_masking_start..blocks.len();
|
||||
let mut steps = Vec::new();
|
||||
|
||||
if trie_catchup_block_count > 0 {
|
||||
steps.push(SaveBlocksPlanStep::new(
|
||||
0..trie_catchup_block_count,
|
||||
Some(state_masking_range.clone()),
|
||||
false,
|
||||
));
|
||||
}
|
||||
if full_persist_block_count > 0 {
|
||||
steps.push(SaveBlocksPlanStep::new(
|
||||
full_persist_start..state_masking_start,
|
||||
Some(state_masking_range.clone()),
|
||||
true,
|
||||
));
|
||||
}
|
||||
if state_masking_block_count > 0 {
|
||||
steps.push(SaveBlocksPlanStep::new(state_masking_range, None, true));
|
||||
}
|
||||
|
||||
Ok(SaveBlocksPlan::new(blocks, steps))
|
||||
Ok(blocks_to_persist)
|
||||
}
|
||||
|
||||
/// This clears the blocks from the in-memory tree state that no longer need to stay resident
|
||||
/// after persistence completes.
|
||||
/// This clears the blocks from the in-memory tree state that have been persisted to the
|
||||
/// database.
|
||||
///
|
||||
/// This also updates the canonical in-memory state to reflect the newest persisted block tip,
|
||||
/// even if trie persistence only advanced through an earlier block.
|
||||
/// This also updates the canonical in-memory state to reflect the newest persisted block
|
||||
/// height.
|
||||
///
|
||||
/// Assumes that `finish` has been called on the `persistence_state` at least once
|
||||
fn on_new_persisted_block(
|
||||
&mut self,
|
||||
in_memory_persisted_block: BlockNumHash,
|
||||
) -> ProviderResult<()> {
|
||||
fn on_new_persisted_block(&mut self) -> ProviderResult<()> {
|
||||
// If we have an on-disk reorg, we need to handle it first before touching the in-memory
|
||||
// state.
|
||||
if let Some(remove_above) = self.find_disk_reorg()? {
|
||||
@@ -2160,11 +2097,11 @@ where
|
||||
}
|
||||
|
||||
let finalized = self.state.forkchoice_state_tracker.last_valid_finalized();
|
||||
self.remove_before(in_memory_persisted_block, finalized)?;
|
||||
self.canonical_in_memory_state.remove_persisted_blocks_until(
|
||||
self.persistence_state.last_persisted_block,
|
||||
in_memory_persisted_block.number,
|
||||
);
|
||||
self.remove_before(self.persistence_state.last_persisted_block, finalized)?;
|
||||
self.canonical_in_memory_state.remove_persisted_blocks(BlockNumHash {
|
||||
number: self.persistence_state.last_persisted_block.number,
|
||||
hash: self.persistence_state.last_persisted_block.hash,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -3092,15 +3029,7 @@ where
|
||||
InsertBlockValidationError::Consensus(err) => self.consensus.is_transient_error(err),
|
||||
_ => false,
|
||||
};
|
||||
if is_transient {
|
||||
warn!(
|
||||
target: "engine::tree",
|
||||
invalid_hash=%block.hash(),
|
||||
invalid_number=block.number(),
|
||||
%validation_err,
|
||||
"Skipping invalid header cache insert for transient validation error",
|
||||
);
|
||||
} else {
|
||||
if !is_transient {
|
||||
self.state.invalid_headers.insert(block.block_with_parent());
|
||||
}
|
||||
self.emit_event(EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::InvalidBlock(
|
||||
|
||||
@@ -970,12 +970,12 @@ mod tests {
|
||||
use rand::Rng;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_ethereum_primitives::{EthPrimitives, TransactionSigned};
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_evm::OnStateHook;
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
use reth_primitives_traits::{Account, Recovered, StorageEntry};
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory},
|
||||
providers::{BlockchainProvider, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory_with_chain_spec,
|
||||
ChainSpecProvider, HashingWriter,
|
||||
};
|
||||
@@ -1250,10 +1250,7 @@ mod tests {
|
||||
std::convert::identity,
|
||||
),
|
||||
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
|
||||
OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::<EthPrimitives>::new(genesis_hash, ChangesetCache::new()),
|
||||
),
|
||||
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
|
||||
&TreeConfig::default(),
|
||||
);
|
||||
|
||||
|
||||
@@ -894,9 +894,7 @@ mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{keccak256, Address, B256, U256};
|
||||
use reth_provider::{
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory,
|
||||
ChainSpecProvider,
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::proof_task::ProofTaskCtx;
|
||||
@@ -985,14 +983,8 @@ mod tests {
|
||||
fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() {
|
||||
let runtime = reth_tasks::Runtime::test();
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let anchor_hash = provider_factory.chain_spec().genesis_hash();
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::<reth_chain_state::EthPrimitives>::new(
|
||||
anchor_hash,
|
||||
ChangesetCache::new(),
|
||||
),
|
||||
);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new());
|
||||
let proof_worker_handle =
|
||||
ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false);
|
||||
|
||||
|
||||
@@ -80,14 +80,13 @@ use reth_primitives_traits::{
|
||||
RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable,
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory,
|
||||
DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader,
|
||||
StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader,
|
||||
StorageChangeSetReader, StorageSettingsCache,
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
use revm_primitives::{Address, KECCAK_EMPTY};
|
||||
@@ -526,11 +525,10 @@ where
|
||||
|
||||
// Create overlay factory for payload processor (StateRootTask path needs it for
|
||||
// multiproofs)
|
||||
let provider_factory = self.provider.clone();
|
||||
let overlay_builder = OverlayBuilder::<N>::new(anchor_hash, self.changeset_cache.clone())
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory.clone(), overlay_builder.clone());
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
|
||||
// Spawn the appropriate processor based on strategy
|
||||
let mut handle = ensure_ok!(self.spawn_payload_processor(
|
||||
@@ -667,7 +665,7 @@ where
|
||||
let task_result = ensure_ok_post_block!(
|
||||
self.await_state_root_with_timeout(
|
||||
&mut handle,
|
||||
provider_builder.clone(),
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
),
|
||||
block
|
||||
@@ -691,9 +689,7 @@ where
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
let _has_diff = self.compare_trie_updates_with_serial(
|
||||
provider_builder.clone(),
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.as_ref().clone(),
|
||||
);
|
||||
@@ -732,11 +728,7 @@ where
|
||||
}
|
||||
StateRootStrategy::Parallel => {
|
||||
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
|
||||
match self.compute_state_root_parallel(
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
&hashed_state,
|
||||
) {
|
||||
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
|
||||
Ok(result) => {
|
||||
let elapsed = root_time.elapsed();
|
||||
info!(
|
||||
@@ -772,12 +764,19 @@ where
|
||||
}
|
||||
|
||||
let (root, updates) = ensure_ok_post_block!(
|
||||
provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)),
|
||||
Self::compute_state_root_serial_with_provider(
|
||||
provider_builder.clone(),
|
||||
&hashed_state
|
||||
),
|
||||
block
|
||||
);
|
||||
|
||||
self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
updates.clone(),
|
||||
);
|
||||
|
||||
if state_root_task_failed {
|
||||
self.metrics.block_validation.state_root_task_fallback_success_total.increment(1);
|
||||
}
|
||||
@@ -1098,8 +1097,7 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
fn compute_state_root_parallel(
|
||||
&self,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder<N>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
|
||||
let hashed_state = hashed_state.get();
|
||||
@@ -1107,24 +1105,42 @@ where
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// ParallelStateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()),
|
||||
);
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone())
|
||||
.incremental_root_with_updates()
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in serial.
|
||||
///
|
||||
/// Uses the same provider construction path as main execution and computes the state root and
|
||||
/// trie updates for this block directly via
|
||||
/// [`reth_provider::StateRootProvider::state_root_with_updates`].
|
||||
/// Uses an overlay factory which provides the state of the parent block, along with the
|
||||
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
|
||||
/// trie updates for this block.
|
||||
fn compute_state_root_serial(
|
||||
state_provider: StateProviderBox,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
state_provider.state_root_with_updates(hashed_state.get().clone())
|
||||
let hashed_state = hashed_state.get();
|
||||
// The `hashed_state` argument will be taken into account as part of the overlay, but we
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// StateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
|
||||
let provider = overlay_factory.database_provider_ro()?;
|
||||
|
||||
Ok(StateRoot::new(&provider, &provider)
|
||||
.with_prefix_sets(prefix_sets)
|
||||
.root_with_updates()?)
|
||||
}
|
||||
|
||||
fn compute_state_root_serial_with_provider(
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
let provider = provider_builder.build()?;
|
||||
provider.state_root_with_updates(hashed_state.get().clone())
|
||||
}
|
||||
|
||||
/// Awaits the state root from the background task, with an optional timeout fallback.
|
||||
@@ -1149,7 +1165,7 @@ where
|
||||
fn await_state_root_with_timeout<Tx, Err, R: Send + Sync + 'static>(
|
||||
&self,
|
||||
handle: &mut PayloadHandle<Tx, Err, R>,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<Result<StateRootComputeOutcome, ParallelStateRootError>> {
|
||||
let Some(timeout) = self.config.state_root_task_timeout() else {
|
||||
@@ -1174,11 +1190,10 @@ where
|
||||
let (seq_tx, seq_rx) =
|
||||
std::sync::mpsc::channel::<ProviderResult<(B256, TrieUpdates)>>();
|
||||
|
||||
let seq_overlay = overlay_factory;
|
||||
let seq_hashed_state = hashed_state.clone();
|
||||
self.payload_processor.executor().spawn_blocking_named("serial-root", move || {
|
||||
let result = state_provider_builder.build().and_then(|provider| {
|
||||
Self::compute_state_root_serial(provider, &seq_hashed_state)
|
||||
});
|
||||
let result = Self::compute_state_root_serial(seq_overlay, &seq_hashed_state);
|
||||
let _ = seq_tx.send(result);
|
||||
});
|
||||
|
||||
@@ -1242,18 +1257,13 @@ where
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder<N>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) -> bool {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match state_provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, hashed_state))
|
||||
{
|
||||
match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
@@ -1262,8 +1272,6 @@ where
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory, overlay_builder);
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
match super::trie_updates::compare_trie_updates(
|
||||
@@ -1447,7 +1455,7 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
txs: T,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P, N>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
strategy: StateRootStrategy,
|
||||
) -> Result<
|
||||
PayloadHandle<
|
||||
@@ -1567,7 +1575,7 @@ where
|
||||
fn get_parent_lazy_overlay(
|
||||
parent_hash: B256,
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> (Option<LazyOverlay<N>>, B256) {
|
||||
) -> (Option<LazyOverlay>, B256) {
|
||||
// Get blocks leading to the parent to determine the anchor
|
||||
let (anchor_hash, blocks) =
|
||||
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
|
||||
@@ -1595,7 +1603,10 @@ where
|
||||
"Creating lazy overlay for in-memory blocks"
|
||||
);
|
||||
|
||||
(Some(LazyOverlay::new(blocks)), anchor_hash)
|
||||
// Extract deferred trie data handles (non-blocking)
|
||||
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
|
||||
|
||||
(Some(LazyOverlay::new(anchor_hash, handles)), anchor_hash)
|
||||
}
|
||||
|
||||
/// Spawns a background task to compute and sort trie data for the executed block.
|
||||
@@ -2027,11 +2038,10 @@ where
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> Option<StateRootHandle> {
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state);
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
self.provider.clone(),
|
||||
OverlayBuilder::<N>::new(anchor_hash, self.changeset_cache.clone())
|
||||
.with_lazy_overlay(lazy_overlay),
|
||||
);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
|
||||
Some(self.payload_processor.spawn_state_root(
|
||||
overlay_factory,
|
||||
|
||||
@@ -22,6 +22,7 @@
|
||||
|
||||
use crate::persistence::PersistenceResult;
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Receiver as CrossbeamReceiver;
|
||||
use reth_primitives_traits::FastInstant as Instant;
|
||||
use tracing::trace;
|
||||
@@ -29,12 +30,10 @@ use tracing::trace;
|
||||
/// The state of the persistence task.
|
||||
#[derive(Debug)]
|
||||
pub struct PersistenceState {
|
||||
/// Hash and number of the highest block whose non-state/trie outputs are persisted.
|
||||
/// Hash and number of the last block persisted.
|
||||
///
|
||||
/// This tracks the highest canonical block with durable block/static-file/plain-state data.
|
||||
/// This tracks the chain height that is persisted on disk
|
||||
pub(crate) last_persisted_block: BlockNumHash,
|
||||
/// Hash and number of the highest block whose state/trie outputs are persisted.
|
||||
pub(crate) last_state_trie_persisted_block: BlockNumHash,
|
||||
/// Receiver end of channel where the result of the persistence task will be
|
||||
/// sent when done. A None value means there's no persistence task in progress.
|
||||
pub(crate) rx:
|
||||
@@ -77,18 +76,13 @@ impl PersistenceState {
|
||||
/// Sets state for a finished persistence task.
|
||||
pub(crate) fn finish(
|
||||
&mut self,
|
||||
last_persisted_block: BlockNumHash,
|
||||
last_state_trie_persisted_block: BlockNumHash,
|
||||
last_persisted_block_hash: B256,
|
||||
last_persisted_block_number: u64,
|
||||
) {
|
||||
trace!(
|
||||
target: "engine::tree",
|
||||
last_persisted_block = %last_persisted_block.number,
|
||||
last_state_trie_persisted_block = %last_state_trie_persisted_block.number,
|
||||
"updating persistence state"
|
||||
);
|
||||
trace!(target: "engine::tree", block= %last_persisted_block_number, hash=%last_persisted_block_hash, "updating persistence state");
|
||||
self.rx = None;
|
||||
self.last_persisted_block = last_persisted_block;
|
||||
self.last_state_trie_persisted_block = last_state_trie_persisted_block;
|
||||
self.last_persisted_block =
|
||||
BlockNumHash::new(last_persisted_block_number, last_persisted_block_hash);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use alloy_primitives::{
|
||||
map::{B256Map, B256Set},
|
||||
BlockNumber, B256,
|
||||
};
|
||||
use reth_chain_state::{EthPrimitives, ExecutedBlock, LazyOverlay};
|
||||
use reth_chain_state::{DeferredTrieData, EthPrimitives, ExecutedBlock, LazyOverlay};
|
||||
use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader};
|
||||
use std::{
|
||||
collections::{btree_map, hash_map, BTreeMap, VecDeque},
|
||||
@@ -43,7 +43,7 @@ pub struct TreeState<N: NodePrimitives = EthPrimitives> {
|
||||
/// This is optimistically prepared after the canonical head changes, so that
|
||||
/// the next payload building on the canonical head can use it immediately
|
||||
/// without recomputing.
|
||||
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay<N>>,
|
||||
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay>,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> TreeState<N> {
|
||||
@@ -106,10 +106,10 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// This should be called after the canonical head changes to optimistically
|
||||
/// prepare the overlay for the next payload that will likely build on it.
|
||||
///
|
||||
/// Returns a clone of the prepared overlay so the caller can spawn a background
|
||||
/// task to trigger computation via [`LazyOverlay::get`] for the cached anchor.
|
||||
/// This ensures the overlay is actually computed before the next payload arrives.
|
||||
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<PreparedCanonicalOverlay<N>> {
|
||||
/// Returns a clone of the [`LazyOverlay`] so the caller can spawn a background
|
||||
/// task to trigger computation via [`LazyOverlay::get`]. This ensures the overlay
|
||||
/// is actually computed before the next payload arrives.
|
||||
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<LazyOverlay> {
|
||||
let canonical_hash = self.current_canonical_head.hash;
|
||||
|
||||
// Get blocks leading to the canonical head
|
||||
@@ -119,23 +119,25 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
return None;
|
||||
};
|
||||
|
||||
let num_blocks = blocks.len();
|
||||
let prepared = PreparedCanonicalOverlay {
|
||||
// Extract deferred trie data handles from blocks (newest to oldest)
|
||||
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
|
||||
|
||||
let overlay = LazyOverlay::new(anchor_hash, handles);
|
||||
self.cached_canonical_overlay = Some(PreparedCanonicalOverlay {
|
||||
parent_hash: canonical_hash,
|
||||
overlay: LazyOverlay::new(blocks),
|
||||
overlay: overlay.clone(),
|
||||
anchor_hash,
|
||||
};
|
||||
self.cached_canonical_overlay = Some(prepared.clone());
|
||||
});
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
%canonical_hash,
|
||||
%anchor_hash,
|
||||
num_blocks,
|
||||
num_blocks = blocks.len(),
|
||||
"Prepared cached canonical overlay"
|
||||
);
|
||||
|
||||
Some(prepared)
|
||||
Some(overlay)
|
||||
}
|
||||
|
||||
/// Returns the cached overlay if it matches the requested parent hash and anchor.
|
||||
@@ -146,7 +148,7 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
&self,
|
||||
parent_hash: B256,
|
||||
expected_anchor: B256,
|
||||
) -> Option<&PreparedCanonicalOverlay<N>> {
|
||||
) -> Option<&PreparedCanonicalOverlay> {
|
||||
self.cached_canonical_overlay.as_ref().filter(|cached| {
|
||||
cached.parent_hash == parent_hash && cached.anchor_hash == expected_anchor
|
||||
})
|
||||
@@ -427,10 +429,10 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// the next payload (which typically builds on the canonical head) to reuse
|
||||
/// the pre-computed overlay immediately without re-traversing in-memory blocks.
|
||||
///
|
||||
/// The overlay captures executed blocks from all in-memory blocks
|
||||
/// The overlay captures deferred trie data handles from all in-memory blocks
|
||||
/// between the canonical head and the persisted anchor. When a new payload
|
||||
/// arrives building on the canonical head, this cached overlay can be used
|
||||
/// directly instead of calling `blocks_by_hash` again.
|
||||
/// directly instead of calling `blocks_by_hash` and collecting handles again.
|
||||
///
|
||||
/// # Invalidation
|
||||
///
|
||||
@@ -438,16 +440,16 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// - Persistence completes (anchor changes)
|
||||
/// - The canonical head changes to a different block
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PreparedCanonicalOverlay<N: NodePrimitives = EthPrimitives> {
|
||||
pub struct PreparedCanonicalOverlay {
|
||||
/// The block hash for which this overlay is prepared as a parent.
|
||||
///
|
||||
/// When a payload arrives with this parent hash, the overlay can be reused.
|
||||
pub parent_hash: B256,
|
||||
/// The pre-computed lazy overlay containing executed blocks for the canonical segment.
|
||||
/// The pre-computed lazy overlay containing deferred trie data handles.
|
||||
///
|
||||
/// This is computed optimistically after `set_canonical_head` so subsequent payloads don't
|
||||
/// need to walk the in-memory chain again.
|
||||
pub overlay: LazyOverlay<N>,
|
||||
/// This is computed optimistically after `set_canonical_head` so subsequent
|
||||
/// payloads don't need to re-collect the handles.
|
||||
pub overlay: LazyOverlay,
|
||||
/// The anchor hash (persisted ancestor) this overlay is based on.
|
||||
///
|
||||
/// Used to verify the overlay is still valid (anchor hasn't changed due to persistence).
|
||||
|
||||
@@ -222,11 +222,7 @@ impl TestHarness {
|
||||
engine_api_tree_state,
|
||||
canonical_in_memory_state,
|
||||
persistence_handle,
|
||||
PersistenceState {
|
||||
last_persisted_block: BlockNumHash::default(),
|
||||
last_state_trie_persisted_block: BlockNumHash::default(),
|
||||
rx: None,
|
||||
},
|
||||
PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None },
|
||||
payload_builder,
|
||||
tree_config,
|
||||
EngineApiKind::Ethereum,
|
||||
@@ -364,17 +360,6 @@ impl TestHarness {
|
||||
}
|
||||
}
|
||||
|
||||
type ExpectedPlanStep = (std::ops::Range<usize>, Option<std::ops::Range<usize>>, bool);
|
||||
|
||||
fn assert_plan_steps(plan: &SaveBlocksPlan<EthPrimitives>, expected: &[ExpectedPlanStep]) {
|
||||
assert_eq!(plan.steps.len(), expected.len());
|
||||
for (step, (block_range, masking_range, persist_rest)) in plan.steps.iter().zip(expected) {
|
||||
assert_eq!(&step.block_range, block_range);
|
||||
assert_eq!(&step.state_trie_masking_range, masking_range);
|
||||
assert_eq!(step.persist_rest, *persist_rest);
|
||||
}
|
||||
}
|
||||
|
||||
/// Simplified test metrics for validation calls
|
||||
#[derive(Debug, Default)]
|
||||
struct TestMetrics {
|
||||
@@ -569,16 +554,12 @@ async fn test_tree_persist_blocks() {
|
||||
|
||||
let received_action =
|
||||
test_harness.action_rx.recv().expect("Failed to receive save blocks action");
|
||||
if let PersistenceAction::SaveBlocks(plan, _) = received_action {
|
||||
if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action {
|
||||
// only blocks.len() - tree_config.memory_block_buffer_target() will be
|
||||
// persisted
|
||||
let expected_persist_len = blocks.len() - tree_config.memory_block_buffer_target() as usize;
|
||||
assert_eq!(plan.blocks.len(), expected_persist_len);
|
||||
assert_eq!(plan.blocks, blocks[..expected_persist_len]);
|
||||
assert_plan_steps(
|
||||
&plan,
|
||||
&[(0..expected_persist_len, Some(expected_persist_len..expected_persist_len), true)],
|
||||
);
|
||||
assert_eq!(saved_blocks.len(), expected_persist_len);
|
||||
assert_eq!(saved_blocks, blocks[..expected_persist_len]);
|
||||
} else {
|
||||
panic!("unexpected action received {received_action:?}");
|
||||
}
|
||||
@@ -723,8 +704,8 @@ fn test_backpressure_waits_for_persistence_before_reading_incoming() {
|
||||
test_harness.tree.config = test_harness
|
||||
.tree
|
||||
.config
|
||||
.with_persistence_threshold(1)
|
||||
.with_persistence_backpressure_threshold(2);
|
||||
.with_persistence_threshold(0)
|
||||
.with_persistence_backpressure_threshold(1);
|
||||
|
||||
let (persist_tx, persist_rx) = crossbeam_channel::bounded(1);
|
||||
let persisted = blocks.last().unwrap().recovered_block().num_hash();
|
||||
@@ -755,7 +736,6 @@ fn test_backpressure_waits_for_persistence_before_reading_incoming() {
|
||||
persist_tx
|
||||
.send(PersistenceResult {
|
||||
last_block: Some(persisted),
|
||||
last_state_trie_block: Some(persisted.number),
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
})
|
||||
.unwrap();
|
||||
@@ -790,10 +770,10 @@ async fn test_tree_state_on_new_head_reorg() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let chain_spec = MAINNET.clone();
|
||||
|
||||
// Keep a single block in memory while still leaving room for the persistence threshold.
|
||||
// Set persistence_threshold to 1
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
test_harness.tree.config =
|
||||
test_harness.tree.config.with_persistence_threshold(2).with_memory_block_buffer_target(1);
|
||||
test_harness.tree.config.with_persistence_threshold(1).with_memory_block_buffer_target(1);
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect();
|
||||
|
||||
@@ -844,16 +824,15 @@ async fn test_tree_state_on_new_head_reorg() {
|
||||
|
||||
// get rid of the prev action
|
||||
let received_action = test_harness.action_rx.recv().unwrap();
|
||||
let PersistenceAction::SaveBlocks(plan, sender) = received_action else {
|
||||
let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else {
|
||||
panic!("received wrong action");
|
||||
};
|
||||
assert_eq!(plan.blocks, vec![blocks[0].clone(), blocks[1].clone()]);
|
||||
assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]);
|
||||
|
||||
// send the response so we can advance again
|
||||
sender
|
||||
.send(PersistenceResult {
|
||||
last_block: Some(blocks[1].recovered_block().num_hash()),
|
||||
last_state_trie_block: Some(blocks[1].recovered_block().number()),
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
})
|
||||
.unwrap();
|
||||
@@ -989,10 +968,8 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
test_harness = test_harness.with_blocks(blocks.clone());
|
||||
|
||||
let last_persisted_block_number = 3;
|
||||
let last_persisted_block =
|
||||
test_harness.tree.persistence_state.last_persisted_block =
|
||||
blocks[last_persisted_block_number as usize].recovered_block.num_hash();
|
||||
test_harness.tree.persistence_state.last_persisted_block = last_persisted_block;
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block = last_persisted_block;
|
||||
|
||||
let persistence_threshold = 4;
|
||||
let memory_block_buffer_target = 3;
|
||||
@@ -1000,15 +977,16 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
.with_persistence_threshold(persistence_threshold)
|
||||
.with_memory_block_buffer_target(memory_block_buffer_target);
|
||||
|
||||
let plan = test_harness.tree.get_save_blocks_plan(PersistTarget::Threshold).unwrap();
|
||||
let blocks_to_persist =
|
||||
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
|
||||
|
||||
let expected_blocks_to_persist_length: usize =
|
||||
(canonical_head_number - memory_block_buffer_target - last_persisted_block_number)
|
||||
.try_into()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(plan.blocks.len(), expected_blocks_to_persist_length);
|
||||
for (i, item) in plan.blocks.iter().enumerate().take(expected_blocks_to_persist_length) {
|
||||
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
|
||||
for (i, item) in blocks_to_persist.iter().enumerate().take(expected_blocks_to_persist_length) {
|
||||
assert_eq!(item.recovered_block().number, last_persisted_block_number + i as u64 + 1);
|
||||
}
|
||||
|
||||
@@ -1019,14 +997,15 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
|
||||
assert!(test_harness.tree.state.tree_state.sealed_header_by_hash(&fork_block_hash).is_some());
|
||||
|
||||
let plan = test_harness.tree.get_save_blocks_plan(PersistTarget::Threshold).unwrap();
|
||||
assert_eq!(plan.blocks.len(), expected_blocks_to_persist_length);
|
||||
let blocks_to_persist =
|
||||
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
|
||||
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
|
||||
|
||||
// check that the fork block is not included in the blocks to persist
|
||||
assert!(!plan.blocks.iter().any(|b| b.recovered_block().hash() == fork_block_hash));
|
||||
assert!(!blocks_to_persist.iter().any(|b| b.recovered_block().hash() == fork_block_hash));
|
||||
|
||||
// check that the original block 4 is still included
|
||||
assert!(plan.blocks.iter().any(|b| b.recovered_block().number == 4 &&
|
||||
assert!(blocks_to_persist.iter().any(|b| b.recovered_block().number == 4 &&
|
||||
b.recovered_block().hash() == blocks[4].recovered_block().hash()));
|
||||
|
||||
// check that if we advance persistence, the persistence action is the correct value
|
||||
@@ -1034,193 +1013,11 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
assert_eq!(
|
||||
test_harness.tree.persistence_state.current_action().cloned(),
|
||||
Some(CurrentPersistenceAction::SavingBlocks {
|
||||
highest: plan.blocks.last().unwrap().recovered_block().num_hash()
|
||||
highest: blocks_to_persist.last().unwrap().recovered_block().num_hash()
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_save_blocks_plan_with_deferred_trie_blocks() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..7).collect();
|
||||
test_harness = test_harness.with_blocks(blocks.clone());
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block =
|
||||
blocks[1].recovered_block().num_hash();
|
||||
test_harness.tree.persistence_state.last_persisted_block =
|
||||
blocks[3].recovered_block().num_hash();
|
||||
test_harness.tree.config = TreeConfig::default()
|
||||
.with_persistence_threshold(4)
|
||||
.with_memory_block_buffer_target(1)
|
||||
.with_num_state_masking_blocks(2);
|
||||
|
||||
let plan = test_harness.tree.get_save_blocks_plan(PersistTarget::Threshold).unwrap();
|
||||
|
||||
assert_plan_steps(&plan, &[(0..2, Some(2..4), false), (2..4, None, true)]);
|
||||
assert_eq!(plan.blocks.len(), 4);
|
||||
assert_eq!(
|
||||
plan.blocks.iter().map(|block| block.recovered_block().number()).collect::<Vec<_>>(),
|
||||
vec![2, 3, 4, 5]
|
||||
);
|
||||
assert_eq!(plan.last_block(), Some(blocks[5].recovered_block().num_hash()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_save_blocks_plan_persists_full_region_before_deferred_tail() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..31).collect();
|
||||
test_harness = test_harness.with_blocks(blocks.clone());
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block =
|
||||
blocks[12].recovered_block().num_hash();
|
||||
test_harness.tree.persistence_state.last_persisted_block =
|
||||
blocks[15].recovered_block().num_hash();
|
||||
test_harness.tree.config = TreeConfig::default()
|
||||
.with_persistence_threshold(5)
|
||||
.with_memory_block_buffer_target(2)
|
||||
.with_num_state_masking_blocks(2);
|
||||
|
||||
let plan = test_harness.tree.get_save_blocks_plan(PersistTarget::Threshold).unwrap();
|
||||
|
||||
assert_plan_steps(
|
||||
&plan,
|
||||
&[(0..3, Some(14..16), false), (3..14, Some(14..16), true), (14..16, None, true)],
|
||||
);
|
||||
assert_eq!(plan.blocks.len(), 16);
|
||||
assert_eq!(
|
||||
plan.blocks.iter().map(|block| block.recovered_block().number()).collect::<Vec<_>>(),
|
||||
(13..=28).collect::<Vec<_>>()
|
||||
);
|
||||
assert_eq!(plan.last_block(), Some(blocks[28].recovered_block().num_hash()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_on_persistence_complete_retains_blocks_above_partial_state_trie() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..7).collect();
|
||||
test_harness = test_harness.with_blocks(blocks.clone());
|
||||
test_harness.tree.persistence_state.last_persisted_block =
|
||||
blocks[1].recovered_block().num_hash();
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block =
|
||||
blocks[1].recovered_block().num_hash();
|
||||
|
||||
let persisted_tip = blocks[5].recovered_block().num_hash();
|
||||
let last_state_trie_block = blocks[3].recovered_block().number();
|
||||
|
||||
test_harness
|
||||
.tree
|
||||
.on_persistence_complete(
|
||||
PersistenceResult {
|
||||
last_block: Some(persisted_tip),
|
||||
last_state_trie_block: Some(last_state_trie_block),
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
},
|
||||
Instant::now(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted_tip);
|
||||
assert_eq!(
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block,
|
||||
blocks[3].recovered_block().num_hash()
|
||||
);
|
||||
assert_eq!(
|
||||
test_harness.tree.canonical_in_memory_state.get_persisted_num_hash(),
|
||||
Some(persisted_tip)
|
||||
);
|
||||
|
||||
for block in &blocks[..=last_state_trie_block as usize] {
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.state
|
||||
.tree_state
|
||||
.executed_block_by_hash(block.recovered_block().hash())
|
||||
.is_none());
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.canonical_in_memory_state
|
||||
.state_by_number(block.recovered_block().number())
|
||||
.is_none());
|
||||
}
|
||||
|
||||
for block in &blocks[last_state_trie_block as usize + 1..] {
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.state
|
||||
.tree_state
|
||||
.executed_block_by_hash(block.recovered_block().hash())
|
||||
.is_some());
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.canonical_in_memory_state
|
||||
.state_by_number(block.recovered_block().number())
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_on_persistence_complete_without_partial_state_trie_prunes_through_tip() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
|
||||
let blocks: Vec<_> = test_block_builder.get_executed_blocks(0..7).collect();
|
||||
test_harness = test_harness.with_blocks(blocks.clone());
|
||||
test_harness.tree.persistence_state.last_persisted_block =
|
||||
blocks[1].recovered_block().num_hash();
|
||||
test_harness.tree.persistence_state.last_state_trie_persisted_block =
|
||||
blocks[1].recovered_block().num_hash();
|
||||
|
||||
let persisted_tip = blocks[5].recovered_block().num_hash();
|
||||
|
||||
test_harness
|
||||
.tree
|
||||
.on_persistence_complete(
|
||||
PersistenceResult {
|
||||
last_block: Some(persisted_tip),
|
||||
last_state_trie_block: None,
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
},
|
||||
Instant::now(),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
for block in &blocks[..=persisted_tip.number as usize] {
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.state
|
||||
.tree_state
|
||||
.executed_block_by_hash(block.recovered_block().hash())
|
||||
.is_none());
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.canonical_in_memory_state
|
||||
.state_by_number(block.recovered_block().number())
|
||||
.is_none());
|
||||
}
|
||||
|
||||
for block in &blocks[persisted_tip.number as usize + 1..] {
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.state
|
||||
.tree_state
|
||||
.executed_block_by_hash(block.recovered_block().hash())
|
||||
.is_some());
|
||||
assert!(test_harness
|
||||
.tree
|
||||
.canonical_in_memory_state
|
||||
.state_by_number(block.recovered_block().number())
|
||||
.is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_missing_head() {
|
||||
let chain_spec = MAINNET.clone();
|
||||
@@ -2315,18 +2112,15 @@ mod forkchoice_updated_tests {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(PersistenceAction::SaveBlocks(plan, sender)) =
|
||||
if let Ok(PersistenceAction::SaveBlocks(saved_blocks, sender)) =
|
||||
action_rx.recv_timeout(std::time::Duration::from_millis(100))
|
||||
{
|
||||
if let Some(last) = plan.last_block() {
|
||||
last_persisted_number = last.number;
|
||||
} else if let Some(last) = plan.blocks.last() {
|
||||
if let Some(last) = saved_blocks.last() {
|
||||
last_persisted_number = last.recovered_block().number;
|
||||
}
|
||||
sender
|
||||
.send(PersistenceResult {
|
||||
last_block: plan.last_block(),
|
||||
last_state_trie_block: plan.last_block().map(|tip| tip.number),
|
||||
last_block: saved_blocks.last().map(|b| b.recovered_block().num_hash()),
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
@@ -287,7 +287,7 @@ where
|
||||
let tx_recovered =
|
||||
tx.try_into_recovered().map_err(|_| ProviderError::SenderRecoveryError)?;
|
||||
let gas_used = match builder.execute_transaction(tx_recovered) {
|
||||
Ok(gas_used) => gas_used.tx_gas_used(),
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
hash,
|
||||
error,
|
||||
|
||||
@@ -48,7 +48,7 @@ tokio.workspace = true
|
||||
|
||||
# revm with required ethereum features
|
||||
# Note: this must be kept to ensure all features are properly enabled/forwarded
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit", "p256-aws-lc-rs"] }
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] }
|
||||
|
||||
# misc
|
||||
eyre.workspace = true
|
||||
|
||||
@@ -342,7 +342,7 @@ where
|
||||
let tx_hash = *tx.tx_hash();
|
||||
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
Ok(gas_used) => gas_used.tx_gas_used(),
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
error, ..
|
||||
})) => {
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor};
|
||||
use alloc::{boxed::Box, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::{BlockHeader, Header};
|
||||
use alloy_eips::eip2718::WithEncoded;
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory, GasOutput};
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
|
||||
use alloy_evm::{
|
||||
block::{CommitChanges, ExecutableTxParts},
|
||||
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
|
||||
@@ -327,7 +327,7 @@ pub trait BlockBuilder {
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result) -> CommitChanges,
|
||||
) -> Result<Option<GasOutput>, BlockExecutionError>;
|
||||
) -> Result<Option<u64>, BlockExecutionError>;
|
||||
|
||||
/// Invokes [`BlockExecutor::execute_transaction_with_result_closure`] and saves the
|
||||
/// transaction in internal state.
|
||||
@@ -335,7 +335,7 @@ pub trait BlockBuilder {
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result),
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
self.execute_transaction_with_commit_condition(tx, |res| {
|
||||
f(res);
|
||||
CommitChanges::Yes
|
||||
@@ -348,7 +348,7 @@ pub trait BlockBuilder {
|
||||
fn execute_transaction(
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
self.execute_transaction_with_result_closure(tx, |_| ())
|
||||
}
|
||||
|
||||
@@ -460,13 +460,13 @@ where
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result) -> CommitChanges,
|
||||
) -> Result<Option<GasOutput>, BlockExecutionError> {
|
||||
) -> Result<Option<u64>, BlockExecutionError> {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
if let Some(gas_used) =
|
||||
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
|
||||
{
|
||||
self.transactions.push(tx);
|
||||
Ok(Some(gas_used))
|
||||
Ok(Some(gas_used.tx_gas_used()))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -225,7 +225,7 @@ impl Discv5 {
|
||||
bootstrap_lookup_interval,
|
||||
bootstrap_lookup_countdown,
|
||||
metrics.clone(),
|
||||
Arc::downgrade(&discv5),
|
||||
discv5.clone(),
|
||||
);
|
||||
|
||||
Ok((
|
||||
@@ -573,19 +573,14 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
bootstrap_lookup_interval: u64,
|
||||
bootstrap_lookup_countdown: u64,
|
||||
metrics: Discv5Metrics,
|
||||
discv5: std::sync::Weak<discv5::Discv5>,
|
||||
discv5: Arc<discv5::Discv5>,
|
||||
) {
|
||||
let local_node_id = discv5.local_enr().node_id();
|
||||
let lookup_interval = Duration::from_secs(lookup_interval);
|
||||
let metrics = metrics.discovered_peers;
|
||||
let mut kbucket_index = MAX_KBUCKET_INDEX;
|
||||
let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval);
|
||||
task::spawn(async move {
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let local_node_id = discv5_handle.local_enr().node_id();
|
||||
drop(discv5_handle);
|
||||
|
||||
// make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest
|
||||
// log2distance from local node
|
||||
for i in (0..bootstrap_lookup_countdown).rev() {
|
||||
@@ -598,12 +593,7 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting bootstrap boost lookup query"
|
||||
);
|
||||
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
|
||||
tokio::time::sleep(pulse_lookup_interval).await;
|
||||
}
|
||||
@@ -620,12 +610,7 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting periodic lookup query"
|
||||
);
|
||||
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
|
||||
if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX {
|
||||
// try to populate bucket one step closer
|
||||
@@ -713,10 +698,6 @@ mod test {
|
||||
use ::enr::{CombinedKey, EnrKey};
|
||||
use rand_08::thread_rng;
|
||||
use reth_chainspec::MAINNET;
|
||||
use std::{
|
||||
net::UdpSocket,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::trace;
|
||||
|
||||
fn discv5_noop() -> Discv5 {
|
||||
@@ -755,61 +736,6 @@ mod test {
|
||||
Discv5::start(&secret_key, discv5_config).await.expect("should build discv5")
|
||||
}
|
||||
|
||||
async fn start_discovery_node_with_key(
|
||||
secret_key: &SecretKey,
|
||||
udp_port_discv5: u16,
|
||||
) -> Result<(Discv5, mpsc::Receiver<discv5::Event>), Error> {
|
||||
let discv5_addr: SocketAddr = format!("127.0.0.1:{udp_port_discv5}").parse().unwrap();
|
||||
let rlpx_addr: SocketAddr = "127.0.0.1:30303".parse().unwrap();
|
||||
|
||||
let discv5_listen_config = ListenConfig::from(discv5_addr);
|
||||
let discv5_config = Config::builder(rlpx_addr)
|
||||
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
|
||||
.build();
|
||||
|
||||
Discv5::start(secret_key, discv5_config).await
|
||||
}
|
||||
|
||||
fn unused_udp_port() -> u16 {
|
||||
UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
|
||||
}
|
||||
|
||||
async fn wait_for_udp_port_release(port: u16, timeout: Duration) {
|
||||
let deadline = Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
match UdpSocket::bind(("127.0.0.1", port)) {
|
||||
Ok(socket) => {
|
||||
drop(socket);
|
||||
return;
|
||||
}
|
||||
Err(err) if Instant::now() < deadline => {
|
||||
trace!(target: "net::discv5::test", %port, %err, "waiting for discv5 port release");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => panic!("discv5 did not release port {port} before timeout: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5_releases_port_on_drop() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let secret_key = SecretKey::new(&mut thread_rng());
|
||||
let port = unused_udp_port();
|
||||
|
||||
let (node, updates) =
|
||||
start_discovery_node_with_key(&secret_key, port).await.expect("should start discv5");
|
||||
drop(updates);
|
||||
drop(node);
|
||||
|
||||
wait_for_udp_port_release(port, Duration::from_secs(1)).await;
|
||||
|
||||
let restarted = start_discovery_node_with_key(&secret_key, port).await;
|
||||
assert!(restarted.is_ok(), "discv5 failed to rebind dropped port: {restarted:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
use crate::{
|
||||
errors::{EthHandshakeError, EthStreamError},
|
||||
handshake::EthereumEthHandshake,
|
||||
message::{EthBroadcastMessage, EthMessageID, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
|
||||
p2pstream::HANDSHAKE_TIMEOUT,
|
||||
CanDisconnect, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, ProtocolMessage,
|
||||
UnifiedStatus,
|
||||
@@ -108,9 +108,6 @@ pub struct EthStreamInner<N> {
|
||||
version: EthVersion,
|
||||
/// Maximum allowed ETH message size.
|
||||
max_message_size: usize,
|
||||
/// When true, `NewBlock` (0x07) and `NewBlockHashes` (0x01) messages are rejected before RLP
|
||||
/// decoding to avoid any memory impact for non-PoW networks.
|
||||
reject_block_announcements: bool,
|
||||
_pd: std::marker::PhantomData<N>,
|
||||
}
|
||||
|
||||
@@ -125,12 +122,7 @@ where
|
||||
|
||||
/// Creates a new [`EthStreamInner`] with the given eth version and message size limit.
|
||||
pub const fn with_max_message_size(version: EthVersion, max_message_size: usize) -> Self {
|
||||
Self {
|
||||
version,
|
||||
max_message_size,
|
||||
reject_block_announcements: false,
|
||||
_pd: std::marker::PhantomData,
|
||||
}
|
||||
Self { version, max_message_size, _pd: std::marker::PhantomData }
|
||||
}
|
||||
|
||||
/// Returns the eth version
|
||||
@@ -139,25 +131,12 @@ where
|
||||
self.version
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.reject_block_announcements = reject;
|
||||
}
|
||||
|
||||
/// Decodes incoming bytes into an [`EthMessage`].
|
||||
pub fn decode_message(&self, bytes: BytesMut) -> Result<EthMessage<N>, EthStreamError> {
|
||||
if bytes.len() > self.max_message_size {
|
||||
return Err(EthStreamError::MessageTooBig(bytes.len()));
|
||||
}
|
||||
|
||||
if self.reject_block_announcements &&
|
||||
let Some(&id) = bytes.first() &&
|
||||
(id == EthMessageID::NewBlock.to_u8() || id == EthMessageID::NewBlockHashes.to_u8())
|
||||
{
|
||||
return Err(EthStreamError::UnsupportedMessage { message_id: id });
|
||||
}
|
||||
|
||||
let msg = match ProtocolMessage::decode_message(self.version, &mut bytes.as_ref()) {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
@@ -229,12 +208,6 @@ impl<S, N: NetworkPrimitives> EthStream<S, N> {
|
||||
self.eth.version()
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.eth.set_reject_block_announcements(reject);
|
||||
}
|
||||
|
||||
/// Returns the underlying stream.
|
||||
#[inline]
|
||||
pub const fn inner(&self) -> &S {
|
||||
|
||||
@@ -13,7 +13,6 @@ use crate::{
|
||||
};
|
||||
use reth_eth_wire::{EthNetworkPrimitives, NetworkPrimitives};
|
||||
use reth_network_api::test_utils::PeersHandleProvider;
|
||||
use reth_storage_api::BalProvider;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -64,10 +63,7 @@ impl<Tx, Eth, N: NetworkPrimitives> NetworkBuilder<Tx, Eth, N> {
|
||||
pub fn request_handler<Client>(
|
||||
self,
|
||||
client: Client,
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N>
|
||||
where
|
||||
Client: BalProvider,
|
||||
{
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N> {
|
||||
let Self { mut network, transactions, .. } = self;
|
||||
let (tx, rx) = mpsc::channel(ETH_REQUEST_CHANNEL_CAPACITY);
|
||||
network.set_eth_request_handler(tx);
|
||||
|
||||
@@ -20,9 +20,7 @@ use reth_eth_wire_types::message::MAX_MESSAGE_SIZE;
|
||||
use reth_ethereum_forks::{ForkFilter, Head};
|
||||
use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer};
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_storage_api::{
|
||||
noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider,
|
||||
};
|
||||
use reth_storage_api::{noop::NoopProvider, BlockNumReader, BlockReader, HeaderProvider};
|
||||
use reth_tasks::Runtime;
|
||||
use secp256k1::SECP256K1;
|
||||
use std::{collections::HashSet, net::SocketAddr, sync::Arc};
|
||||
@@ -159,8 +157,7 @@ where
|
||||
impl<C, N> NetworkConfig<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
+ HeaderProvider
|
||||
+ Clone
|
||||
+ Unpin
|
||||
|
||||
@@ -18,7 +18,7 @@ use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::error::RequestResult;
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_primitives_traits::Block;
|
||||
use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider};
|
||||
use reth_storage_api::{BlockReader, HeaderProvider};
|
||||
use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
@@ -282,6 +282,27 @@ where
|
||||
let _ = response.send(Ok(Receipts70 { last_block_incomplete, receipts }));
|
||||
}
|
||||
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
|
||||
/// <https://eips.ethereum.org/EIPS/eip-8159>
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
// TODO: BAL serving is not fully implemented yet. Per EIP-8159, unavailable BALs are
|
||||
// returned as empty BAL entries while preserving request order, so we currently return
|
||||
// one RLP-encoded empty BAL (`0xc0`) per requested hash.
|
||||
let access_lists = request
|
||||
.0
|
||||
.into_iter()
|
||||
.map(|_| Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]))
|
||||
.collect();
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_receipts_response<T, F>(&self, request: GetReceipts, transform_fn: F) -> Vec<Vec<T>>
|
||||
where
|
||||
@@ -311,55 +332,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, N> EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider,
|
||||
{
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
|
||||
/// <https://eips.ethereum.org/EIPS/eip-8159>
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit(SOFT_RESPONSE_LIMIT);
|
||||
let access_lists = self
|
||||
.client
|
||||
.bal_store()
|
||||
.get_by_hashes_with_limit(&request.0, limit)
|
||||
.unwrap_or_else(|_| empty_block_access_lists_with_limit(request.0.len(), limit));
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the error fallback response while still enforcing the BAL response soft limit.
|
||||
fn empty_block_access_lists_with_limit(count: usize, limit: GetBlockAccessListLimit) -> Vec<Bytes> {
|
||||
let mut out = Vec::with_capacity(count);
|
||||
let mut size = 0;
|
||||
for _ in 0..count {
|
||||
let bal = Bytes::from_static(&[0xc0]);
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An endless future.
|
||||
///
|
||||
/// This should be spawned or used as part of `tokio::select!`.
|
||||
impl<C, N> Future for EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
+ HeaderProvider<Header = N::BlockHeader>
|
||||
+ Unpin,
|
||||
{
|
||||
|
||||
@@ -6,7 +6,7 @@ use futures::{future, future::Either};
|
||||
use reth_eth_wire::{BlockAccessLists, EthNetworkPrimitives, NetworkPrimitives};
|
||||
use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::{
|
||||
block_access_lists::client::{BalRequirement, BlockAccessListsClient},
|
||||
block_access_lists::client::BlockAccessListsClient,
|
||||
bodies::client::{BodiesClient, BodiesFut},
|
||||
download::DownloadClient,
|
||||
error::{PeerRequestResult, RequestError},
|
||||
@@ -135,29 +135,11 @@ impl<N: NetworkPrimitives> BlockAccessListsClient for FetchClient<N> {
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
) -> Self::Output {
|
||||
self.get_block_access_lists_with_priority_and_requirement(
|
||||
hashes,
|
||||
priority,
|
||||
BalRequirement::Mandatory,
|
||||
)
|
||||
}
|
||||
|
||||
fn get_block_access_lists_with_priority_and_requirement(
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
requirement: BalRequirement,
|
||||
) -> Self::Output {
|
||||
let (response, rx) = oneshot::channel();
|
||||
if self
|
||||
.request_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: hashes,
|
||||
response,
|
||||
priority,
|
||||
requirement,
|
||||
})
|
||||
.send(DownloadRequest::GetBlockAccessLists { request: hashes, response, priority })
|
||||
.is_ok()
|
||||
{
|
||||
Box::pin(FlattenedResponse::from(rx))
|
||||
|
||||
@@ -13,7 +13,6 @@ use reth_eth_wire::{
|
||||
};
|
||||
use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::{
|
||||
block_access_lists::client::BalRequirement,
|
||||
error::{EthResponseValidator, PeerRequestResult, RequestError, RequestResult},
|
||||
headers::client::HeadersRequest,
|
||||
priority::Priority,
|
||||
@@ -160,10 +159,15 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
/// full history available
|
||||
fn next_best_peer(&self, requirement: BestPeerRequirements) -> Option<PeerId> {
|
||||
// filter out peers that aren't idle or don't meet the requirement
|
||||
let mut idle = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(_, peer)| peer.state.is_idle() && peer.satisfies(&requirement));
|
||||
let mut idle = self.peers.iter().filter(|(_, peer)| {
|
||||
peer.state.is_idle() &&
|
||||
match &requirement {
|
||||
BestPeerRequirements::EthVersion(ver) => {
|
||||
peer.capabilities.supports_eth_at_least(ver)
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
});
|
||||
|
||||
let mut best_peer = idle.next()?;
|
||||
|
||||
@@ -191,14 +195,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
Some(*best_peer.0)
|
||||
}
|
||||
|
||||
/// Returns whether any connected peer can serve BAL requests.
|
||||
fn has_eth71_peer(&self) -> bool {
|
||||
self.peers.values().any(|peer| {
|
||||
!matches!(peer.state, PeerState::Closing) &&
|
||||
peer.capabilities.supports_eth_at_least(&EthVersion::Eth71)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the next action to return
|
||||
fn poll_action(&mut self) -> PollAction {
|
||||
// we only check and not pop here since we don't know yet whether a peer is available.
|
||||
@@ -212,15 +208,9 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
|
||||
let request = self.queued_requests.pop_front().expect("not empty");
|
||||
let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else {
|
||||
// Optional BAL requests can lose their eth/71 peer while queued; complete them
|
||||
// instead of waiting for future peer churn.
|
||||
if request.is_optional_bal() && !self.has_eth71_peer() {
|
||||
request.send_err_response(RequestError::UnsupportedCapability);
|
||||
} else {
|
||||
// no peer matches this request's requirements; requeue at the back so other
|
||||
// queued requests get a chance on the next poll instead of head-of-line blocking.
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
// no peer matches this request's requirements; requeue at the back so other
|
||||
// queued requests get a chance on the next poll instead of head-of-line blocking.
|
||||
self.queued_requests.push_back(request);
|
||||
return PollAction::NoPeersAvailable
|
||||
};
|
||||
|
||||
@@ -242,30 +232,21 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
loop {
|
||||
// poll incoming requests
|
||||
match self.download_requests_rx.poll_next_unpin(cx) {
|
||||
Poll::Ready(Some(request)) => {
|
||||
// Optional BAL requests should not wait for future peer churn if no
|
||||
// connected peer can serve them right now.
|
||||
if request.is_optional_bal() && !self.has_eth71_peer() {
|
||||
request.send_err_response(RequestError::UnsupportedCapability);
|
||||
continue
|
||||
Poll::Ready(Some(request)) => match request.get_priority() {
|
||||
Priority::High => {
|
||||
// find the first normal request and queue before, add this request to
|
||||
// the back of the high-priority queue
|
||||
let pos = self
|
||||
.queued_requests
|
||||
.iter()
|
||||
.position(|req| req.is_normal_priority())
|
||||
.unwrap_or(0);
|
||||
self.queued_requests.insert(pos, request);
|
||||
}
|
||||
|
||||
match request.get_priority() {
|
||||
Priority::High => {
|
||||
// find first normal request and queue before it; add this request
|
||||
// to the back of the high-priority queue
|
||||
let pos = self
|
||||
.queued_requests
|
||||
.iter()
|
||||
.position(|req| req.is_normal_priority())
|
||||
.unwrap_or(0);
|
||||
self.queued_requests.insert(pos, request);
|
||||
}
|
||||
Priority::Normal => {
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
Priority::Normal => {
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
}
|
||||
},
|
||||
Poll::Ready(None) => {
|
||||
unreachable!("channel can't close")
|
||||
}
|
||||
@@ -288,15 +269,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
peer.state = req.peer_state();
|
||||
}
|
||||
|
||||
self.prepare_inflight_block_request(peer_id, req)
|
||||
}
|
||||
|
||||
/// Tracks an inflight request and converts it into a peer request.
|
||||
fn prepare_inflight_block_request(
|
||||
&mut self,
|
||||
peer_id: PeerId,
|
||||
req: DownloadRequest<N>,
|
||||
) -> BlockRequest {
|
||||
match req {
|
||||
DownloadRequest::GetBlockHeaders { request, response, .. } => {
|
||||
let inflight = Request { request: request.clone(), response };
|
||||
@@ -327,23 +299,12 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a queued followup request the peer can serve.
|
||||
///
|
||||
/// This is an immediate scheduling shortcut after a successful response. It skips queued
|
||||
/// requests whose hard requirements do not match this peer, leaving them for the regular peer
|
||||
/// selection path.
|
||||
/// Returns a new followup request for the peer.
|
||||
///
|
||||
/// Caution: this expects that the peer is _not_ closed.
|
||||
fn followup_request(&mut self, peer_id: PeerId) -> Option<BlockResponseOutcome> {
|
||||
let peer = self.peers.get_mut(&peer_id)?;
|
||||
let req_idx = self.queued_requests.iter().position(|req| {
|
||||
// Find the first queued request this peer can serve.
|
||||
peer.satisfies(&req.best_peer_requirements())
|
||||
})?;
|
||||
let req = self.queued_requests.remove(req_idx).expect("valid request index");
|
||||
|
||||
peer.state = req.peer_state();
|
||||
let req = self.prepare_inflight_block_request(peer_id, req);
|
||||
let req = self.queued_requests.pop_front()?;
|
||||
let req = self.prepare_block_request(peer_id, req);
|
||||
Some(BlockResponseOutcome::Request(peer_id, req))
|
||||
}
|
||||
|
||||
@@ -515,16 +476,6 @@ impl Peer {
|
||||
self.range_info.as_ref().map(|info| info.range())
|
||||
}
|
||||
|
||||
/// Returns whether this peer can serve requests with the given hard requirements.
|
||||
fn satisfies(&self, requirement: &BestPeerRequirements) -> bool {
|
||||
match requirement {
|
||||
BestPeerRequirements::EthVersion(ver) => self.capabilities.supports_eth_at_least(ver),
|
||||
BestPeerRequirements::None |
|
||||
BestPeerRequirements::FullBlock |
|
||||
BestPeerRequirements::FullBlockRange(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this peer has a better range than the other peer for serving the requested
|
||||
/// range.
|
||||
///
|
||||
@@ -651,7 +602,6 @@ pub(crate) enum DownloadRequest<N: NetworkPrimitives> {
|
||||
request: Vec<B256>,
|
||||
response: oneshot::Sender<PeerRequestResult<BlockAccessLists>>,
|
||||
priority: Priority,
|
||||
requirement: BalRequirement,
|
||||
},
|
||||
/// Download receipts for the given block hashes and send response through channel
|
||||
GetReceipts {
|
||||
@@ -689,21 +639,6 @@ impl<N: NetworkPrimitives> DownloadRequest<N> {
|
||||
self.get_priority().is_normal()
|
||||
}
|
||||
|
||||
/// Returns `true` if this is an optional BAL request.
|
||||
const fn is_optional_bal(&self) -> bool {
|
||||
matches!(self, Self::GetBlockAccessLists { requirement: BalRequirement::Optional, .. })
|
||||
}
|
||||
|
||||
/// Sends an error response to the waiting caller.
|
||||
fn send_err_response(self, err: RequestError) {
|
||||
let _ = match self {
|
||||
Self::GetBlockHeaders { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetBlockBodies { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetReceipts { response, .. } => response.send(Err(err)).ok(),
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the best peer requirements for this request.
|
||||
fn best_peer_requirements(&self) -> BestPeerRequirements {
|
||||
match self {
|
||||
@@ -1469,98 +1404,6 @@ mod tests {
|
||||
assert!(matches!(outcome, Some(BlockResponseOutcome::Request(pid, _)) if pid == peer_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_followup_skips_request_peer_cannot_serve() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (followup_tx, _followup_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![B256::random()],
|
||||
response: followup_tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
});
|
||||
|
||||
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
|
||||
|
||||
let resp = ReceiptsResponse::new(vec![vec![]]);
|
||||
assert!(fetcher.on_receipts_response(peer_id, Ok(resp)).is_none());
|
||||
assert!(fetcher.peers[&peer_id].state.is_idle());
|
||||
assert!(!fetcher.inflight_bals_requests.contains_key(&peer_id));
|
||||
assert!(matches!(
|
||||
fetcher.queued_requests.front(),
|
||||
Some(DownloadRequest::GetBlockAccessLists {
|
||||
requirement: BalRequirement::Optional,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_followup_uses_first_satisfiable_request() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (bal_tx, _bal_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![B256::random()],
|
||||
response: bal_tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
});
|
||||
|
||||
let (bodies_tx, _bodies_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockBodies {
|
||||
request: vec![B256::random()],
|
||||
response: bodies_tx,
|
||||
priority: Priority::Normal,
|
||||
range_hint: None,
|
||||
});
|
||||
|
||||
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
|
||||
|
||||
let resp = ReceiptsResponse::new(vec![vec![]]);
|
||||
let outcome = fetcher.on_receipts_response(peer_id, Ok(resp));
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
Some(BlockResponseOutcome::Request(pid, BlockRequest::GetBlockBodies(_))) if pid == peer_id
|
||||
));
|
||||
assert!(fetcher.inflight_bodies_requests.contains_key(&peer_id));
|
||||
assert!(matches!(fetcher.peers[&peer_id].state, PeerState::GetBlockBodies));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
assert!(matches!(
|
||||
fetcher.queued_requests.front(),
|
||||
Some(DownloadRequest::GetBlockAccessLists {
|
||||
requirement: BalRequirement::Optional,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prepare_block_request_creates_inflight_receipts() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
@@ -1698,7 +1541,6 @@ mod tests {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Mandatory,
|
||||
});
|
||||
|
||||
let waker = noop_waker();
|
||||
@@ -1741,138 +1583,4 @@ mod tests {
|
||||
assert_eq!(peer_id, peer_71);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optional_bal_request_rejected_without_eth71_peer() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_old = B512::random();
|
||||
let caps_old = Arc::new(Capabilities::new(vec![]));
|
||||
fetcher.new_active_peer(
|
||||
peer_old,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_old,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert!(fetcher.queued_requests.is_empty());
|
||||
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optional_bal_request_waits_for_busy_eth71_peer() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (tx, _rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queued_optional_bal_request_rejected_after_eth71_disconnect() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_old = B512::random();
|
||||
let caps_old = Arc::new(Capabilities::new(vec![]));
|
||||
fetcher.new_active_peer(
|
||||
peer_old,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_old,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
|
||||
fetcher.on_session_closed(&peer_71);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert!(fetcher.queued_requests.is_empty());
|
||||
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,7 +318,6 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
extra_protocols,
|
||||
handshake,
|
||||
eth_max_message_size,
|
||||
network_mode.is_stake(),
|
||||
);
|
||||
|
||||
let state = NetworkState::new(
|
||||
|
||||
@@ -79,9 +79,6 @@ const TIMEOUT_SCALING: u32 = 3;
|
||||
/// before reading any more messages from the remote peer, throttling the peer.
|
||||
const MAX_QUEUED_OUTGOING_RESPONSES: usize = 4;
|
||||
|
||||
/// Minimum capacity to retain for buffered incoming requests from the remote peer.
|
||||
const MIN_RECEIVED_REQUESTS_CAPACITY: usize = 1;
|
||||
|
||||
/// Soft limit for the total number of buffered outgoing broadcast items (e.g. transaction hashes).
|
||||
///
|
||||
/// Many small broadcast messages carrying a single tx hash each are equivalent in cost to one
|
||||
@@ -207,8 +204,8 @@ impl<N: NetworkPrimitives> ActiveSession<N> {
|
||||
|
||||
/// Shrinks the capacity of the internal buffers.
|
||||
pub fn shrink_to_fit(&mut self) {
|
||||
self.received_requests_from_remote.shrink_to(MIN_RECEIVED_REQUESTS_CAPACITY);
|
||||
self.queued_outgoing.shrink_to(MAX_QUEUED_OUTGOING_RESPONSES);
|
||||
self.received_requests_from_remote.shrink_to_fit();
|
||||
self.queued_outgoing.shrink_to_fit();
|
||||
}
|
||||
|
||||
/// Returns how many responses we've currently queued up.
|
||||
@@ -1093,8 +1090,8 @@ impl<N: NetworkPrimitives> QueuedOutgoingMessages<N> {
|
||||
self.count.increment(1);
|
||||
}
|
||||
|
||||
pub(crate) fn shrink_to(&mut self, min_capacity: usize) {
|
||||
self.messages.shrink_to(min_capacity);
|
||||
pub(crate) fn shrink_to_fit(&mut self) {
|
||||
self.messages.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -93,15 +93,6 @@ impl<N: NetworkPrimitives> EthRlpxConnection<N> {
|
||||
Self::Satellite(conn) => conn.primary_mut().start_send_raw(msg),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding to avoid memory amplification from deserializing blocks that will be discarded.
|
||||
pub fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
match self {
|
||||
Self::EthOnly(conn) => conn.set_reject_block_announcements(reject),
|
||||
Self::Satellite(conn) => conn.primary_mut().set_reject_block_announcements(reject),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NetworkPrimitives> From<EthPeerConnection<N>> for EthRlpxConnection<N> {
|
||||
|
||||
@@ -123,9 +123,6 @@ pub struct SessionManager<N: NetworkPrimitives> {
|
||||
/// Shared local range information that gets propagated to active sessions.
|
||||
/// This represents the range of blocks that this node can serve to other peers.
|
||||
local_range_info: BlockRangeInfo,
|
||||
/// When true, block announcement messages (`NewBlock`, `NewBlockHashes`) are rejected before
|
||||
/// RLP decoding on new sessions to avoid memory amplification.
|
||||
reject_block_announcements: bool,
|
||||
}
|
||||
|
||||
// === impl SessionManager ===
|
||||
@@ -143,7 +140,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
extra_protocols: RlpxSubProtocols,
|
||||
handshake: Arc<dyn EthRlpxHandshake>,
|
||||
eth_max_message_size: usize,
|
||||
reject_block_announcements: bool,
|
||||
) -> Self {
|
||||
let (pending_sessions_tx, pending_sessions_rx) = mpsc::channel(config.session_event_buffer);
|
||||
let (active_session_tx, active_session_rx) = mpsc::channel(config.session_event_buffer);
|
||||
@@ -180,7 +176,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
handshake,
|
||||
eth_max_message_size,
|
||||
local_range_info,
|
||||
reject_block_announcements,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -501,7 +496,7 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
local_addr,
|
||||
peer_id,
|
||||
capabilities,
|
||||
mut conn,
|
||||
conn,
|
||||
status,
|
||||
direction,
|
||||
client_id,
|
||||
@@ -568,10 +563,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
BlockRangeInfo::new(update.earliest, update.latest, update.latest_hash)
|
||||
});
|
||||
|
||||
if self.reject_block_announcements {
|
||||
conn.set_reject_block_announcements(true);
|
||||
}
|
||||
|
||||
let session = ActiveSession {
|
||||
next_id: 0,
|
||||
remote_peer_id: peer_id,
|
||||
|
||||
@@ -27,8 +27,7 @@ use reth_network_api::{
|
||||
};
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_storage_api::{
|
||||
noop::NoopProvider, BalProvider, BlockReader, BlockReaderIdExt, HeaderProvider,
|
||||
StateProviderFactory,
|
||||
noop::NoopProvider, BlockReader, BlockReaderIdExt, HeaderProvider, StateProviderFactory,
|
||||
};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_tokio_util::EventStream;
|
||||
@@ -248,7 +247,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Clone
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
@@ -321,7 +319,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
Pool: TransactionPool<
|
||||
@@ -465,10 +462,7 @@ where
|
||||
}
|
||||
|
||||
/// Set a new request handler that's connected to the peer's network
|
||||
pub fn install_request_handler(&mut self)
|
||||
where
|
||||
C: BalProvider,
|
||||
{
|
||||
pub fn install_request_handler(&mut self) {
|
||||
let (tx, rx) = channel(ETH_REQUEST_CHANNEL_CAPACITY);
|
||||
self.network.set_eth_request_handler(tx);
|
||||
let peers = self.network.peers_handle();
|
||||
@@ -579,7 +573,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
Pool: TransactionPool<
|
||||
|
||||
@@ -2,29 +2,23 @@
|
||||
//! Tests for eth related requests
|
||||
|
||||
use alloy_consensus::Header;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use rand::Rng;
|
||||
use reth_eth_wire::{BlockAccessLists, EthVersion, GetBlockAccessLists, HeadersDirection};
|
||||
use reth_eth_wire::{EthVersion, HeadersDirection};
|
||||
use reth_ethereum_primitives::Block;
|
||||
use reth_network::{
|
||||
eth_requests::SOFT_RESPONSE_LIMIT,
|
||||
test_utils::{NetworkEventStream, PeerConfig, Testnet, TestnetHandle},
|
||||
test_utils::{NetworkEventStream, PeerConfig, Testnet},
|
||||
BlockDownloaderProvider, NetworkEventListenerProvider,
|
||||
};
|
||||
use reth_network_api::{NetworkInfo, Peers};
|
||||
use reth_network_p2p::{
|
||||
bodies::client::BodiesClient,
|
||||
error::RequestError,
|
||||
headers::client::{HeadersClient, HeadersRequest},
|
||||
BalRequirement, BlockAccessListsClient,
|
||||
};
|
||||
use reth_provider::{test_utils::MockEthProvider, BalStoreHandle, InMemoryBalStore};
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
use reth_transaction_pool::test_utils::{TestPool, TransactionGenerator};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::oneshot;
|
||||
|
||||
type BalTestnetHandle = TestnetHandle<Arc<MockEthProvider>, TestPool>;
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_get_body() {
|
||||
reth_tracing::init_test_tracing();
|
||||
@@ -532,178 +526,3 @@ async fn test_eth69_get_receipts() {
|
||||
assert_eq!(receipts_response.0[0][1].cumulative_gas_used, 42000);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth71_get_block_access_lists() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, bal_store) = spawn_eth71_bal_testnet().await;
|
||||
|
||||
let hash0 = B256::random();
|
||||
let hash1 = B256::random();
|
||||
let hash2 = B256::random();
|
||||
let bal0 = Bytes::from_static(&[0xc1, 0x01]);
|
||||
let bal2 = Bytes::from_static(&[0xc1, 0x02]);
|
||||
|
||||
bal_store.insert(hash0, 1, bal0.clone()).unwrap();
|
||||
bal_store.insert(hash2, 3, bal2.clone()).unwrap();
|
||||
|
||||
let response = request_block_access_lists(&net, vec![hash0, hash1, hash2]).await;
|
||||
assert_eq!(
|
||||
response,
|
||||
BlockAccessLists(vec![bal0, Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]), bal2,])
|
||||
);
|
||||
}
|
||||
|
||||
// Ensures BAL responses stop at the soft response limit while keeping the item that crosses it.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth71_get_block_access_lists_respects_response_soft_limit() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, bal_store) = spawn_eth71_bal_testnet().await;
|
||||
|
||||
let hash0 = B256::random();
|
||||
let hash1 = B256::random();
|
||||
let hash2 = B256::random();
|
||||
let bal0 = raw_bal_with_len(2);
|
||||
let bal1 = raw_bal_with_len(SOFT_RESPONSE_LIMIT);
|
||||
let bal2 = raw_bal_with_len(2);
|
||||
assert!(bal0.len() + bal1.len() > SOFT_RESPONSE_LIMIT);
|
||||
|
||||
bal_store.insert(hash0, 1, bal0.clone()).unwrap();
|
||||
bal_store.insert(hash1, 2, bal1.clone()).unwrap();
|
||||
bal_store.insert(hash2, 3, bal2).unwrap();
|
||||
|
||||
let response = request_block_access_lists(&net, vec![hash0, hash1, hash2]).await;
|
||||
|
||||
assert_eq!(response, BlockAccessLists(vec![bal0, bal1]));
|
||||
}
|
||||
|
||||
// Ensures a single BAL larger than the soft limit is still returned.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth71_get_block_access_lists_returns_single_oversized_bal() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, bal_store) = spawn_eth71_bal_testnet().await;
|
||||
|
||||
let hash0 = B256::random();
|
||||
let hash1 = B256::random();
|
||||
let bal0 = raw_bal_with_len(SOFT_RESPONSE_LIMIT + 1);
|
||||
let bal1 = raw_bal_with_len(2);
|
||||
|
||||
bal_store.insert(hash0, 1, bal0.clone()).unwrap();
|
||||
bal_store.insert(hash1, 2, bal1).unwrap();
|
||||
|
||||
let response = request_block_access_lists(&net, vec![hash0, hash1]).await;
|
||||
|
||||
assert_eq!(response, BlockAccessLists(vec![bal0]));
|
||||
}
|
||||
|
||||
// Ensures an empty BAL request roundtrips to an empty response.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth71_get_block_access_lists_empty_request() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, _) = spawn_eth71_bal_testnet().await;
|
||||
|
||||
let response = request_block_access_lists(&net, Vec::new()).await;
|
||||
|
||||
assert_eq!(response, BlockAccessLists(Vec::new()));
|
||||
}
|
||||
|
||||
// Ensures the fetch client can request BALs through an eth/71 peer.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth71_fetch_client_get_block_access_lists() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, bal_store) = spawn_eth71_bal_testnet().await;
|
||||
|
||||
let hash0 = B256::random();
|
||||
let hash1 = B256::random();
|
||||
let bal0 = Bytes::from_static(&[0xc1, 0x01]);
|
||||
|
||||
bal_store.insert(hash0, 1, bal0.clone()).unwrap();
|
||||
|
||||
let fetch = net.peers()[0].network().fetch_client().await.unwrap();
|
||||
let response = fetch.get_block_access_lists(vec![hash0, hash1]).await.unwrap().into_data();
|
||||
|
||||
assert_eq!(
|
||||
response,
|
||||
BlockAccessLists(vec![bal0, Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE])])
|
||||
);
|
||||
}
|
||||
|
||||
// Ensures fetch client BAL requests are rejected when no eth/71 peer is available.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_eth70_fetch_client_rejects_optional_block_access_lists_request() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let (net, _) = spawn_bal_testnet([EthVersion::Eth70, EthVersion::Eth70]).await;
|
||||
|
||||
let fetch = net.peers()[0].network().fetch_client().await.unwrap();
|
||||
let err = fetch
|
||||
.get_block_access_lists_with_requirement(vec![B256::random()], BalRequirement::Optional)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert_eq!(err, RequestError::UnsupportedCapability);
|
||||
}
|
||||
|
||||
async fn spawn_eth71_bal_testnet() -> (BalTestnetHandle, BalStoreHandle) {
|
||||
spawn_bal_testnet([EthVersion::Eth71, EthVersion::Eth71]).await
|
||||
}
|
||||
|
||||
// Spawns a BAL testnet with one peer per requested eth protocol version.
|
||||
async fn spawn_bal_testnet(
|
||||
versions: impl IntoIterator<Item = EthVersion>,
|
||||
) -> (BalTestnetHandle, BalStoreHandle) {
|
||||
let mut mock_provider = MockEthProvider::default();
|
||||
let bal_store = BalStoreHandle::new(InMemoryBalStore::default());
|
||||
mock_provider.bal_store = bal_store.clone();
|
||||
let mock_provider = Arc::new(mock_provider);
|
||||
|
||||
let mut net: Testnet<Arc<MockEthProvider>, TestPool> = Testnet::default();
|
||||
|
||||
for version in versions {
|
||||
let peer = PeerConfig::with_protocols(mock_provider.clone(), Some(version.into()));
|
||||
net.add_peer_with_config(peer).await.unwrap();
|
||||
}
|
||||
|
||||
net.for_each_mut(|peer| peer.install_request_handler());
|
||||
|
||||
let net = net.spawn();
|
||||
net.connect_peers().await;
|
||||
|
||||
(net, bal_store)
|
||||
}
|
||||
|
||||
// Sends a GetBlockAccessLists request from peer 0 to peer 1.
|
||||
async fn request_block_access_lists(net: &BalTestnetHandle, hashes: Vec<B256>) -> BlockAccessLists {
|
||||
let requester = &net.peers()[0];
|
||||
let responder = &net.peers()[1];
|
||||
let (tx, rx) = oneshot::channel();
|
||||
|
||||
requester.network().send_request(
|
||||
*responder.peer_id(),
|
||||
reth_network::PeerRequest::GetBlockAccessLists {
|
||||
request: GetBlockAccessLists(hashes),
|
||||
response: tx,
|
||||
},
|
||||
);
|
||||
|
||||
rx.await.unwrap().unwrap()
|
||||
}
|
||||
|
||||
// Builds a complete raw RLP list item with the requested encoded byte length.
|
||||
fn raw_bal_with_len(len: usize) -> Bytes {
|
||||
assert!(len > 0);
|
||||
|
||||
let mut payload_length = len - 1;
|
||||
loop {
|
||||
let header_length = alloy_rlp::Header { list: true, payload_length }.length();
|
||||
let next_payload_length = len.checked_sub(header_length).unwrap();
|
||||
if next_payload_length == payload_length {
|
||||
break
|
||||
}
|
||||
payload_length = next_payload_length;
|
||||
}
|
||||
|
||||
let mut out = Vec::with_capacity(len);
|
||||
alloy_rlp::Header { list: true, payload_length }.encode(&mut out);
|
||||
out.resize(len, alloy_rlp::EMPTY_LIST_CODE);
|
||||
Bytes::from(out)
|
||||
}
|
||||
|
||||
@@ -4,17 +4,6 @@ use auto_impl::auto_impl;
|
||||
use futures::Future;
|
||||
use reth_eth_wire_types::BlockAccessLists;
|
||||
|
||||
/// Controls whether a BAL request must wait for a capable peer or may complete early when none are
|
||||
/// available.
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub enum BalRequirement {
|
||||
/// Keep waiting until an eth/71-capable peer is available.
|
||||
#[default]
|
||||
Mandatory,
|
||||
/// Return early if no connected peer can serve BALs.
|
||||
Optional,
|
||||
}
|
||||
|
||||
/// A client capable of downloading block access lists.
|
||||
#[auto_impl(&, Arc, Box)]
|
||||
pub trait BlockAccessListsClient: DownloadClient {
|
||||
@@ -23,24 +12,7 @@ pub trait BlockAccessListsClient: DownloadClient {
|
||||
|
||||
/// Fetches the block access lists for given hashes.
|
||||
fn get_block_access_lists(&self, hashes: Vec<B256>) -> Self::Output {
|
||||
self.get_block_access_lists_with_priority_and_requirement(
|
||||
hashes,
|
||||
Priority::Normal,
|
||||
BalRequirement::Mandatory,
|
||||
)
|
||||
}
|
||||
|
||||
/// Fetches the block access lists for given hashes with the requested BAL availability policy.
|
||||
fn get_block_access_lists_with_requirement(
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
requirement: BalRequirement,
|
||||
) -> Self::Output {
|
||||
self.get_block_access_lists_with_priority_and_requirement(
|
||||
hashes,
|
||||
Priority::Normal,
|
||||
requirement,
|
||||
)
|
||||
self.get_block_access_lists_with_priority(hashes, Priority::Normal)
|
||||
}
|
||||
|
||||
/// Fetches the block access lists for given hashes with priority
|
||||
@@ -48,19 +20,5 @@ pub trait BlockAccessListsClient: DownloadClient {
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
) -> Self::Output {
|
||||
self.get_block_access_lists_with_priority_and_requirement(
|
||||
hashes,
|
||||
priority,
|
||||
BalRequirement::Mandatory,
|
||||
)
|
||||
}
|
||||
|
||||
/// Fetches the block access lists for given hashes with priority and BAL availability policy.
|
||||
fn get_block_access_lists_with_priority_and_requirement(
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
requirement: BalRequirement,
|
||||
) -> Self::Output;
|
||||
}
|
||||
|
||||
@@ -57,8 +57,8 @@ impl<H: BlockHeader> EthResponseValidator for RequestResult<Vec<H>> {
|
||||
/// [`RequestError::ConnectionDropped`] should be ignored here because this is already handled
|
||||
/// when the dropped connection is handled.
|
||||
///
|
||||
/// [`RequestError::UnsupportedCapability`] is also used for locally rejected optional requests,
|
||||
/// which should not affect peer reputation.
|
||||
/// [`RequestError::UnsupportedCapability`] is not used yet because we only support active
|
||||
/// session for eth protocol.
|
||||
fn reputation_change_err(&self) -> Option<ReputationChangeKind> {
|
||||
if let Err(err) = self {
|
||||
match err {
|
||||
|
||||
@@ -1110,11 +1110,10 @@ mod tests {
|
||||
impl BlockAccessListsClient for FullBlockWithAccessListsClient {
|
||||
type Output = futures::future::Ready<PeerRequestResult<BlockAccessLists>>;
|
||||
|
||||
fn get_block_access_lists_with_priority_and_requirement(
|
||||
fn get_block_access_lists_with_priority(
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
_priority: Priority,
|
||||
_requirement: crate::block_access_lists::client::BalRequirement,
|
||||
) -> Self::Output {
|
||||
self.access_list_requests.fetch_add(1, Ordering::SeqCst);
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ pub mod snap;
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
pub mod test_utils;
|
||||
|
||||
pub use block_access_lists::client::{BalRequirement, BlockAccessListsClient};
|
||||
pub use block_access_lists::client::BlockAccessListsClient;
|
||||
pub use bodies::client::BodiesClient;
|
||||
pub use headers::client::HeadersClient;
|
||||
pub use receipts::client::ReceiptsClient;
|
||||
|
||||
@@ -66,8 +66,8 @@ use reth_node_metrics::{
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
|
||||
BlockHashReader, BlockNumReader, DatabaseProviderFactory, ProviderError, ProviderFactory,
|
||||
ProviderResult, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder,
|
||||
BlockHashReader, BlockNumReader, ProviderError, ProviderFactory, ProviderResult,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune::{PruneModes, PrunerBuilder};
|
||||
@@ -75,7 +75,7 @@ use reth_rpc_builder::config::RethRpcServerConfig;
|
||||
use reth_rpc_layer::JwtSecret;
|
||||
use reth_stages::{
|
||||
sets::DefaultStages, stages::EraImportSource, MetricEvent, PipelineBuilder, PipelineTarget,
|
||||
StageCheckpoint, StageId,
|
||||
StageId,
|
||||
};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use reth_tasks::TaskExecutor;
|
||||
@@ -167,9 +167,8 @@ impl LaunchContext {
|
||||
|
||||
info!(target: "reth::cli", path = ?config_path, "Configuration loaded");
|
||||
|
||||
// Update the config with the command line arguments. Only override when the CLI flag is
|
||||
// set, so the TOML value is preserved when the flag is not passed.
|
||||
toml_config.peers.trusted_nodes_only |= config.network.trusted_only;
|
||||
// Update the config with the command line arguments
|
||||
toml_config.peers.trusted_nodes_only = config.network.trusted_only;
|
||||
|
||||
// Merge static file CLI arguments with config file, giving priority to CLI
|
||||
toml_config.static_files =
|
||||
@@ -518,26 +517,19 @@ where
|
||||
// the unwind targets for each storage layer if inconsistencies are
|
||||
// found.
|
||||
let (rocksdb_unwind, static_file_unwind) = factory.check_consistency()?;
|
||||
let partial_trie_unwind = partial_trie_unwind_target(
|
||||
factory.database_provider_ro()?.get_stage_checkpoint(StageId::Finish)?,
|
||||
);
|
||||
|
||||
// Take the minimum block number to ensure all storage layers are consistent.
|
||||
let unwind_target =
|
||||
[rocksdb_unwind, static_file_unwind, partial_trie_unwind].into_iter().flatten().min();
|
||||
let unwind_target = [rocksdb_unwind, static_file_unwind].into_iter().flatten().min();
|
||||
|
||||
if let Some(unwind_block) = unwind_target {
|
||||
let inconsistency_source = [
|
||||
rocksdb_unwind.map(|_| "RocksDB"),
|
||||
static_file_unwind.map(|_| "static file"),
|
||||
partial_trie_unwind.map(|_| "partial state trie"),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" and ");
|
||||
// Highly unlikely to happen, and given its destructive nature, it's better to panic
|
||||
// instead. Unwinding to 0 would leave MDBX with a huge free list size.
|
||||
let inconsistency_source = match (rocksdb_unwind, static_file_unwind) {
|
||||
(Some(_), Some(_)) => "RocksDB and static file",
|
||||
(Some(_), None) => "RocksDB",
|
||||
(None, Some(_)) => "static file",
|
||||
(None, None) => unreachable!(),
|
||||
};
|
||||
assert_ne!(
|
||||
unwind_block, 0,
|
||||
"A {} inconsistency was found that would trigger an unwind to block 0",
|
||||
@@ -1276,19 +1268,11 @@ pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>)
|
||||
.build()
|
||||
}
|
||||
|
||||
fn partial_trie_unwind_target(finish_checkpoint: Option<StageCheckpoint>) -> Option<BlockNumber> {
|
||||
let finish_checkpoint = finish_checkpoint?;
|
||||
let partial_state_trie = finish_checkpoint.finish_stage_checkpoint()?.partial_state_trie?;
|
||||
|
||||
(partial_state_trie != finish_checkpoint.block_number).then_some(partial_state_trie)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{partial_trie_unwind_target, LaunchContext, NodeConfig};
|
||||
use super::{LaunchContext, NodeConfig};
|
||||
use reth_config::Config;
|
||||
use reth_node_core::args::PruningArgs;
|
||||
use reth_stages::{FinishCheckpoint, StageCheckpoint};
|
||||
|
||||
const EXTENSION: &str = "toml";
|
||||
|
||||
@@ -1340,24 +1324,4 @@ mod tests {
|
||||
assert_eq!(reth_config, loaded_config);
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_trie_unwind_target_uses_partial_finish_checkpoint() {
|
||||
let finish_checkpoint = StageCheckpoint::new(42)
|
||||
.with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(21) });
|
||||
|
||||
assert_eq!(partial_trie_unwind_target(Some(finish_checkpoint)), Some(21));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_trie_unwind_target_ignores_matching_or_missing_partial_checkpoint() {
|
||||
let matching_finish_checkpoint = StageCheckpoint::new(42)
|
||||
.with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(42) });
|
||||
let missing_partial_finish_checkpoint = StageCheckpoint::new(42)
|
||||
.with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: None });
|
||||
|
||||
assert_eq!(partial_trie_unwind_target(Some(matching_finish_checkpoint)), None);
|
||||
assert_eq!(partial_trie_unwind_target(Some(missing_partial_finish_checkpoint)), None);
|
||||
assert_eq!(partial_trie_unwind_target(None), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -205,12 +205,10 @@ impl EngineNodeLauncher {
|
||||
ctx.blockchain_db().clone(),
|
||||
ctx.components().evm_config().clone(),
|
||||
|| async {
|
||||
// Create a separate cache for reorg validator (not shared with main engine)
|
||||
let reorg_cache = ChangesetCache::new();
|
||||
validator_builder
|
||||
.build_tree_validator(
|
||||
&add_ons_ctx,
|
||||
engine_tree_config.clone(),
|
||||
changeset_cache.clone(),
|
||||
)
|
||||
.build_tree_validator(&add_ons_ctx, engine_tree_config.clone(), reorg_cache)
|
||||
.await
|
||||
},
|
||||
node_config.debug.reorg_frequency,
|
||||
|
||||
@@ -46,15 +46,6 @@ pub struct BenchmarkArgs {
|
||||
)]
|
||||
pub engine_rpc_url: String,
|
||||
|
||||
/// The RPC url to use for non-authenticated node RPC requests.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "LOCAL_RPC_URL",
|
||||
verbatim_doc_comment,
|
||||
default_value = "http://localhost:8545"
|
||||
)]
|
||||
pub local_rpc_url: String,
|
||||
|
||||
/// The `WebSocket` RPC URL to use for persistence subscriptions.
|
||||
///
|
||||
/// If not provided, will attempt to derive from engine-rpc-url by:
|
||||
@@ -250,7 +241,6 @@ mod tests {
|
||||
fn test_parse_benchmark_args() {
|
||||
let default_args = BenchmarkArgs {
|
||||
engine_rpc_url: "http://localhost:8551".to_string(),
|
||||
local_rpc_url: "http://localhost:8545".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let args = CommandParser::<BenchmarkArgs>::parse_from(["reth-bench"]).args;
|
||||
|
||||
@@ -60,13 +60,6 @@ pub struct DatabaseArgs {
|
||||
value_parser = value_parser!(SyncMode),
|
||||
)]
|
||||
pub sync_mode: Option<SyncMode>,
|
||||
/// `RocksDB` block cache size (e.g., 512MB, 4GB).
|
||||
///
|
||||
/// Controls the size of the in-memory LRU cache for decompressed `RocksDB` blocks.
|
||||
/// A larger cache reduces repeated decompression of hot blocks, improving read
|
||||
/// performance for history lookups.
|
||||
#[arg(long = "db.rocksdb-block-cache-size", value_parser = parse_byte_size)]
|
||||
pub rocksdb_block_cache_size: Option<usize>,
|
||||
}
|
||||
|
||||
impl DatabaseArgs {
|
||||
|
||||
@@ -4,9 +4,9 @@ use clap::{builder::Resettable, Args};
|
||||
use eyre::ensure;
|
||||
use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms};
|
||||
use reth_engine_primitives::{
|
||||
default_persistence_backpressure_threshold, TreeConfig, DEFAULT_DEFERRED_TRIE_BLOCKS,
|
||||
DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE,
|
||||
DEFAULT_SPARSE_TRIE_MAX_HOT_ACCOUNTS, DEFAULT_SPARSE_TRIE_MAX_HOT_SLOTS,
|
||||
TreeConfig, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE,
|
||||
DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, DEFAULT_SPARSE_TRIE_MAX_HOT_ACCOUNTS,
|
||||
DEFAULT_SPARSE_TRIE_MAX_HOT_SLOTS,
|
||||
};
|
||||
use std::{sync::OnceLock, time::Duration};
|
||||
|
||||
@@ -24,8 +24,7 @@ static ENGINE_DEFAULTS: OnceLock<DefaultEngineValues> = OnceLock::new();
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct DefaultEngineValues {
|
||||
persistence_threshold: u64,
|
||||
persistence_backpressure_threshold: Option<u64>,
|
||||
deferred_trie_blocks: u64,
|
||||
persistence_backpressure_threshold: u64,
|
||||
memory_block_buffer_target: u64,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
legacy_state_root_task_enabled: bool,
|
||||
@@ -74,26 +73,9 @@ impl DefaultEngineValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// Get the default persistence backpressure threshold.
|
||||
pub const fn persistence_backpressure_threshold(&self) -> u64 {
|
||||
match self.persistence_backpressure_threshold {
|
||||
Some(v) => v,
|
||||
None => default_persistence_backpressure_threshold(
|
||||
self.persistence_threshold,
|
||||
self.memory_block_buffer_target,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the default persistence backpressure threshold
|
||||
pub const fn with_persistence_backpressure_threshold(mut self, v: u64) -> Self {
|
||||
self.persistence_backpressure_threshold = Some(v);
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default deferred trie block target
|
||||
pub const fn with_deferred_trie_blocks(mut self, v: u64) -> Self {
|
||||
self.deferred_trie_blocks = v;
|
||||
self.persistence_backpressure_threshold = v;
|
||||
self
|
||||
}
|
||||
|
||||
@@ -279,8 +261,7 @@ impl Default for DefaultEngineValues {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
persistence_backpressure_threshold: None,
|
||||
deferred_trie_blocks: DEFAULT_DEFERRED_TRIE_BLOCKS,
|
||||
persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
|
||||
memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET,
|
||||
invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
|
||||
legacy_state_root_task_enabled: false,
|
||||
@@ -330,14 +311,9 @@ pub struct EngineArgs {
|
||||
/// Configure the maximum canonical-minus-persisted gap before engine API processing stalls.
|
||||
///
|
||||
/// This value must be greater than `--engine.persistence-threshold`.
|
||||
#[arg(long = "engine.persistence-backpressure-threshold", default_value_t = DefaultEngineValues::get_global().persistence_backpressure_threshold())]
|
||||
#[arg(long = "engine.persistence-backpressure-threshold", default_value_t = DefaultEngineValues::get_global().persistence_backpressure_threshold)]
|
||||
pub persistence_backpressure_threshold: u64,
|
||||
|
||||
/// Configure how many of the blocks being persisted should only mask state/trie writes instead
|
||||
/// of durably persisting their state/trie updates in the current cycle.
|
||||
#[arg(long = "engine.deferred-trie-blocks", default_value_t = DefaultEngineValues::get_global().deferred_trie_blocks)]
|
||||
pub deferred_trie_blocks: u64,
|
||||
|
||||
/// Configure the target number of blocks to keep in memory.
|
||||
#[arg(long = "engine.memory-block-buffer-target", default_value_t = DefaultEngineValues::get_global().memory_block_buffer_target)]
|
||||
pub memory_block_buffer_target: u64,
|
||||
@@ -570,7 +546,6 @@ impl Default for EngineArgs {
|
||||
let DefaultEngineValues {
|
||||
persistence_threshold,
|
||||
persistence_backpressure_threshold,
|
||||
deferred_trie_blocks,
|
||||
memory_block_buffer_target,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
legacy_state_root_task_enabled,
|
||||
@@ -603,15 +578,7 @@ impl Default for EngineArgs {
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
persistence_threshold,
|
||||
persistence_backpressure_threshold: persistence_backpressure_threshold.unwrap_or_else(
|
||||
|| {
|
||||
default_persistence_backpressure_threshold(
|
||||
persistence_threshold,
|
||||
memory_block_buffer_target,
|
||||
)
|
||||
},
|
||||
),
|
||||
deferred_trie_blocks,
|
||||
persistence_backpressure_threshold,
|
||||
memory_block_buffer_target,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
legacy_state_root_task_enabled,
|
||||
@@ -663,13 +630,6 @@ impl EngineArgs {
|
||||
self.persistence_backpressure_threshold,
|
||||
self.persistence_threshold
|
||||
);
|
||||
ensure!(
|
||||
self.deferred_trie_blocks + self.memory_block_buffer_target < self.persistence_threshold,
|
||||
"--engine.deferred-trie-blocks ({}) + --engine.memory-block-buffer-target ({}) must be less than --engine.persistence-threshold ({})",
|
||||
self.deferred_trie_blocks,
|
||||
self.memory_block_buffer_target,
|
||||
self.persistence_threshold,
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -678,7 +638,6 @@ impl EngineArgs {
|
||||
let config = TreeConfig::default()
|
||||
.with_persistence_threshold(self.persistence_threshold)
|
||||
.with_persistence_backpressure_threshold(self.persistence_backpressure_threshold)
|
||||
.with_num_state_masking_blocks(self.deferred_trie_blocks)
|
||||
.with_memory_block_buffer_target(self.memory_block_buffer_target)
|
||||
.with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold)
|
||||
.with_legacy_state_root(self.legacy_state_root_task_enabled)
|
||||
@@ -736,48 +695,12 @@ mod tests {
|
||||
assert_eq!(args, default_args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_engine_values_derive_backpressure_threshold() {
|
||||
let defaults = DefaultEngineValues::default()
|
||||
.with_persistence_threshold(10)
|
||||
.with_memory_block_buffer_target(3);
|
||||
|
||||
assert_eq!(defaults.persistence_backpressure_threshold(), 26);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_backpressure_default_override_is_preserved() {
|
||||
let defaults = DefaultEngineValues::default()
|
||||
.with_persistence_backpressure_threshold(99)
|
||||
.with_persistence_threshold(10)
|
||||
.with_memory_block_buffer_target(3);
|
||||
|
||||
assert_eq!(defaults.persistence_backpressure_threshold(), 99);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn engine_args_default_thresholds_match_expected_defaults() {
|
||||
let args = EngineArgs::default();
|
||||
|
||||
assert_eq!(args.persistence_threshold, DEFAULT_PERSISTENCE_THRESHOLD);
|
||||
assert_eq!(args.deferred_trie_blocks, DEFAULT_DEFERRED_TRIE_BLOCKS);
|
||||
assert_eq!(args.memory_block_buffer_target, DEFAULT_MEMORY_BLOCK_BUFFER_TARGET);
|
||||
assert_eq!(
|
||||
args.persistence_backpressure_threshold,
|
||||
default_persistence_backpressure_threshold(
|
||||
args.persistence_threshold,
|
||||
args.memory_block_buffer_target,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn engine_args() {
|
||||
let args = EngineArgs {
|
||||
persistence_threshold: 100,
|
||||
persistence_backpressure_threshold: 101,
|
||||
deferred_trie_blocks: 25,
|
||||
memory_block_buffer_target: 50,
|
||||
invalid_header_hit_eviction_threshold: 7,
|
||||
legacy_state_root_task_enabled: true,
|
||||
@@ -822,8 +745,6 @@ mod tests {
|
||||
"100",
|
||||
"--engine.persistence-backpressure-threshold",
|
||||
"101",
|
||||
"--engine.deferred-trie-blocks",
|
||||
"25",
|
||||
"--engine.memory-block-buffer-target",
|
||||
"50",
|
||||
"--engine.invalid-header-cache-hit-eviction-threshold",
|
||||
@@ -867,21 +788,6 @@ mod tests {
|
||||
assert_eq!(parsed_args, args);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_deferred_trie_blocks() {
|
||||
let args = CommandParser::<EngineArgs>::parse_from([
|
||||
"reth",
|
||||
"--engine.persistence-threshold",
|
||||
"8",
|
||||
"--engine.deferred-trie-blocks",
|
||||
"7",
|
||||
])
|
||||
.args;
|
||||
|
||||
assert_eq!(args.deferred_trie_blocks, 7);
|
||||
assert_eq!(args.tree_config().num_state_masking_blocks(), 7);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_invalid_backpressure_threshold() {
|
||||
let args = EngineArgs {
|
||||
@@ -895,21 +801,6 @@ mod tests {
|
||||
assert!(err.contains("engine.persistence-threshold"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_rejects_state_masking_window_at_or_above_threshold() {
|
||||
let args = EngineArgs {
|
||||
persistence_threshold: 4,
|
||||
deferred_trie_blocks: 2,
|
||||
memory_block_buffer_target: 2,
|
||||
..EngineArgs::default()
|
||||
};
|
||||
|
||||
let err = args.validate().unwrap_err().to_string();
|
||||
assert!(err.contains("engine.deferred-trie-blocks"));
|
||||
assert!(err.contains("engine.memory-block-buffer-target"));
|
||||
assert!(err.contains("engine.persistence-threshold"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_slow_block_threshold() {
|
||||
// Test default value (None - disabled)
|
||||
|
||||
@@ -338,7 +338,7 @@ pub trait LoadPendingBlock:
|
||||
}
|
||||
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
Ok(gas_used) => gas_used.tx_gas_used(),
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
error,
|
||||
..
|
||||
|
||||
@@ -115,11 +115,6 @@ pub trait EthState: LoadState + SpawnBlocking {
|
||||
block_id: Option<BlockId>,
|
||||
) -> impl Future<Output = Result<HashMap<Address, Vec<B256>>, Self::Error>> + Send {
|
||||
async move {
|
||||
if requests.is_empty() {
|
||||
return Err(Self::Error::from_eth_err(EthApiError::InvalidParams(
|
||||
"empty request".to_string(),
|
||||
)));
|
||||
}
|
||||
let total_slots: usize = requests.values().map(|slots| slots.len()).sum();
|
||||
if total_slots > DEFAULT_MAX_STORAGE_VALUES_SLOTS {
|
||||
return Err(Self::Error::from_eth_err(EthApiError::InvalidParams(
|
||||
|
||||
@@ -283,7 +283,6 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> + Call {
|
||||
let block_hash = block.hash();
|
||||
|
||||
let block_number = evm_env.block_env.number().saturating_to();
|
||||
let block_timestamp = evm_env.block_env.timestamp().saturating_to();
|
||||
let base_fee = evm_env.block_env.basefee();
|
||||
|
||||
this.apply_pre_execution_changes(&block, &mut db)?;
|
||||
@@ -310,8 +309,8 @@ pub trait Trace: LoadState<Error: FromEvmError<Self::Evm>> + Call {
|
||||
index: Some(idx),
|
||||
block_hash: Some(block_hash),
|
||||
block_number: Some(block_number),
|
||||
block_timestamp: Some(block_timestamp),
|
||||
base_fee: Some(base_fee),
|
||||
..Default::default()
|
||||
};
|
||||
idx += 1;
|
||||
|
||||
|
||||
@@ -325,16 +325,15 @@ pub trait EthTransactions: LoadTransaction<Provider: BlockReaderIdExt> {
|
||||
if let Some(block) = self.recovered_block(block_id).await? {
|
||||
let block_hash = block.hash();
|
||||
let block_number = block.number();
|
||||
let block_timestamp = block.timestamp();
|
||||
let base_fee_per_gas = block.base_fee_per_gas();
|
||||
if let Some((signer, tx)) = block.transactions_with_sender().nth(index) {
|
||||
let tx_info = TransactionInfo {
|
||||
hash: Some(*tx.tx_hash()),
|
||||
block_hash: Some(block_hash),
|
||||
block_number: Some(block_number),
|
||||
block_timestamp: Some(block_timestamp),
|
||||
base_fee: base_fee_per_gas,
|
||||
index: Some(index as u64),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
return Ok(Some(
|
||||
@@ -396,7 +395,6 @@ pub trait EthTransactions: LoadTransaction<Provider: BlockReaderIdExt> {
|
||||
.and_then(|block| {
|
||||
let block_hash = block.hash();
|
||||
let block_number = block.number();
|
||||
let block_timestamp = block.timestamp();
|
||||
let base_fee_per_gas = block.base_fee_per_gas();
|
||||
|
||||
block
|
||||
@@ -408,9 +406,9 @@ pub trait EthTransactions: LoadTransaction<Provider: BlockReaderIdExt> {
|
||||
hash: Some(*tx.tx_hash()),
|
||||
block_hash: Some(block_hash),
|
||||
block_number: Some(block_number),
|
||||
block_timestamp: Some(block_timestamp),
|
||||
base_fee: base_fee_per_gas,
|
||||
index: Some(index as u64),
|
||||
..Default::default()
|
||||
};
|
||||
Ok(self.converter().fill(tx.clone().with_signer(*signer), tx_info)?)
|
||||
})
|
||||
@@ -683,7 +681,6 @@ pub trait LoadTransaction: SpawnBlocking + FullEthApiTypes + RpcNodeCoreExt {
|
||||
index: meta.index,
|
||||
block_hash: meta.block_hash,
|
||||
block_number: meta.block_number,
|
||||
block_timestamp: meta.timestamp,
|
||||
base_fee: meta.base_fee,
|
||||
}));
|
||||
}
|
||||
|
||||
@@ -53,7 +53,6 @@ impl<B: Block, R> CachedTransaction<B, R> {
|
||||
index: self.tx_index as u64,
|
||||
block_hash: self.block.hash(),
|
||||
block_number: self.block.number(),
|
||||
block_timestamp: self.block.timestamp(),
|
||||
base_fee: self.block.base_fee_per_gas(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -169,10 +169,7 @@ pub fn get_filter_block_range(
|
||||
|
||||
// we cannot query blocks that don't exist yet
|
||||
if to_block_number > info.best_number {
|
||||
return Err(FilterBlockRangeError::BlockRangeExceedsHead {
|
||||
requested: to_block_number,
|
||||
head: info.best_number,
|
||||
});
|
||||
return Err(FilterBlockRangeError::BlockRangeExceedsHead);
|
||||
}
|
||||
|
||||
Ok((from_block_number, to_block_number))
|
||||
@@ -187,13 +184,8 @@ pub enum FilterBlockRangeError {
|
||||
#[error("invalid block range params")]
|
||||
InvalidBlockRange,
|
||||
/// Block range extends beyond current head
|
||||
#[error("block range extends beyond current head block: requested {requested}, head {head}")]
|
||||
BlockRangeExceedsHead {
|
||||
/// The requested `toBlock` number
|
||||
requested: u64,
|
||||
/// The current head block number
|
||||
head: u64,
|
||||
},
|
||||
#[error("block range extends beyond current head block")]
|
||||
BlockRangeExceedsHead,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -235,10 +227,7 @@ mod tests {
|
||||
let to = 15000002u64;
|
||||
let info = ChainInfo { best_number: 15000000, ..Default::default() };
|
||||
let err = get_filter_block_range(Some(from), Some(to), info.best_number, info).unwrap_err();
|
||||
assert_eq!(
|
||||
err,
|
||||
FilterBlockRangeError::BlockRangeExceedsHead { requested: to, head: info.best_number }
|
||||
);
|
||||
assert_eq!(err, FilterBlockRangeError::BlockRangeExceedsHead);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -274,10 +263,7 @@ mod tests {
|
||||
let to = 200;
|
||||
let info = ChainInfo { best_number: 150, ..Default::default() };
|
||||
let err = get_filter_block_range(Some(from), Some(to), 0, info).unwrap_err();
|
||||
assert_eq!(
|
||||
err,
|
||||
FilterBlockRangeError::BlockRangeExceedsHead { requested: to, head: info.best_number }
|
||||
);
|
||||
assert_eq!(err, FilterBlockRangeError::BlockRangeExceedsHead);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -25,8 +25,6 @@ pub enum TransactionSource<T = TransactionSigned> {
|
||||
block_hash: B256,
|
||||
/// Number of the block.
|
||||
block_number: u64,
|
||||
/// Timestamp of the block.
|
||||
block_timestamp: u64,
|
||||
/// base fee of the block.
|
||||
base_fee: Option<u64>,
|
||||
},
|
||||
@@ -50,21 +48,14 @@ impl<T: SignedTransaction> TransactionSource<T> {
|
||||
{
|
||||
match self {
|
||||
Self::Pool(tx) => resp_builder.fill_pending(tx),
|
||||
Self::Block {
|
||||
transaction,
|
||||
index,
|
||||
block_hash,
|
||||
block_number,
|
||||
block_timestamp,
|
||||
base_fee,
|
||||
} => {
|
||||
Self::Block { transaction, index, block_hash, block_number, base_fee } => {
|
||||
let tx_info = TransactionInfo {
|
||||
hash: Some(transaction.trie_hash()),
|
||||
index: Some(index),
|
||||
block_hash: Some(block_hash),
|
||||
block_number: Some(block_number),
|
||||
block_timestamp: Some(block_timestamp),
|
||||
base_fee,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
resp_builder.fill(transaction, tx_info)
|
||||
@@ -79,14 +70,7 @@ impl<T: SignedTransaction> TransactionSource<T> {
|
||||
let hash = tx.trie_hash();
|
||||
(tx, TransactionInfo { hash: Some(hash), ..Default::default() })
|
||||
}
|
||||
Self::Block {
|
||||
transaction,
|
||||
index,
|
||||
block_hash,
|
||||
block_number,
|
||||
block_timestamp,
|
||||
base_fee,
|
||||
} => {
|
||||
Self::Block { transaction, index, block_hash, block_number, base_fee } => {
|
||||
let hash = transaction.trie_hash();
|
||||
(
|
||||
transaction,
|
||||
@@ -95,8 +79,8 @@ impl<T: SignedTransaction> TransactionSource<T> {
|
||||
index: Some(index),
|
||||
block_hash: Some(block_hash),
|
||||
block_number: Some(block_number),
|
||||
block_timestamp: Some(block_timestamp),
|
||||
base_fee,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
@@ -564,10 +564,7 @@ where
|
||||
if let Some(t) = to &&
|
||||
t > info.best_number
|
||||
{
|
||||
return Err(EthFilterError::BlockRangeExceedsHead {
|
||||
requested: t,
|
||||
head: info.best_number,
|
||||
});
|
||||
return Err(EthFilterError::BlockRangeExceedsHead);
|
||||
}
|
||||
|
||||
if let Some(f) = from &&
|
||||
@@ -945,13 +942,8 @@ pub enum EthFilterError {
|
||||
#[error("invalid block range params")]
|
||||
InvalidBlockRangeParams,
|
||||
/// Block range extends beyond current head.
|
||||
#[error("block range extends beyond current head block: requested {requested}, head {head}")]
|
||||
BlockRangeExceedsHead {
|
||||
/// The requested `toBlock` number
|
||||
requested: u64,
|
||||
/// The current head block number
|
||||
head: u64,
|
||||
},
|
||||
#[error("block range extends beyond current head block")]
|
||||
BlockRangeExceedsHead,
|
||||
/// Query scope is too broad.
|
||||
#[error("query exceeds max block range {0}")]
|
||||
QueryExceedsMaxBlocks(u64),
|
||||
@@ -987,7 +979,7 @@ impl From<EthFilterError> for jsonrpsee::types::error::ErrorObject<'static> {
|
||||
err @ (EthFilterError::InvalidBlockRangeParams |
|
||||
EthFilterError::QueryExceedsMaxBlocks(_) |
|
||||
EthFilterError::QueryExceedsMaxResults { .. } |
|
||||
EthFilterError::BlockRangeExceedsHead { .. }) => {
|
||||
EthFilterError::BlockRangeExceedsHead) => {
|
||||
rpc_error_with_code(jsonrpsee::types::error::INVALID_PARAMS_CODE, err.to_string())
|
||||
}
|
||||
}
|
||||
@@ -1004,9 +996,7 @@ impl From<logs_utils::FilterBlockRangeError> for EthFilterError {
|
||||
fn from(err: logs_utils::FilterBlockRangeError) -> Self {
|
||||
match err {
|
||||
logs_utils::FilterBlockRangeError::InvalidBlockRange => Self::InvalidBlockRangeParams,
|
||||
logs_utils::FilterBlockRangeError::BlockRangeExceedsHead { requested, head } => {
|
||||
Self::BlockRangeExceedsHead { requested, head }
|
||||
}
|
||||
logs_utils::FilterBlockRangeError::BlockRangeExceedsHead => Self::BlockRangeExceedsHead,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -179,7 +179,7 @@ where
|
||||
|
||||
let tip = tx.effective_tip_per_gas(base_fee).unwrap_or_default();
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
Ok(gas_used) => gas_used.tx_gas_used(),
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(err) => {
|
||||
if skip_invalid_transactions {
|
||||
debug!(
|
||||
|
||||
@@ -295,8 +295,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed, // 1 seeded block body + batch size
|
||||
total // seeded headers
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: false }) if block_number < 200 &&
|
||||
processed == batch_size + 1 && total == previous_stage + 1
|
||||
);
|
||||
@@ -334,8 +333,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
},
|
||||
done: true
|
||||
}) if processed + 1 == total && total == previous_stage + 1
|
||||
@@ -372,8 +370,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: false }) if block_number >= 10 &&
|
||||
processed - 1 == batch_size && total == previous_stage + 1
|
||||
);
|
||||
@@ -394,8 +391,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: true }) if block_number > first_run_checkpoint.block_number &&
|
||||
processed + 1 == total && total == previous_stage + 1
|
||||
);
|
||||
@@ -436,8 +432,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage &&
|
||||
processed + 1 == total && total == previous_stage + 1
|
||||
);
|
||||
@@ -465,8 +460,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed: 1,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}}) if total == previous_stage + 1
|
||||
);
|
||||
|
||||
|
||||
@@ -298,7 +298,7 @@ mod tests {
|
||||
assert_matches!(
|
||||
output,
|
||||
Ok(ExecOutput {
|
||||
checkpoint: StageCheckpoint { block_number, stage_checkpoint: None, .. },
|
||||
checkpoint: StageCheckpoint { block_number, stage_checkpoint: None },
|
||||
done: false
|
||||
}) if block_number == era_cap
|
||||
);
|
||||
@@ -318,7 +318,7 @@ mod tests {
|
||||
assert_matches!(
|
||||
output,
|
||||
Ok(ExecOutput {
|
||||
checkpoint: StageCheckpoint { block_number, stage_checkpoint: None, .. },
|
||||
checkpoint: StageCheckpoint { block_number, stage_checkpoint: None },
|
||||
done: true
|
||||
}) if block_number == target
|
||||
);
|
||||
|
||||
@@ -1015,52 +1015,46 @@ mod tests {
|
||||
processed,
|
||||
total
|
||||
}
|
||||
})),
|
||||
..
|
||||
}))
|
||||
},
|
||||
done: true
|
||||
} if processed == total && total == block.gas_used);
|
||||
|
||||
{
|
||||
let provider = factory.provider().unwrap();
|
||||
let provider = factory.provider().unwrap();
|
||||
|
||||
// check post state
|
||||
let account1 = address!("0x1000000000000000000000000000000000000000");
|
||||
let account1_info =
|
||||
Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: Some(code_hash) };
|
||||
let account2 = address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba");
|
||||
let account2_info = Account {
|
||||
balance: U256::from(0x1bc16d674ece94bau128),
|
||||
nonce: 0x00,
|
||||
bytecode_hash: None,
|
||||
};
|
||||
let account3 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b");
|
||||
let account3_info = Account {
|
||||
balance: U256::from(0x3635c9adc5de996b46u128),
|
||||
nonce: 0x01,
|
||||
bytecode_hash: None,
|
||||
};
|
||||
// check post state
|
||||
let account1 = address!("0x1000000000000000000000000000000000000000");
|
||||
let account1_info =
|
||||
Account { balance: U256::ZERO, nonce: 0x00, bytecode_hash: Some(code_hash) };
|
||||
let account2 = address!("0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba");
|
||||
let account2_info = Account {
|
||||
balance: U256::from(0x1bc16d674ece94bau128),
|
||||
nonce: 0x00,
|
||||
bytecode_hash: None,
|
||||
};
|
||||
let account3 = address!("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b");
|
||||
let account3_info = Account {
|
||||
balance: U256::from(0x3635c9adc5de996b46u128),
|
||||
nonce: 0x01,
|
||||
bytecode_hash: None,
|
||||
};
|
||||
|
||||
// assert accounts
|
||||
assert!(matches!(
|
||||
provider.basic_account(&account1),
|
||||
Ok(Some(acc)) if acc == account1_info
|
||||
));
|
||||
assert!(matches!(
|
||||
provider.basic_account(&account2),
|
||||
Ok(Some(acc)) if acc == account2_info
|
||||
));
|
||||
assert!(matches!(
|
||||
provider.basic_account(&account3),
|
||||
Ok(Some(acc)) if acc == account3_info
|
||||
));
|
||||
// assert storage
|
||||
// Get on dupsort would return only first value. This is good enough for this test.
|
||||
assert!(matches!(
|
||||
provider.tx_ref().get::<tables::PlainStorageState>(account1),
|
||||
Ok(Some(entry)) if entry.key == B256::with_last_byte(1) && entry.value == U256::from(2)
|
||||
));
|
||||
}
|
||||
// assert accounts
|
||||
assert!(
|
||||
matches!(provider.basic_account(&account1), Ok(Some(acc)) if acc == account1_info)
|
||||
);
|
||||
assert!(
|
||||
matches!(provider.basic_account(&account2), Ok(Some(acc)) if acc == account2_info)
|
||||
);
|
||||
assert!(
|
||||
matches!(provider.basic_account(&account3), Ok(Some(acc)) if acc == account3_info)
|
||||
);
|
||||
// assert storage
|
||||
// Get on dupsort would return only first value. This is good enough for this test.
|
||||
assert!(matches!(
|
||||
provider.tx_ref().get::<tables::PlainStorageState>(account1),
|
||||
Ok(Some(entry)) if entry.key == B256::with_last_byte(1) && entry.value == U256::from(2)
|
||||
));
|
||||
|
||||
let mut provider = factory.database_provider_rw().unwrap();
|
||||
let mut stage = stage();
|
||||
@@ -1171,8 +1165,7 @@ mod tests {
|
||||
processed: 0,
|
||||
total
|
||||
}
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}
|
||||
} if total == block.gas_used);
|
||||
|
||||
|
||||
@@ -397,7 +397,6 @@ mod tests {
|
||||
},
|
||||
..
|
||||
})),
|
||||
..
|
||||
},
|
||||
done: true,
|
||||
}) if block_number == previous_stage &&
|
||||
|
||||
@@ -594,8 +594,7 @@ mod tests {
|
||||
processed,
|
||||
total,
|
||||
}
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: true }) if block_number == tip.number &&
|
||||
from == checkpoint && to == previous_stage &&
|
||||
// -1 because we don't need to download the local head
|
||||
@@ -667,8 +666,7 @@ mod tests {
|
||||
processed,
|
||||
total,
|
||||
}
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: true }) if block_number == tip.number &&
|
||||
from == checkpoint && to == previous_stage &&
|
||||
// -1 because we don't need to download the local head
|
||||
|
||||
@@ -502,8 +502,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
},
|
||||
done: true
|
||||
}) if block_number == previous_stage && processed == total &&
|
||||
@@ -543,8 +542,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
},
|
||||
done: true
|
||||
}) if block_number == previous_stage && processed == total &&
|
||||
@@ -586,8 +584,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
})),
|
||||
..
|
||||
}))
|
||||
},
|
||||
done: true
|
||||
}) if block_number == previous_stage && processed == total &&
|
||||
|
||||
@@ -527,8 +527,7 @@ mod tests {
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed: 1,
|
||||
total: 1
|
||||
})),
|
||||
..
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage
|
||||
);
|
||||
|
||||
|
||||
@@ -337,12 +337,12 @@ mod tests {
|
||||
result,
|
||||
Ok(ExecOutput {
|
||||
checkpoint: StageCheckpoint {
|
||||
block_number,
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage && processed == total &&
|
||||
block_number,
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage && processed == total &&
|
||||
total == runner.db.count_entries::<tables::Transactions>().unwrap() as u64
|
||||
);
|
||||
|
||||
@@ -383,12 +383,12 @@ mod tests {
|
||||
result,
|
||||
Ok(ExecOutput {
|
||||
checkpoint: StageCheckpoint {
|
||||
block_number,
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage && processed == total &&
|
||||
block_number,
|
||||
stage_checkpoint: Some(StageUnitCheckpoint::Entities(EntitiesCheckpoint {
|
||||
processed,
|
||||
total
|
||||
}))
|
||||
}, done: true }) if block_number == previous_stage && processed == total &&
|
||||
total == runner.db.count_entries::<tables::Transactions>().unwrap() as u64
|
||||
);
|
||||
|
||||
|
||||
@@ -379,9 +379,6 @@ pub struct StageCheckpoint {
|
||||
pub stage_checkpoint: Option<StageUnitCheckpoint>,
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "reth-codec"))]
|
||||
reth_codecs::impl_compression_for_compact!(StageCheckpoint);
|
||||
|
||||
impl StageCheckpoint {
|
||||
/// Creates a new [`StageCheckpoint`] with only `block_number` set.
|
||||
pub fn new(block_number: BlockNumber) -> Self {
|
||||
@@ -434,21 +431,13 @@ impl StageCheckpoint {
|
||||
progress: entities,
|
||||
..
|
||||
}) => Some(entities),
|
||||
StageUnitCheckpoint::MerkleChangeSets(_) | StageUnitCheckpoint::Finish(_) => None,
|
||||
StageUnitCheckpoint::MerkleChangeSets(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Saves the progress of the Finish stage.
|
||||
#[derive(Default, Debug, Copy, Clone, PartialEq, Eq)]
|
||||
#[cfg_attr(any(test, feature = "test-utils"), derive(arbitrary::Arbitrary))]
|
||||
#[cfg_attr(any(test, feature = "reth-codec"), derive(reth_codecs::Compact))]
|
||||
#[cfg_attr(any(test, feature = "reth-codec"), reth_codecs::add_arbitrary_tests(compact))]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub struct FinishCheckpoint {
|
||||
/// The highest block with a partially persisted state and trie.
|
||||
pub partial_state_trie: Option<BlockNumber>,
|
||||
}
|
||||
#[cfg(any(test, feature = "reth-codec"))]
|
||||
reth_codecs::impl_compression_for_compact!(StageCheckpoint);
|
||||
|
||||
// TODO(alexey): add a merkle checkpoint. Currently it's hard because [`MerkleCheckpoint`]
|
||||
// is not a Copy type.
|
||||
@@ -476,8 +465,6 @@ pub enum StageUnitCheckpoint {
|
||||
/// Note: This variant is only kept for backward compatibility with the Compact codec.
|
||||
/// The `MerkleChangeSets` stage has been removed.
|
||||
MerkleChangeSets(MerkleChangeSetsCheckpoint),
|
||||
/// Saves the progress of the Finish stage.
|
||||
Finish(FinishCheckpoint),
|
||||
}
|
||||
|
||||
impl StageUnitCheckpoint {
|
||||
@@ -586,15 +573,6 @@ stage_unit_checkpoints!(
|
||||
index_history_stage_checkpoint,
|
||||
/// Sets the stage checkpoint to index history.
|
||||
with_index_history_stage_checkpoint
|
||||
),
|
||||
(
|
||||
6,
|
||||
Finish,
|
||||
FinishCheckpoint,
|
||||
/// Returns the finish stage checkpoint, if any.
|
||||
finish_stage_checkpoint,
|
||||
/// Sets the stage checkpoint to finish.
|
||||
with_finish_stage_checkpoint
|
||||
)
|
||||
);
|
||||
|
||||
@@ -686,15 +664,4 @@ mod tests {
|
||||
let (decoded, _) = MerkleCheckpoint::from_compact(&buf, encoded);
|
||||
assert_eq!(decoded, checkpoint);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn finish_checkpoint_roundtrip() {
|
||||
let checkpoint = StageCheckpoint::new(42)
|
||||
.with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(21) });
|
||||
|
||||
let mut buf = Vec::new();
|
||||
let encoded = checkpoint.to_compact(&mut buf);
|
||||
let (decoded, _) = StageCheckpoint::from_compact(&buf, encoded);
|
||||
assert_eq!(decoded, checkpoint);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ pub use id::StageId;
|
||||
mod checkpoints;
|
||||
pub use checkpoints::{
|
||||
AccountHashingCheckpoint, CheckpointBlockRange, EntitiesCheckpoint, ExecutionCheckpoint,
|
||||
FinishCheckpoint, HeadersCheckpoint, IndexHistoryCheckpoint, MerkleCheckpoint, StageCheckpoint,
|
||||
HeadersCheckpoint, IndexHistoryCheckpoint, MerkleCheckpoint, StageCheckpoint,
|
||||
StageUnitCheckpoint, StorageHashingCheckpoint, StorageRootMerkleCheckpoint,
|
||||
};
|
||||
|
||||
|
||||
@@ -17,7 +17,7 @@ use roaring::RoaringTreemap;
|
||||
/// - Direct access: elements can be accessed or queried without needing to decode the entire list.
|
||||
/// - [`RoaringTreemap`] backing: internally backed by [`RoaringTreemap`], which supports 64-bit
|
||||
/// integers.
|
||||
#[derive(Clone, PartialEq, Eq, Default, Deref)]
|
||||
#[derive(Clone, PartialEq, Default, Deref)]
|
||||
pub struct IntegerList(pub RoaringTreemap);
|
||||
|
||||
impl fmt::Debug for IntegerList {
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bytes};
|
||||
use parking_lot::RwLock;
|
||||
use reth_storage_api::{BalStore, GetBlockAccessListLimit};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
|
||||
/// Basic in-memory BAL store keyed by block hash.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct InMemoryBalStore {
|
||||
entries: Arc<RwLock<HashMap<BlockHash, Bytes>>>,
|
||||
}
|
||||
|
||||
impl BalStore for InMemoryBalStore {
|
||||
fn insert(
|
||||
&self,
|
||||
block_hash: BlockHash,
|
||||
_block_number: BlockNumber,
|
||||
bal: Bytes,
|
||||
) -> ProviderResult<()> {
|
||||
self.entries.write().insert(block_hash, bal);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult<Vec<Option<Bytes>>> {
|
||||
let entries = self.entries.read();
|
||||
let mut result = Vec::with_capacity(block_hashes.len());
|
||||
|
||||
for hash in block_hashes {
|
||||
result.push(entries.get(hash).cloned());
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn append_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
out: &mut Vec<Bytes>,
|
||||
) -> ProviderResult<()> {
|
||||
let entries = self.entries.read();
|
||||
let mut size = 0;
|
||||
|
||||
for hash in block_hashes {
|
||||
let bal = entries.get(hash).cloned().unwrap_or_else(|| Bytes::from_static(&[0xc0]));
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_by_range(&self, _start: BlockNumber, _count: u64) -> ProviderResult<Vec<Bytes>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::B256;
|
||||
|
||||
#[test]
|
||||
fn insert_and_lookup_by_hash() {
|
||||
let store = InMemoryBalStore::default();
|
||||
let hash = B256::random();
|
||||
let missing = B256::random();
|
||||
let bal = Bytes::from_static(b"bal");
|
||||
|
||||
store.insert(hash, 1, bal.clone()).unwrap();
|
||||
|
||||
assert_eq!(store.get_by_hashes(&[hash, missing]).unwrap(), vec![Some(bal), None]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn range_lookup_is_empty() {
|
||||
let store = InMemoryBalStore::default();
|
||||
|
||||
assert!(store.get_by_range(1, 10).unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn limited_lookup_returns_prefix() {
|
||||
let store = InMemoryBalStore::default();
|
||||
let hash0 = B256::random();
|
||||
let hash1 = B256::random();
|
||||
let hash2 = B256::random();
|
||||
let bal0 = Bytes::from_static(&[0xc1, 0x01]);
|
||||
let bal1 = Bytes::from_static(&[0xc1, 0x02]);
|
||||
let bal2 = Bytes::from_static(&[0xc1, 0x03]);
|
||||
|
||||
store.insert(hash0, 1, bal0.clone()).unwrap();
|
||||
store.insert(hash1, 2, bal1.clone()).unwrap();
|
||||
store.insert(hash2, 3, bal2).unwrap();
|
||||
|
||||
let limited = store
|
||||
.get_by_hashes_with_limit(
|
||||
&[hash0, hash1, hash2],
|
||||
GetBlockAccessListLimit::ResponseSizeSoftLimit(2),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(limited, vec![bal0, bal1]);
|
||||
}
|
||||
}
|
||||
@@ -24,8 +24,8 @@ pub mod providers;
|
||||
pub use providers::{
|
||||
DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider,
|
||||
HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory,
|
||||
PruneShardOutcome, PrunedIndices, SaveBlocksMode, SaveBlocksPlan, SaveBlocksPlanStep,
|
||||
StaticFileAccess, StaticFileProviderBuilder, StaticFileWriteCtx, StaticFileWriter,
|
||||
PruneShardOutcome, PrunedIndices, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder,
|
||||
StaticFileWriteCtx, StaticFileWriter,
|
||||
};
|
||||
|
||||
pub mod changeset_walker;
|
||||
@@ -38,9 +38,6 @@ pub mod test_utils;
|
||||
pub mod either_writer;
|
||||
pub use either_writer::*;
|
||||
|
||||
mod bal;
|
||||
pub use bal::InMemoryBalStore;
|
||||
|
||||
pub use reth_chain_state::{
|
||||
CanonStateNotification, CanonStateNotificationSender, CanonStateNotificationStream,
|
||||
CanonStateNotifications, CanonStateSubscriptions,
|
||||
@@ -51,9 +48,8 @@ pub use revm_database::states::OriginalValuesKnown;
|
||||
// reexport traits to avoid breaking changes
|
||||
pub use reth_static_file_types as static_file;
|
||||
pub use reth_storage_api::{
|
||||
BalProvider, BalStore, BalStoreHandle, GetBlockAccessListLimit, HistoryWriter,
|
||||
MetadataProvider, MetadataWriter, NoopBalStore, StateWriteConfig, StatsReader, StorageSettings,
|
||||
StorageSettingsCache,
|
||||
BalProvider, BalStore, BalStoreHandle, HistoryWriter, MetadataProvider, MetadataWriter,
|
||||
NoopBalStore, StateWriteConfig, StatsReader, StorageSettings, StorageSettingsCache,
|
||||
};
|
||||
/// Re-export provider error.
|
||||
pub use reth_storage_errors::provider::{ProviderError, ProviderResult};
|
||||
|
||||
@@ -6,8 +6,8 @@ use crate::{
|
||||
AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader,
|
||||
BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications,
|
||||
CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, InMemoryBalStore,
|
||||
ProviderError, ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, ProviderError,
|
||||
ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory,
|
||||
StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider,
|
||||
};
|
||||
@@ -111,7 +111,7 @@ impl<N: ProviderNodeTypes> BlockchainProvider<N> {
|
||||
finalized_header,
|
||||
safe_header,
|
||||
),
|
||||
bal_store: BalStoreHandle::new(InMemoryBalStore::default()),
|
||||
bal_store: BalStoreHandle::default(),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -790,8 +790,7 @@ mod tests {
|
||||
create_test_provider_factory, create_test_provider_factory_with_chain_spec,
|
||||
MockNodeTypesWithDB,
|
||||
},
|
||||
BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksMode, SaveBlocksPlan,
|
||||
SaveBlocksPlanStep,
|
||||
BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksMode,
|
||||
};
|
||||
use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag};
|
||||
use alloy_primitives::{BlockNumber, TxNumber, B256};
|
||||
@@ -1008,15 +1007,7 @@ mod tests {
|
||||
|
||||
// Push to disk
|
||||
let provider_rw = hook_provider.database_provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&SaveBlocksPlan::new(
|
||||
vec![lowest_memory_block],
|
||||
vec![SaveBlocksPlanStep::new(0..1, Some(1..1), true)],
|
||||
),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.save_blocks(vec![lowest_memory_block], SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
// Remove from memory
|
||||
|
||||
@@ -5,11 +5,11 @@ use crate::{
|
||||
},
|
||||
to_range,
|
||||
traits::{BlockSource, ReceiptProvider},
|
||||
BalProvider, BalStoreHandle, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider,
|
||||
DatabaseProviderFactory, EitherWriterDestination, HashedPostStateProvider, HeaderProvider,
|
||||
HeaderSyncGapProvider, MetadataProvider, ProviderError, PruneCheckpointReader,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StaticFileProviderFactory,
|
||||
StaticFileWriter, TransactionVariant, TransactionsProvider,
|
||||
BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory,
|
||||
EitherWriterDestination, HashedPostStateProvider, HeaderProvider, HeaderSyncGapProvider,
|
||||
MetadataProvider, ProviderError, PruneCheckpointReader, RocksDBProviderFactory,
|
||||
StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, StaticFileWriter,
|
||||
TransactionVariant, TransactionsProvider,
|
||||
};
|
||||
use alloy_consensus::transaction::TransactionMeta;
|
||||
use alloy_eips::BlockHashOrNumber;
|
||||
@@ -51,9 +51,6 @@ pub use provider::{
|
||||
CommitOrder, DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode,
|
||||
};
|
||||
|
||||
mod save_blocks;
|
||||
pub use save_blocks::{SaveBlocksPlan, SaveBlocksPlanStep};
|
||||
|
||||
use super::ProviderNodeTypes;
|
||||
use reth_trie::KeccakKeyHasher;
|
||||
|
||||
@@ -93,8 +90,6 @@ pub struct ProviderFactory<N: NodeTypesWithDB> {
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
/// Changeset cache for trie unwinding
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Store for block access lists.
|
||||
bal_store: BalStoreHandle,
|
||||
/// Task runtime for spawning parallel I/O work.
|
||||
runtime: reth_tasks::Runtime,
|
||||
/// Minimum distance from tip required before pruning can occur.
|
||||
@@ -157,7 +152,6 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
storage_settings: Arc::new(RwLock::new(storage_settings)),
|
||||
rocksdb_provider,
|
||||
changeset_cache: ChangesetCache::new(),
|
||||
bal_store: BalStoreHandle::default(),
|
||||
runtime,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
read_only_sync: None,
|
||||
@@ -589,12 +583,6 @@ impl<N: NodeTypesWithDB> NodePrimitivesProvider for ProviderFactory<N> {
|
||||
type Primitives = N::Primitives;
|
||||
}
|
||||
|
||||
impl<N: NodeTypesWithDB> BalProvider for ProviderFactory<N> {
|
||||
fn bal_store(&self) -> &BalStoreHandle {
|
||||
&self.bal_store
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: ProviderNodeTypes> DatabaseProviderFactory for ProviderFactory<N> {
|
||||
type DB = N::DB;
|
||||
type Provider = DatabaseProvider<<N::DB as Database>::TX, N>;
|
||||
@@ -967,7 +955,6 @@ where
|
||||
storage_settings,
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
bal_store,
|
||||
runtime,
|
||||
minimum_pruning_distance,
|
||||
read_only_sync,
|
||||
@@ -981,7 +968,6 @@ where
|
||||
.field("storage_settings", &*storage_settings.read())
|
||||
.field("rocksdb_provider", &rocksdb_provider)
|
||||
.field("changeset_cache", &changeset_cache)
|
||||
.field("bal_store", &bal_store)
|
||||
.field("runtime", &runtime)
|
||||
.field("minimum_pruning_distance", &minimum_pruning_distance)
|
||||
.field(
|
||||
@@ -1003,7 +989,6 @@ impl<N: NodeTypesWithDB> Clone for ProviderFactory<N> {
|
||||
storage_settings: self.storage_settings.clone(),
|
||||
rocksdb_provider: self.rocksdb_provider.clone(),
|
||||
changeset_cache: self.changeset_cache.clone(),
|
||||
bal_store: self.bal_store.clone(),
|
||||
runtime: self.runtime.clone(),
|
||||
minimum_pruning_distance: self.minimum_pruning_distance,
|
||||
read_only_sync: self.read_only_sync.clone(),
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,90 +0,0 @@
|
||||
use alloy_eips::BlockNumHash;
|
||||
use reth_chain_state::ExecutedBlock;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use std::ops::Range;
|
||||
|
||||
/// A single persistence step over a contiguous region of [`SaveBlocksPlan::blocks`].
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SaveBlocksPlanStep {
|
||||
/// Range of [`SaveBlocksPlan::blocks`] covered by this step.
|
||||
pub block_range: Range<usize>,
|
||||
/// Optional range of blocks whose state/trie updates should be used to mask this step's
|
||||
/// durable state/trie writes.
|
||||
///
|
||||
/// `Some(empty_range)` means persist state/trie without any masking. `None` means skip
|
||||
/// durable state/trie persistence for this step.
|
||||
pub state_trie_masking_range: Option<Range<usize>>,
|
||||
/// Whether to persist non-state/trie data for this step.
|
||||
pub persist_rest: bool,
|
||||
}
|
||||
|
||||
impl SaveBlocksPlanStep {
|
||||
/// Creates a new persistence step.
|
||||
pub const fn new(
|
||||
block_range: Range<usize>,
|
||||
state_trie_masking_range: Option<Range<usize>>,
|
||||
persist_rest: bool,
|
||||
) -> Self {
|
||||
Self { block_range, state_trie_masking_range, persist_rest }
|
||||
}
|
||||
|
||||
/// Returns `true` if this step persists state/trie data.
|
||||
pub const fn persists_state_trie(&self) -> bool {
|
||||
self.state_trie_masking_range.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
/// Plan for a single `save_blocks` persistence cycle.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SaveBlocksPlan<N: NodePrimitives = EthPrimitives> {
|
||||
/// Canonical blocks covered by this plan.
|
||||
pub blocks: Vec<ExecutedBlock<N>>,
|
||||
/// Ordered persistence steps over [`Self::blocks`].
|
||||
pub steps: Vec<SaveBlocksPlanStep>,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> SaveBlocksPlan<N> {
|
||||
/// Creates a new save plan.
|
||||
pub const fn new(blocks: Vec<ExecutedBlock<N>>, steps: Vec<SaveBlocksPlanStep>) -> Self {
|
||||
Self { blocks, steps }
|
||||
}
|
||||
|
||||
/// Returns `true` if the plan contains no blocks to persist.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.last_block().is_none()
|
||||
}
|
||||
|
||||
/// Returns the highest block covered by this plan.
|
||||
pub fn last_block(&self) -> Option<BlockNumHash> {
|
||||
let last_index =
|
||||
self.steps.iter().rev().find_map(|step| step.block_range.end.checked_sub(1))?;
|
||||
self.blocks.get(last_index).map(|block| block.recovered_block().num_hash())
|
||||
}
|
||||
|
||||
/// Returns the highest block whose state/trie data is durably persisted by this plan.
|
||||
pub fn last_state_trie_block(&self) -> Option<BlockNumHash> {
|
||||
let last_index = self
|
||||
.steps
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|step| step.persists_state_trie())?
|
||||
.block_range
|
||||
.end
|
||||
.checked_sub(1)?;
|
||||
self.blocks.get(last_index).map(|block| block.recovered_block().num_hash())
|
||||
}
|
||||
|
||||
/// Returns the contiguous range of blocks whose non-state/trie outputs are persisted.
|
||||
pub fn persist_rest_range(&self) -> Option<Range<usize>> {
|
||||
let mut ranges =
|
||||
self.steps.iter().filter(|step| step.persist_rest).map(|step| &step.block_range);
|
||||
let first = ranges.next()?.clone();
|
||||
let merged = ranges.fold(first, |mut merged, range| {
|
||||
debug_assert_eq!(merged.end, range.start, "persist_rest steps must be contiguous");
|
||||
merged.end = range.end;
|
||||
merged
|
||||
});
|
||||
Some(merged)
|
||||
}
|
||||
}
|
||||
@@ -20,7 +20,7 @@ pub use state::{
|
||||
HistoricalStateProviderRef, HistoryInfo, LowestAvailableBlocks,
|
||||
},
|
||||
latest::{LatestStateProvider, LatestStateProviderRef},
|
||||
overlay::{OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory},
|
||||
overlay::{OverlayStateProvider, OverlayStateProviderFactory},
|
||||
};
|
||||
|
||||
mod consistent_view;
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
use super::overlay::{Overlay, OverlayBuilder, OverlaySource};
|
||||
use crate::{
|
||||
AccountReader, BlockHashReader, ChangeSetReader, EitherReader, HashedPostStateProvider,
|
||||
ProviderError, RocksDBProviderFactory, StateProvider, StateRootProvider,
|
||||
@@ -12,11 +11,10 @@ use reth_db_api::{
|
||||
transaction::DbTx,
|
||||
BlockNumberList,
|
||||
};
|
||||
use reth_primitives_traits::{Account, Bytecode, NodePrimitives};
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use reth_storage_api::{
|
||||
BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, PruneCheckpointReader,
|
||||
StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider,
|
||||
StorageSettingsCache,
|
||||
BlockNumReader, BytecodeReader, DBProvider, NodePrimitivesProvider, StateProofProvider,
|
||||
StorageChangeSetReader, StorageRootProvider, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_trie::{
|
||||
@@ -25,15 +23,16 @@ use reth_trie::{
|
||||
trie_cursor::InMemoryTrieCursorFactory,
|
||||
updates::TrieUpdates,
|
||||
witness::TrieWitness,
|
||||
AccountProof, ExecutionWitnessMode, HashedPostState, HashedStorage, KeccakKeyHasher,
|
||||
MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, TrieInput,
|
||||
TrieInputSorted,
|
||||
AccountProof, ExecutionWitnessMode, HashedPostState, HashedPostStateSorted, HashedStorage,
|
||||
KeccakKeyHasher, MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot,
|
||||
TrieInput, TrieInputSorted,
|
||||
};
|
||||
use reth_trie_db::{
|
||||
ChangesetCache, DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot,
|
||||
hashed_storage_from_reverts_with_provider, DatabaseProof, DatabaseStateRoot,
|
||||
DatabaseStorageProof, DatabaseStorageRoot,
|
||||
};
|
||||
|
||||
use std::{fmt::Debug, marker::PhantomData, sync::Arc};
|
||||
use std::fmt::Debug;
|
||||
|
||||
type DbStateRoot<'a, TX, A> = StateRoot<
|
||||
reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>,
|
||||
@@ -121,47 +120,21 @@ impl HistoryInfo {
|
||||
/// - [`tables::AccountChangeSets`]
|
||||
/// - [`tables::StorageChangeSets`]
|
||||
#[derive(Debug)]
|
||||
pub struct HistoricalStateProviderRef<
|
||||
'b,
|
||||
Provider,
|
||||
N: NodePrimitives = <Provider as NodePrimitivesProvider>::Primitives,
|
||||
> where
|
||||
Provider: NodePrimitivesProvider<Primitives = N>,
|
||||
{
|
||||
pub struct HistoricalStateProviderRef<'b, Provider> {
|
||||
/// Database provider
|
||||
provider: &'b Provider,
|
||||
/// Changeset cache handle for retrieving trie changesets.
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Block number is main index for the history state of accounts and storages.
|
||||
block_number: BlockNumber,
|
||||
/// Lowest blocks at which different parts of the state are available.
|
||||
lowest_available_blocks: LowestAvailableBlocks,
|
||||
/// Marker for the provider's node primitives.
|
||||
_primitives: PhantomData<N>,
|
||||
}
|
||||
|
||||
impl<'b, Provider, N> HistoricalStateProviderRef<'b, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<'b, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumReader>
|
||||
HistoricalStateProviderRef<'b, Provider>
|
||||
{
|
||||
/// Create new `StateProvider` for historical block number
|
||||
pub fn new(
|
||||
provider: &'b Provider,
|
||||
block_number: BlockNumber,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
changeset_cache,
|
||||
block_number,
|
||||
lowest_available_blocks: Default::default(),
|
||||
_primitives: PhantomData,
|
||||
}
|
||||
pub fn new(provider: &'b Provider, block_number: BlockNumber) -> Self {
|
||||
Self { provider, block_number, lowest_available_blocks: Default::default() }
|
||||
}
|
||||
|
||||
/// Create new `StateProvider` for historical block number and lowest block numbers at which
|
||||
@@ -170,15 +143,8 @@ where
|
||||
provider: &'b Provider,
|
||||
block_number: BlockNumber,
|
||||
lowest_available_blocks: LowestAvailableBlocks,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
changeset_cache,
|
||||
block_number,
|
||||
lowest_available_blocks,
|
||||
_primitives: PhantomData,
|
||||
}
|
||||
Self { provider, block_number, lowest_available_blocks }
|
||||
}
|
||||
|
||||
/// Lookup an account in the `AccountsHistory` table using `EitherReader`.
|
||||
@@ -287,11 +253,17 @@ where
|
||||
Ok(tip.saturating_sub(self.block_number) > limit)
|
||||
}
|
||||
|
||||
fn build_overlay(&self, input: TrieInputSorted) -> ProviderResult<TrieInputSorted>
|
||||
/// Retrieve revert hashed state for this history provider.
|
||||
fn revert_state(&self) -> ProviderResult<HashedPostStateSorted>
|
||||
where
|
||||
Provider:
|
||||
BlockHashReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache,
|
||||
Provider: StorageSettingsCache,
|
||||
{
|
||||
if !self.lowest_available_blocks.is_account_history_available(self.block_number) ||
|
||||
!self.lowest_available_blocks.is_storage_history_available(self.block_number)
|
||||
{
|
||||
return Err(ProviderError::StateAtBlockPruned(self.block_number))
|
||||
}
|
||||
|
||||
if self.check_distance_against_limit(EPOCH_SLOTS)? {
|
||||
tracing::warn!(
|
||||
target: "providers::historical_sp",
|
||||
@@ -300,21 +272,27 @@ where
|
||||
);
|
||||
}
|
||||
|
||||
// Historical providers expose state at the start of `self.block_number`, so the overlay
|
||||
// builder needs the previous canonical block hash to preserve those semantics.
|
||||
let target_block = self.block_number.saturating_sub(1);
|
||||
let anchor_hash = self
|
||||
.provider
|
||||
.block_hash(target_block)?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?;
|
||||
reth_trie_db::from_reverts_auto(self.provider, self.block_number..)
|
||||
}
|
||||
|
||||
let TrieInputSorted { nodes, state, prefix_sets } = input;
|
||||
let overlay_builder = OverlayBuilder::<N>::new(anchor_hash, self.changeset_cache.clone())
|
||||
.with_overlay_source(Some(OverlaySource::Immediate { trie: nodes, state }));
|
||||
let Overlay { trie_updates, hashed_post_state } =
|
||||
overlay_builder.build_overlay(self.provider)?;
|
||||
/// Retrieve revert hashed storage for this history provider and target address.
|
||||
fn revert_storage(&self, address: Address) -> ProviderResult<HashedStorage>
|
||||
where
|
||||
Provider: StorageSettingsCache,
|
||||
{
|
||||
if !self.lowest_available_blocks.is_storage_history_available(self.block_number) {
|
||||
return Err(ProviderError::StateAtBlockPruned(self.block_number))
|
||||
}
|
||||
|
||||
Ok(TrieInputSorted::new(trie_updates, hashed_post_state, prefix_sets))
|
||||
if self.check_distance_against_limit(EPOCH_SLOTS * 10)? {
|
||||
tracing::warn!(
|
||||
target: "providers::historical_sp",
|
||||
target = self.block_number,
|
||||
"Attempt to calculate storage root for an old block might result in OOM"
|
||||
);
|
||||
}
|
||||
|
||||
hashed_storage_from_reverts_with_provider(self.provider, address, self.block_number)
|
||||
}
|
||||
|
||||
/// Set the lowest block number at which the account history is available.
|
||||
@@ -336,26 +314,21 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider + BlockNumReader + NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
{
|
||||
impl<Provider: DBProvider + BlockNumReader> HistoricalStateProviderRef<'_, Provider> {
|
||||
fn tx(&self) -> &Provider::Tx {
|
||||
self.provider.tx_ref()
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> AccountReader for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
> AccountReader for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Get basic account information.
|
||||
fn basic_account(&self, address: &Address) -> ProviderResult<Option<Account>> {
|
||||
@@ -383,11 +356,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> BlockHashReader for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider:
|
||||
DBProvider + BlockNumReader + BlockHashReader + NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<Provider: DBProvider + BlockNumReader + BlockHashReader> BlockHashReader
|
||||
for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Get block hash by number.
|
||||
fn block_hash(&self, number: u64) -> ProviderResult<Option<B256>> {
|
||||
@@ -403,32 +373,31 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> StateRootProvider for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ PruneCheckpointReader
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
> StateRootProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
fn state_root(&self, hashed_state: HashedPostState) -> ProviderResult<B256> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(
|
||||
TrieInput::from_state(hashed_state),
|
||||
))?;
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes(self.tx(), input)?)
|
||||
let mut revert_state = self.revert_state()?;
|
||||
let hashed_state_sorted = hashed_state.into_sorted();
|
||||
revert_state.extend_ref_and_sort(&hashed_state_sorted);
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root(self.tx(), &revert_state)?)
|
||||
})
|
||||
}
|
||||
|
||||
fn state_root_from_nodes(&self, input: TrieInput) -> ProviderResult<B256> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?;
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes(self.tx(), input)?)
|
||||
let mut input = input;
|
||||
input.prepend(self.revert_state()?.into());
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes(
|
||||
self.tx(),
|
||||
TrieInputSorted::from_unsorted(input),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -437,10 +406,10 @@ where
|
||||
hashed_state: HashedPostState,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(
|
||||
TrieInput::from_state(hashed_state),
|
||||
))?;
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes_with_updates(self.tx(), input)?)
|
||||
let mut revert_state = self.revert_state()?;
|
||||
let hashed_state_sorted = hashed_state.into_sorted();
|
||||
revert_state.extend_ref_and_sort(&hashed_state_sorted);
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_with_updates(self.tx(), &revert_state)?)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -449,24 +418,23 @@ where
|
||||
input: TrieInput,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(input))?;
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes_with_updates(self.tx(), input)?)
|
||||
let mut input = input;
|
||||
input.prepend(self.revert_state()?.into());
|
||||
Ok(<DbStateRoot<'_, _, A>>::overlay_root_from_nodes_with_updates(
|
||||
self.tx(),
|
||||
TrieInputSorted::from_unsorted(input),
|
||||
)?)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> StorageRootProvider for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ PruneCheckpointReader
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
> StorageRootProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
fn storage_root(
|
||||
&self,
|
||||
@@ -474,20 +442,9 @@ where
|
||||
hashed_storage: HashedStorage,
|
||||
) -> ProviderResult<B256> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(
|
||||
TrieInput::from_state(HashedPostState::from_hashed_storage(
|
||||
alloy_primitives::keccak256(address),
|
||||
hashed_storage,
|
||||
)),
|
||||
))?;
|
||||
let hashed_storage = input
|
||||
.state
|
||||
.account_storages()
|
||||
.get(&alloy_primitives::keccak256(address))
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into();
|
||||
<DbStorageRoot<'_, _, A>>::overlay_root(self.tx(), address, hashed_storage)
|
||||
let mut revert_storage = self.revert_storage(address)?;
|
||||
revert_storage.extend(&hashed_storage);
|
||||
<DbStorageRoot<'_, _, A>>::overlay_root(self.tx(), address, revert_storage)
|
||||
.map_err(|err| ProviderError::Database(err.into()))
|
||||
})
|
||||
}
|
||||
@@ -499,24 +456,13 @@ where
|
||||
hashed_storage: HashedStorage,
|
||||
) -> ProviderResult<reth_trie::StorageProof> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(
|
||||
TrieInput::from_state(HashedPostState::from_hashed_storage(
|
||||
alloy_primitives::keccak256(address),
|
||||
hashed_storage,
|
||||
)),
|
||||
))?;
|
||||
let hashed_storage = input
|
||||
.state
|
||||
.account_storages()
|
||||
.get(&alloy_primitives::keccak256(address))
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into();
|
||||
let mut revert_storage = self.revert_storage(address)?;
|
||||
revert_storage.extend(&hashed_storage);
|
||||
<DbStorageProof<'_, _, A>>::overlay_storage_proof(
|
||||
self.tx(),
|
||||
address,
|
||||
slot,
|
||||
hashed_storage,
|
||||
revert_storage,
|
||||
)
|
||||
.map_err(ProviderError::from)
|
||||
})
|
||||
@@ -529,42 +475,26 @@ where
|
||||
hashed_storage: HashedStorage,
|
||||
) -> ProviderResult<StorageMultiProof> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let input = self.build_overlay(TrieInputSorted::from_unsorted(
|
||||
TrieInput::from_state(HashedPostState::from_hashed_storage(
|
||||
alloy_primitives::keccak256(address),
|
||||
hashed_storage,
|
||||
)),
|
||||
))?;
|
||||
let hashed_storage = input
|
||||
.state
|
||||
.account_storages()
|
||||
.get(&alloy_primitives::keccak256(address))
|
||||
.cloned()
|
||||
.unwrap_or_default()
|
||||
.into();
|
||||
let mut revert_storage = self.revert_storage(address)?;
|
||||
revert_storage.extend(&hashed_storage);
|
||||
<DbStorageProof<'_, _, A>>::overlay_storage_multiproof(
|
||||
self.tx(),
|
||||
address,
|
||||
slots,
|
||||
hashed_storage,
|
||||
revert_storage,
|
||||
)
|
||||
.map_err(ProviderError::from)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> StateProofProvider for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ PruneCheckpointReader
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
> StateProofProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Get account and storage proofs.
|
||||
fn proof(
|
||||
@@ -574,13 +504,8 @@ where
|
||||
slots: &[B256],
|
||||
) -> ProviderResult<AccountProof> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let TrieInputSorted { nodes, state, prefix_sets } =
|
||||
self.build_overlay(TrieInputSorted::from_unsorted(input))?;
|
||||
let input = TrieInput::new(
|
||||
Arc::unwrap_or_clone(nodes).into(),
|
||||
Arc::unwrap_or_clone(state).into(),
|
||||
prefix_sets,
|
||||
);
|
||||
let mut input = input;
|
||||
input.prepend(self.revert_state()?.into());
|
||||
let proof = <DbProof<'_, _, A> as DatabaseProof>::from_tx(self.tx());
|
||||
proof.overlay_account_proof(input, address, slots).map_err(ProviderError::from)
|
||||
})
|
||||
@@ -592,13 +517,8 @@ where
|
||||
targets: MultiProofTargets,
|
||||
) -> ProviderResult<MultiProof> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let TrieInputSorted { nodes, state, prefix_sets } =
|
||||
self.build_overlay(TrieInputSorted::from_unsorted(input))?;
|
||||
let input = TrieInput::new(
|
||||
Arc::unwrap_or_clone(nodes).into(),
|
||||
Arc::unwrap_or_clone(state).into(),
|
||||
prefix_sets,
|
||||
);
|
||||
let mut input = input;
|
||||
input.prepend(self.revert_state()?.into());
|
||||
let proof = <DbProof<'_, _, A> as DatabaseProof>::from_tx(self.tx());
|
||||
proof.overlay_multiproof(input, targets).map_err(ProviderError::from)
|
||||
})
|
||||
@@ -611,19 +531,21 @@ where
|
||||
mode: ExecutionWitnessMode,
|
||||
) -> ProviderResult<Vec<Bytes>> {
|
||||
reth_trie_db::with_adapter!(self.provider, |A| {
|
||||
let TrieInputSorted { nodes, state, prefix_sets } =
|
||||
self.build_overlay(TrieInputSorted::from_unsorted(input))?;
|
||||
let mut input = input;
|
||||
input.prepend(self.revert_state()?.into());
|
||||
let nodes_sorted = input.nodes.into_sorted();
|
||||
let state_sorted = input.state.into_sorted();
|
||||
let witness = TrieWitness::new(
|
||||
InMemoryTrieCursorFactory::new(
|
||||
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
|
||||
nodes.as_ref(),
|
||||
&nodes_sorted,
|
||||
),
|
||||
HashedPostStateCursorFactory::new(
|
||||
reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()),
|
||||
state.as_ref(),
|
||||
&state_sorted,
|
||||
),
|
||||
)
|
||||
.with_prefix_sets_mut(prefix_sets)
|
||||
.with_prefix_sets_mut(input.prefix_sets)
|
||||
.with_execution_witness_mode(mode);
|
||||
let witness =
|
||||
if mode.is_canonical() { witness } else { witness.always_include_root_node() };
|
||||
@@ -638,29 +560,22 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> HashedPostStateProvider for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
{
|
||||
impl<Provider> HashedPostStateProvider for HistoricalStateProviderRef<'_, Provider> {
|
||||
fn hashed_post_state(&self, bundle_state: &revm_database::BundleState) -> HashedPostState {
|
||||
HashedPostState::from_bundle_state::<KeccakKeyHasher>(bundle_state.state())
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> StateProvider for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ PruneCheckpointReader
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ BlockNumReader
|
||||
+ BlockHashReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
> StateProvider for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Expects a plain (unhashed) storage key slot.
|
||||
fn storage(
|
||||
@@ -672,10 +587,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider, N> BytecodeReader for HistoricalStateProviderRef<'_, Provider, N>
|
||||
where
|
||||
Provider: DBProvider + BlockNumReader + NodePrimitivesProvider<Primitives = N>,
|
||||
N: NodePrimitives,
|
||||
impl<Provider: DBProvider + BlockNumReader> BytecodeReader
|
||||
for HistoricalStateProviderRef<'_, Provider>
|
||||
{
|
||||
/// Get account code by its hash
|
||||
fn bytecode_by_hash(&self, code_hash: &B256) -> ProviderResult<Option<Bytecode>> {
|
||||
@@ -689,8 +602,6 @@ where
|
||||
pub struct HistoricalStateProvider<Provider> {
|
||||
/// Database provider.
|
||||
provider: Provider,
|
||||
/// Changeset cache handle for retrieving trie changesets.
|
||||
changeset_cache: ChangesetCache,
|
||||
/// State at the block number is the main indexer of the state.
|
||||
block_number: BlockNumber,
|
||||
/// Lowest blocks at which different parts of the state are available.
|
||||
@@ -701,17 +612,8 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
|
||||
HistoricalStateProvider<Provider>
|
||||
{
|
||||
/// Create new `StateProvider` for historical block number
|
||||
pub fn new(
|
||||
provider: Provider,
|
||||
block_number: BlockNumber,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self {
|
||||
Self {
|
||||
provider,
|
||||
changeset_cache,
|
||||
block_number,
|
||||
lowest_available_blocks: Default::default(),
|
||||
}
|
||||
pub fn new(provider: Provider, block_number: BlockNumber) -> Self {
|
||||
Self { provider, block_number, lowest_available_blocks: Default::default() }
|
||||
}
|
||||
|
||||
/// Set the lowest block number at which the account history is available.
|
||||
@@ -731,30 +633,20 @@ impl<Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + BlockNumR
|
||||
self.lowest_available_blocks.storage_history_block_number = Some(block_number);
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<
|
||||
Provider: DBProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader
|
||||
+ NodePrimitivesProvider,
|
||||
> HistoricalStateProvider<Provider>
|
||||
{
|
||||
/// Returns a new provider that takes the `TX` as reference
|
||||
#[inline(always)]
|
||||
fn as_ref(&self) -> HistoricalStateProviderRef<'_, Provider> {
|
||||
const fn as_ref(&self) -> HistoricalStateProviderRef<'_, Provider> {
|
||||
HistoricalStateProviderRef::new_with_lowest_available_blocks(
|
||||
&self.provider,
|
||||
self.block_number,
|
||||
self.lowest_available_blocks,
|
||||
self.changeset_cache.clone(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Delegates all provider impls to [HistoricalStateProviderRef]
|
||||
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + PruneCheckpointReader + StageCheckpointReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
|
||||
reth_storage_api::macros::delegate_provider_impls!(HistoricalStateProvider<Provider> where [Provider: DBProvider + BlockNumReader + BlockHashReader + ChangeSetReader + StorageChangeSetReader + StorageSettingsCache + RocksDBProviderFactory + NodePrimitivesProvider]);
|
||||
|
||||
/// Lowest blocks at which different parts of the state are available.
|
||||
/// They may be [Some] if pruning is enabled.
|
||||
@@ -887,11 +779,9 @@ mod tests {
|
||||
use reth_primitives_traits::{Account, StorageEntry};
|
||||
use reth_storage_api::{
|
||||
BlockHashReader, BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory,
|
||||
NodePrimitivesProvider, PruneCheckpointReader, StageCheckpointReader,
|
||||
StorageChangeSetReader, StorageSettingsCache,
|
||||
NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderError;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
|
||||
const ADDRESS: Address = address!("0x0000000000000000000000000000000000000001");
|
||||
const HIGHER_ADDRESS: Address = address!("0x0000000000000000000000000000000000000005");
|
||||
@@ -906,8 +796,6 @@ mod tests {
|
||||
+ BlockHashReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ PruneCheckpointReader
|
||||
+ StageCheckpointReader
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory
|
||||
+ NodePrimitivesProvider,
|
||||
@@ -982,49 +870,48 @@ mod tests {
|
||||
|
||||
// run
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 1).basic_account(&ADDRESS),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 2, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 2).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at3
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 3).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at3
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 4).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at7
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 7).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at7
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 9).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at10
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 10).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at10
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 11).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_at15
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).basic_account(&ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 16).basic_account(&ADDRESS),
|
||||
Ok(Some(acc)) if acc == acc_plain
|
||||
));
|
||||
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new())
|
||||
.basic_account(&HIGHER_ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 1).basic_account(&HIGHER_ADDRESS),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).basic_account(&HIGHER_ADDRESS),
|
||||
HistoricalStateProviderRef::new(&db, 1000).basic_account(&HIGHER_ADDRESS),
|
||||
Ok(Some(acc)) if acc == higher_acc_plain
|
||||
));
|
||||
}
|
||||
@@ -1083,46 +970,43 @@ mod tests {
|
||||
|
||||
// run
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(U256::ZERO))
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at7.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at7.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at10.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at10.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at15.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_plain.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new())
|
||||
.storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == higher_entry_plain.value
|
||||
));
|
||||
}
|
||||
@@ -1141,7 +1025,6 @@ mod tests {
|
||||
account_history_block_number: Some(3),
|
||||
storage_history_block_number: Some(3),
|
||||
},
|
||||
ChangesetCache::new(),
|
||||
);
|
||||
assert!(matches!(
|
||||
provider.account_history_lookup(ADDRESS),
|
||||
@@ -1161,7 +1044,6 @@ mod tests {
|
||||
account_history_block_number: Some(2),
|
||||
storage_history_block_number: Some(2),
|
||||
},
|
||||
ChangesetCache::new(),
|
||||
);
|
||||
assert!(matches!(
|
||||
provider.account_history_lookup(ADDRESS),
|
||||
@@ -1181,7 +1063,6 @@ mod tests {
|
||||
account_history_block_number: Some(1),
|
||||
storage_history_block_number: Some(1),
|
||||
},
|
||||
ChangesetCache::new(),
|
||||
);
|
||||
assert!(matches!(
|
||||
provider.account_history_lookup(ADDRESS),
|
||||
@@ -1262,46 +1143,43 @@ mod tests {
|
||||
let db = factory.provider().unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(U256::ZERO))
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at7.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at7.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at10.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at10.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_at15.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == entry_plain.value
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new())
|
||||
.storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(Some(expected_value)) if expected_value == higher_entry_plain.value
|
||||
));
|
||||
}
|
||||
@@ -1405,46 +1283,43 @@ mod tests {
|
||||
let db = factory.provider().unwrap();
|
||||
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 0).storage(ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new())
|
||||
.storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 3).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(U256::ZERO))
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 4).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(7)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 7).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(7)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 9).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(10)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 10).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(10)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 11).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(15)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 16).storage(ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(100)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new())
|
||||
.storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(None)
|
||||
));
|
||||
assert!(matches!(
|
||||
HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE),
|
||||
HistoricalStateProviderRef::new(&db, 1000).storage(HIGHER_ADDRESS, STORAGE),
|
||||
Ok(Some(v)) if v == U256::from(1000)
|
||||
));
|
||||
}
|
||||
|
||||
@@ -1,14 +1,10 @@
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, B256};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use metrics::{Counter, Histogram};
|
||||
use reth_chain_state::{EthPrimitives, LazyOverlay};
|
||||
use reth_chain_state::LazyOverlay;
|
||||
use reth_db_api::{tables, transaction::DbTx, DatabaseError};
|
||||
use reth_errors::{ProviderError, ProviderResult};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::{
|
||||
dashmap::{self, DashMap},
|
||||
NodePrimitives,
|
||||
};
|
||||
use reth_primitives_traits::dashmap::{self, DashMap};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::StageId;
|
||||
use reth_storage_api::{
|
||||
@@ -28,7 +24,6 @@ use reth_trie_db::{
|
||||
PackedStoragesTrie,
|
||||
};
|
||||
use std::{
|
||||
ops::RangeInclusive,
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -56,9 +51,9 @@ pub(crate) struct OverlayStateProviderMetrics {
|
||||
|
||||
/// Contains all fields required to initialize an [`OverlayStateProvider`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) struct Overlay {
|
||||
pub(super) trie_updates: Arc<TrieUpdatesSorted>,
|
||||
pub(super) hashed_post_state: Arc<HashedPostStateSorted>,
|
||||
struct Overlay {
|
||||
trie_updates: Arc<TrieUpdatesSorted>,
|
||||
hashed_post_state: Arc<HashedPostStateSorted>,
|
||||
}
|
||||
|
||||
/// Source of overlay data for [`OverlayStateProviderFactory`].
|
||||
@@ -66,7 +61,7 @@ pub(super) struct Overlay {
|
||||
/// Either provides immediate pre-computed overlay data, or a lazy overlay that computes
|
||||
/// on first access.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(super) enum OverlaySource<N: NodePrimitives = EthPrimitives> {
|
||||
pub enum OverlaySource {
|
||||
/// Immediate overlay with already-computed data.
|
||||
Immediate {
|
||||
/// Trie updates overlay.
|
||||
@@ -75,69 +70,85 @@ pub(super) enum OverlaySource<N: NodePrimitives = EthPrimitives> {
|
||||
state: Arc<HashedPostStateSorted>,
|
||||
},
|
||||
/// Lazy overlay computed on first access.
|
||||
Lazy(LazyOverlay<N>),
|
||||
Lazy(LazyOverlay),
|
||||
}
|
||||
|
||||
/// Builder for calculating trie and hashed-state overlays.
|
||||
impl OverlaySource {
|
||||
/// Resolve the overlay source into (trie, state) tuple.
|
||||
///
|
||||
/// For lazy overlays, this may block waiting for deferred data.
|
||||
fn resolve(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
match self {
|
||||
Self::Immediate { trie, state } => (Arc::clone(trie), Arc::clone(state)),
|
||||
Self::Lazy(lazy) => lazy.as_overlay(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Factory for creating overlay state providers with optional reverts and overlays.
|
||||
///
|
||||
/// This stores the overlay configuration and the logic for resolving immediate/lazy overlays and
|
||||
/// collecting reverts. It is intentionally independent from any provider factory or overlay cache.
|
||||
/// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a
|
||||
/// particular block, and/or with additional overlay information added on top.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OverlayBuilder<N: NodePrimitives = EthPrimitives> {
|
||||
/// Anchor hash to revert the DB state to before applying overlays.
|
||||
anchor_hash: B256,
|
||||
pub struct OverlayStateProviderFactory<F> {
|
||||
/// The underlying database provider factory
|
||||
factory: F,
|
||||
/// Optional block hash for collecting reverts
|
||||
block_hash: Option<B256>,
|
||||
/// Optional overlay source (lazy or immediate).
|
||||
overlay_source: Option<OverlaySource<N>>,
|
||||
overlay_source: Option<OverlaySource>,
|
||||
/// Changeset cache handle for retrieving trie changesets
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Metrics for tracking provider operations
|
||||
metrics: OverlayStateProviderMetrics,
|
||||
/// A cache which maps `db_tip -> Overlay`. If the db tip changes during usage of the factory
|
||||
/// then a new entry will get added to this, but in most cases only one entry is present.
|
||||
overlay_cache: Arc<DashMap<BlockNumber, Overlay>>,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
/// Create a new overlay builder.
|
||||
pub fn new(anchor_hash: B256, changeset_cache: ChangesetCache) -> Self {
|
||||
impl<F> OverlayStateProviderFactory<F> {
|
||||
/// Create a new overlay state provider factory
|
||||
pub fn new(factory: F, changeset_cache: ChangesetCache) -> Self {
|
||||
Self {
|
||||
anchor_hash,
|
||||
factory,
|
||||
block_hash: None,
|
||||
overlay_source: None,
|
||||
changeset_cache,
|
||||
metrics: OverlayStateProviderMetrics::default(),
|
||||
overlay_cache: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the block hash for collecting reverts. All state will be reverted to the point
|
||||
/// _after_ this block has been processed.
|
||||
pub const fn with_block_hash(mut self, block_hash: Option<B256>) -> Self {
|
||||
self.block_hash = block_hash;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the overlay source (lazy or immediate).
|
||||
///
|
||||
/// This overlay will be applied on top of any reverts applied via `anchor_hash`.
|
||||
pub(super) fn with_overlay_source(mut self, source: Option<OverlaySource<N>>) -> Self {
|
||||
if let Some(OverlaySource::Lazy(lazy_overlay)) = source.as_ref() {
|
||||
self.assert_lazy_overlay_anchor(lazy_overlay);
|
||||
}
|
||||
/// This overlay will be applied on top of any reverts applied via `with_block_hash`.
|
||||
pub fn with_overlay_source(mut self, source: Option<OverlaySource>) -> Self {
|
||||
self.overlay_source = source;
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
fn assert_lazy_overlay_anchor(&self, lazy_overlay: &LazyOverlay<N>) {
|
||||
let Some(lazy_overlay_anchor) = lazy_overlay.anchor_hash() else { return };
|
||||
assert!(
|
||||
lazy_overlay_anchor == self.anchor_hash,
|
||||
"LazyOverlay's anchor ({}) != OverlayBuilder's anchor ({})",
|
||||
lazy_overlay_anchor,
|
||||
self.anchor_hash,
|
||||
);
|
||||
}
|
||||
|
||||
/// Set a lazy overlay that will be computed on first access.
|
||||
///
|
||||
/// Panics if the [`LazyOverlay`]'s anchor hash does not match [`Self`]'s `anchor_hash`.
|
||||
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay<N>>) -> Self {
|
||||
if let Some(lazy_overlay) = lazy_overlay.as_ref() {
|
||||
self.assert_lazy_overlay_anchor(lazy_overlay);
|
||||
}
|
||||
/// Convenience method that wraps the lazy overlay in `OverlaySource::Lazy`.
|
||||
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay>) -> Self {
|
||||
self.overlay_source = lazy_overlay.map(OverlaySource::Lazy);
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the hashed state overlay.
|
||||
///
|
||||
/// This overlay will be applied on top of any reverts applied via `with_block_hash`.
|
||||
pub fn with_hashed_state_overlay(
|
||||
mut self,
|
||||
hashed_state_overlay: Option<Arc<HashedPostStateSorted>>,
|
||||
@@ -147,6 +158,8 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
trie: Arc::new(TrieUpdatesSorted::default()),
|
||||
state,
|
||||
});
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
}
|
||||
self
|
||||
}
|
||||
@@ -160,9 +173,9 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
Some(OverlaySource::Immediate { state, .. }) => {
|
||||
Arc::make_mut(state).extend_ref_and_sort(&other);
|
||||
}
|
||||
Some(OverlaySource::Lazy(overlay)) => {
|
||||
Some(OverlaySource::Lazy(lazy)) => {
|
||||
// Resolve lazy overlay and convert to immediate with extension
|
||||
let (trie, mut state) = overlay.as_overlay(self.anchor_hash);
|
||||
let (trie, mut state) = lazy.as_overlay();
|
||||
Arc::make_mut(&mut state).extend_ref_and_sort(&other);
|
||||
self.overlay_source = Some(OverlaySource::Immediate { trie, state });
|
||||
}
|
||||
@@ -173,72 +186,60 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
});
|
||||
}
|
||||
}
|
||||
// Clear the overlay cache since we've updated the source.
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<F> OverlayStateProviderFactory<F>
|
||||
where
|
||||
F: DatabaseProviderFactory,
|
||||
F::Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
/// Resolves the effective overlay (trie updates, hashed state).
|
||||
///
|
||||
/// If an overlay source is set, it is resolved (blocking if lazy).
|
||||
/// Otherwise, returns empty defaults.
|
||||
fn resolve_overlays(
|
||||
&self,
|
||||
anchor_hash: BlockHash,
|
||||
) -> ProviderResult<(Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>)> {
|
||||
fn resolve_overlays(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
match &self.overlay_source {
|
||||
Some(OverlaySource::Lazy(lazy_overlay)) => Ok(lazy_overlay.as_overlay(anchor_hash)),
|
||||
Some(OverlaySource::Immediate { trie, state }) => {
|
||||
if anchor_hash != self.anchor_hash {
|
||||
return Err(ProviderError::other(std::io::Error::other(format!(
|
||||
"anchor_hash {anchor_hash} doesn't match OverlayBuilder's configured anchor ({})",
|
||||
self.anchor_hash
|
||||
))))
|
||||
}
|
||||
Ok((Arc::clone(trie), Arc::clone(state)))
|
||||
Some(source) => source.resolve(),
|
||||
None => {
|
||||
(Arc::new(TrieUpdatesSorted::default()), Arc::new(HashedPostStateSorted::default()))
|
||||
}
|
||||
None => Ok((
|
||||
Arc::new(TrieUpdatesSorted::default()),
|
||||
Arc::new(HashedPostStateSorted::default()),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the block number for [`Self`]'s `anchor_hash` field.
|
||||
fn get_block_number<Provider>(&self, provider: &Provider) -> ProviderResult<BlockNumber>
|
||||
where
|
||||
Provider: BlockNumReader,
|
||||
{
|
||||
provider
|
||||
.convert_hash_or_number(self.anchor_hash.into())?
|
||||
.ok_or(ProviderError::BlockHashNotFound(self.anchor_hash))
|
||||
/// Returns the block number for [`Self`]'s `block_hash` field, if any.
|
||||
fn get_requested_block_number(
|
||||
&self,
|
||||
provider: &F::Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>> {
|
||||
if let Some(block_hash) = self.block_hash {
|
||||
Ok(Some(
|
||||
provider
|
||||
.convert_hash_or_number(block_hash.into())?
|
||||
.ok_or_else(|| ProviderError::BlockHashNotFound(block_hash))?,
|
||||
))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the highest blocks whose state/trie data and non-state/trie data are durably
|
||||
/// available in the database.
|
||||
fn get_db_tip_blocks<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<(BlockNumHash, BlockNumHash)>
|
||||
where
|
||||
Provider: StageCheckpointReader + BlockNumReader,
|
||||
{
|
||||
let checkpoint = provider.get_stage_checkpoint(StageId::Finish)?.ok_or_else(|| {
|
||||
ProviderError::InsufficientChangesets { requested: 0, available: 0..=0 }
|
||||
})?;
|
||||
let block_number = checkpoint
|
||||
.finish_stage_checkpoint()
|
||||
.and_then(|finish| finish.partial_state_trie)
|
||||
.unwrap_or(checkpoint.block_number);
|
||||
let state_trie_tip_hash = provider
|
||||
.convert_number(block_number.into())?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(block_number.into()))?;
|
||||
let finish_tip_number = checkpoint.block_number;
|
||||
let finish_tip_hash = provider
|
||||
.convert_number(finish_tip_number.into())?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(finish_tip_number.into()))?;
|
||||
Ok((
|
||||
BlockNumHash::new(block_number, state_trie_tip_hash),
|
||||
BlockNumHash::new(finish_tip_number, finish_tip_hash),
|
||||
))
|
||||
/// Returns the block which is at the tip of the DB, i.e. the block which the state tables of
|
||||
/// the DB are currently synced to.
|
||||
fn get_db_tip_block_number(&self, provider: &F::Provider) -> ProviderResult<BlockNumber> {
|
||||
provider
|
||||
.get_stage_checkpoint(StageId::Finish)?
|
||||
.as_ref()
|
||||
.map(|chk| chk.block_number)
|
||||
.ok_or_else(|| ProviderError::InsufficientChangesets { requested: 0, available: 0..=0 })
|
||||
}
|
||||
|
||||
/// Returns whether or not it is required to collect reverts, and validates that there are
|
||||
@@ -246,24 +247,18 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
///
|
||||
/// Takes into account both the stage checkpoint and the prune checkpoint to determine the
|
||||
/// available data range.
|
||||
fn reverts_required<Provider>(
|
||||
fn reverts_required(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
state_trie_tip_block: BlockNumHash,
|
||||
finish_tip_block: BlockNumHash,
|
||||
) -> ProviderResult<Option<RangeInclusive<BlockNumber>>>
|
||||
where
|
||||
Provider: BlockNumReader + PruneCheckpointReader,
|
||||
{
|
||||
// If the anchor is the current durable state/trie frontier then there won't be any
|
||||
// reverts
|
||||
// necessary.
|
||||
if state_trie_tip_block.hash == self.anchor_hash {
|
||||
return Ok(None)
|
||||
provider: &F::Provider,
|
||||
db_tip_block: BlockNumber,
|
||||
requested_block: BlockNumber,
|
||||
) -> ProviderResult<bool> {
|
||||
// If the requested block is the DB tip then there won't be any reverts necessary, and we
|
||||
// can simply return Ok.
|
||||
if db_tip_block == requested_block {
|
||||
return Ok(false)
|
||||
}
|
||||
|
||||
let anchor_number = self.get_block_number(provider)?;
|
||||
|
||||
// Check account history prune checkpoint to determine the lower bound of available data.
|
||||
// The prune checkpoint's block_number is the highest pruned block, so data is available
|
||||
// starting from the next block.
|
||||
@@ -273,49 +268,31 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
.map(|block_number| block_number + 1)
|
||||
.unwrap_or_default();
|
||||
|
||||
let available_range = lower_bound..=finish_tip_block.number;
|
||||
let available_range = lower_bound..=db_tip_block;
|
||||
|
||||
// Check if the requested block is within the available range
|
||||
if !available_range.contains(&anchor_number) {
|
||||
if !available_range.contains(&requested_block) {
|
||||
return Err(ProviderError::InsufficientChangesets {
|
||||
requested: anchor_number,
|
||||
requested: requested_block,
|
||||
available: available_range,
|
||||
});
|
||||
}
|
||||
|
||||
if anchor_number > state_trie_tip_block.number {
|
||||
return Err(ProviderError::InsufficientChangesets {
|
||||
requested: anchor_number,
|
||||
available: lower_bound..=state_trie_tip_block.number,
|
||||
})
|
||||
}
|
||||
|
||||
Ok(Some(anchor_number + 1..=finish_tip_block.number))
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Calculates a new [`Overlay`] given a transaction and the current durable state/trie
|
||||
/// frontier.
|
||||
/// Calculates a new [`Overlay`] given a transaction and the current db tip.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "providers::state::overlay",
|
||||
skip_all,
|
||||
fields(?state_trie_tip_block, ?finish_tip_block, anchor_hash = ?self.anchor_hash)
|
||||
fields(%db_tip_block)
|
||||
)]
|
||||
fn calculate_overlay<Provider>(
|
||||
fn calculate_overlay(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
state_trie_tip_block: BlockNumHash,
|
||||
finish_tip_block: BlockNumHash,
|
||||
) -> ProviderResult<Overlay>
|
||||
where
|
||||
Provider: ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
provider: &F::Provider,
|
||||
db_tip_block: BlockNumber,
|
||||
) -> ProviderResult<Overlay> {
|
||||
//
|
||||
// Set up variables we'll use for recording metrics. There's two different code-paths here,
|
||||
// and we want to make sure both record metrics, so we do metrics recording after.
|
||||
@@ -324,13 +301,18 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
let trie_updates_total_len;
|
||||
let hashed_state_updates_total_len;
|
||||
|
||||
// Collect any reverts which are required to bring the DB view back to the anchor hash.
|
||||
let (trie_updates, hashed_post_state) = if let Some(revert_blocks) =
|
||||
self.reverts_required(provider, state_trie_tip_block, finish_tip_block)?
|
||||
// If block_hash is provided, collect reverts
|
||||
let (trie_updates, hashed_post_state) = if let Some(from_block) =
|
||||
self.get_requested_block_number(provider)? &&
|
||||
self.reverts_required(provider, db_tip_block, from_block)?
|
||||
{
|
||||
debug!(
|
||||
target: "providers::state::overlay",
|
||||
?revert_blocks,
|
||||
block_hash = ?self.block_hash,
|
||||
from_block,
|
||||
db_tip_block,
|
||||
range_start = from_block + 1,
|
||||
range_end = db_tip_block,
|
||||
"Collecting trie reverts for overlay state provider"
|
||||
);
|
||||
|
||||
@@ -344,8 +326,9 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
|
||||
// Use changeset cache to retrieve and accumulate reverts to restore state after
|
||||
// from_block
|
||||
let accumulated_reverts =
|
||||
self.changeset_cache.get_or_compute_range(provider, revert_blocks.clone())?;
|
||||
let accumulated_reverts = self
|
||||
.changeset_cache
|
||||
.get_or_compute_range(provider, (from_block + 1)..=db_tip_block)?;
|
||||
|
||||
retrieve_trie_reverts_duration = start.elapsed();
|
||||
accumulated_reverts
|
||||
@@ -356,14 +339,14 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
let _guard = debug_span!(target: "providers::state::overlay", "retrieving_hashed_state_reverts").entered();
|
||||
|
||||
let start = Instant::now();
|
||||
let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?;
|
||||
let res = reth_trie_db::from_reverts_auto(provider, from_block + 1..)?;
|
||||
retrieve_hashed_state_reverts_duration = start.elapsed();
|
||||
res
|
||||
};
|
||||
|
||||
// Resolve overlays (lazy or immediate) and extend reverts with them.
|
||||
// If reverts are empty, use overlays directly to avoid cloning.
|
||||
let (overlay_trie, overlay_state) = self.resolve_overlays(self.anchor_hash)?;
|
||||
let (overlay_trie, overlay_state) = self.resolve_overlays();
|
||||
|
||||
let trie_updates = if trie_reverts.is_empty() {
|
||||
overlay_trie
|
||||
@@ -388,6 +371,8 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
|
||||
debug!(
|
||||
target: "providers::state::overlay",
|
||||
block_hash = ?self.block_hash,
|
||||
?from_block,
|
||||
num_trie_updates = ?trie_updates_total_len,
|
||||
num_state_updates = ?hashed_state_updates_total_len,
|
||||
"Reverted to target block",
|
||||
@@ -395,9 +380,8 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
|
||||
(trie_updates, hashed_state_updates)
|
||||
} else {
|
||||
// If no reverts are needed then the requested anchor is exactly the durable
|
||||
// state/trie frontier. Use overlays directly from that frontier.
|
||||
let (trie_updates, hashed_state) = self.resolve_overlays(state_trie_tip_block.hash)?;
|
||||
// If no block_hash, use overlays directly (resolving lazy if set)
|
||||
let (trie_updates, hashed_state) = self.resolve_overlays();
|
||||
|
||||
retrieve_trie_reverts_duration = Duration::ZERO;
|
||||
retrieve_hashed_state_reverts_duration = Duration::ZERO;
|
||||
@@ -420,108 +404,37 @@ impl<N: NodePrimitives> OverlayBuilder<N> {
|
||||
Ok(Overlay { trie_updates, hashed_post_state })
|
||||
}
|
||||
|
||||
/// Builds the effective overlay for the given provider.
|
||||
/// Fetches an [`Overlay`] from the cache based on the current db tip block. If there is no
|
||||
/// cached value then this calculates the [`Overlay`] and populates the cache.
|
||||
#[instrument(level = "debug", target = "providers::state::overlay", skip_all)]
|
||||
pub(super) fn build_overlay<Provider>(&self, provider: &Provider) -> ProviderResult<Overlay>
|
||||
where
|
||||
Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
let (state_trie_tip_block, finish_tip_block) = self.get_db_tip_blocks(provider)?;
|
||||
self.calculate_overlay(provider, state_trie_tip_block, finish_tip_block)
|
||||
}
|
||||
}
|
||||
fn get_overlay(&self, provider: &F::Provider) -> ProviderResult<Overlay> {
|
||||
// No anchor block — just resolve the in-memory overlay directly.
|
||||
if self.block_hash.is_none() {
|
||||
let (trie_updates, hashed_post_state) = self.resolve_overlays();
|
||||
return Ok(Overlay { trie_updates, hashed_post_state })
|
||||
}
|
||||
|
||||
/// Factory for creating overlay state providers with optional reverts and overlays.
|
||||
///
|
||||
/// This factory allows building an `OverlayStateProvider` whose DB state has been reverted to a
|
||||
/// particular block, and/or with additional overlay information added on top.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct OverlayStateProviderFactory<F, N: NodePrimitives = EthPrimitives> {
|
||||
/// The underlying database provider factory
|
||||
factory: F,
|
||||
/// Overlay builder containing the configuration and overlay calculation logic.
|
||||
overlay_builder: OverlayBuilder<N>,
|
||||
/// A cache which maps `(state_trie_tip_hash, finish_tip_hash) -> Overlay`.
|
||||
///
|
||||
/// Under partial persistence the overlay depends on both the durable trie frontier and the
|
||||
/// fully durable Finish frontier, so both hashes are part of the cache key.
|
||||
overlay_cache: Arc<DashMap<(BlockHash, BlockHash), Overlay>>,
|
||||
}
|
||||
let db_tip_block = self.get_db_tip_block_number(provider)?;
|
||||
|
||||
impl<F, N: NodePrimitives> OverlayStateProviderFactory<F, N> {
|
||||
/// Create a new overlay state provider factory
|
||||
pub fn new(factory: F, overlay_builder: OverlayBuilder<N>) -> Self {
|
||||
Self { factory, overlay_builder, overlay_cache: Default::default() }
|
||||
}
|
||||
|
||||
/// Set a lazy overlay that will be computed on first access.
|
||||
pub fn with_lazy_overlay(mut self, lazy_overlay: Option<LazyOverlay<N>>) -> Self {
|
||||
self.overlay_builder = self.overlay_builder.with_lazy_overlay(lazy_overlay);
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the hashed state overlay.
|
||||
pub fn with_hashed_state_overlay(
|
||||
mut self,
|
||||
hashed_state_overlay: Option<Arc<HashedPostStateSorted>>,
|
||||
) -> Self {
|
||||
self.overlay_builder = self.overlay_builder.with_hashed_state_overlay(hashed_state_overlay);
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Extends the existing hashed state overlay with the given [`HashedPostStateSorted`].
|
||||
pub fn with_extended_hashed_state_overlay(mut self, other: HashedPostStateSorted) -> Self {
|
||||
self.overlay_builder = self.overlay_builder.with_extended_hashed_state_overlay(other);
|
||||
self.overlay_cache = Default::default();
|
||||
self
|
||||
}
|
||||
|
||||
/// Fetches an [`Overlay`] from the cache based on the current durable frontiers. If there is
|
||||
/// no cached value then this calculates the [`Overlay`] and populates the cache.
|
||||
#[instrument(level = "debug", target = "providers::state::overlay", skip_all)]
|
||||
fn get_overlay<Provider>(&self, provider: &Provider) -> ProviderResult<Overlay>
|
||||
where
|
||||
Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
let (state_trie_tip_block, finish_tip_block) =
|
||||
self.overlay_builder.get_db_tip_blocks(provider)?;
|
||||
|
||||
let overlay =
|
||||
match self.overlay_cache.entry((state_trie_tip_block.hash, finish_tip_block.hash)) {
|
||||
dashmap::Entry::Occupied(entry) => entry.get().clone(),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
self.overlay_builder.metrics.overlay_cache_misses.increment(1);
|
||||
let overlay = self.overlay_builder.build_overlay(provider)?;
|
||||
entry.insert(overlay.clone());
|
||||
overlay
|
||||
}
|
||||
};
|
||||
let overlay = match self.overlay_cache.entry(db_tip_block) {
|
||||
dashmap::Entry::Occupied(entry) => entry.get().clone(),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
self.metrics.overlay_cache_misses.increment(1);
|
||||
let overlay = self.calculate_overlay(provider, db_tip_block)?;
|
||||
entry.insert(overlay.clone());
|
||||
overlay
|
||||
}
|
||||
};
|
||||
|
||||
Ok(overlay)
|
||||
}
|
||||
}
|
||||
|
||||
impl<F, N> DatabaseProviderROFactory for OverlayStateProviderFactory<F, N>
|
||||
impl<F> DatabaseProviderROFactory for OverlayStateProviderFactory<F>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
F: DatabaseProviderFactory,
|
||||
F::Provider: StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ DBProvider
|
||||
+ BlockNumReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
@@ -538,14 +451,14 @@ where
|
||||
let provider = {
|
||||
let start = Instant::now();
|
||||
let res = self.factory.database_provider_ro()?;
|
||||
self.overlay_builder.metrics.create_provider_duration.record(start.elapsed());
|
||||
self.metrics.create_provider_duration.record(start.elapsed());
|
||||
res
|
||||
};
|
||||
|
||||
let Overlay { trie_updates, hashed_post_state } = self.get_overlay(&provider)?;
|
||||
|
||||
let is_v2 = provider.cached_storage_settings().is_v2();
|
||||
self.overlay_builder.metrics.database_provider_ro_duration.record(overall_start.elapsed());
|
||||
self.metrics.database_provider_ro_duration.record(overall_start.elapsed());
|
||||
Ok(OverlayStateProvider::new(provider, trie_updates, hashed_post_state, is_v2))
|
||||
}
|
||||
}
|
||||
@@ -667,191 +580,3 @@ where
|
||||
hashed_cursor_factory.hashed_storage_cursor(hashed_address)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
test_utils::create_test_provider_factory, BlockWriter, SaveBlocksMode, SaveBlocksPlan,
|
||||
SaveBlocksPlanStep,
|
||||
};
|
||||
use alloy_primitives::{B256, U256};
|
||||
use reth_chain_state::{test_utils::TestBlockBuilder, ComputedTrieData, ExecutedBlock};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_stages_types::{FinishCheckpoint, StageCheckpoint};
|
||||
use reth_storage_api::StageCheckpointWriter;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostState, HashedStorage};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn full_save_plan(
|
||||
blocks: impl IntoIterator<Item = ExecutedBlock<EthPrimitives>>,
|
||||
) -> SaveBlocksPlan<EthPrimitives> {
|
||||
let blocks = blocks.into_iter().collect::<Vec<_>>();
|
||||
let full_range = 0..blocks.len();
|
||||
SaveBlocksPlan::new(
|
||||
blocks,
|
||||
vec![SaveBlocksPlanStep::new(
|
||||
full_range.clone(),
|
||||
Some(full_range.end..full_range.end),
|
||||
true,
|
||||
)],
|
||||
)
|
||||
}
|
||||
|
||||
fn partial_save_plan(
|
||||
blocks: impl IntoIterator<Item = ExecutedBlock<EthPrimitives>>,
|
||||
steps: Vec<SaveBlocksPlanStep>,
|
||||
) -> SaveBlocksPlan<EthPrimitives> {
|
||||
SaveBlocksPlan::new(blocks.into_iter().collect(), steps)
|
||||
}
|
||||
|
||||
fn with_unique_state(
|
||||
block: &ExecutedBlock<EthPrimitives>,
|
||||
id: u8,
|
||||
) -> ExecutedBlock<EthPrimitives> {
|
||||
let hashed_address = B256::with_last_byte(id);
|
||||
let hashed_slot = B256::with_last_byte(id.saturating_add(32));
|
||||
let hashed_state = HashedPostState::default()
|
||||
.with_accounts([(hashed_address, Some(Account::default()))])
|
||||
.with_storages([(
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]),
|
||||
)])
|
||||
.into_sorted();
|
||||
|
||||
ExecutedBlock::new(
|
||||
Arc::clone(&block.recovered_block),
|
||||
Arc::clone(&block.execution_output),
|
||||
ComputedTrieData::without_trie_input(
|
||||
Arc::new(hashed_state),
|
||||
Arc::new(TrieUpdatesSorted::default()),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_overlay_uses_partial_trie_frontier_as_lazy_overlay_base() {
|
||||
let factory = create_test_provider_factory();
|
||||
let mut block_builder = TestBlockBuilder::eth();
|
||||
let blocks = block_builder
|
||||
.get_executed_blocks(0..5)
|
||||
.enumerate()
|
||||
.map(|(index, block)| with_unique_state(&block, index as u8 + 1))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let state_trie_tip = &blocks[1];
|
||||
let finish_tip = &blocks[3];
|
||||
let lazy_overlay_blocks = vec![blocks[4].clone(), blocks[3].clone(), blocks[2].clone()];
|
||||
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
provider_rw.insert_block(blocks[0].recovered_block()).unwrap();
|
||||
provider_rw.insert_block(state_trie_tip.recovered_block()).unwrap();
|
||||
provider_rw.insert_block(blocks[2].recovered_block()).unwrap();
|
||||
provider_rw.insert_block(finish_tip.recovered_block()).unwrap();
|
||||
provider_rw
|
||||
.save_stage_checkpoint(
|
||||
StageId::Finish,
|
||||
StageCheckpoint::new(finish_tip.block_number()).with_finish_stage_checkpoint(
|
||||
FinishCheckpoint { partial_state_trie: Some(state_trie_tip.block_number()) },
|
||||
),
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider = factory.provider().unwrap();
|
||||
let overlay = OverlayBuilder::<EthPrimitives>::new(
|
||||
state_trie_tip.recovered_block().hash(),
|
||||
ChangesetCache::new(),
|
||||
)
|
||||
.with_lazy_overlay(Some(LazyOverlay::new(lazy_overlay_blocks)))
|
||||
.build_overlay(&provider)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(overlay.hashed_post_state.accounts.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_overlay_rejects_anchor_between_state_trie_frontier_and_finish() {
|
||||
let factory = create_test_provider_factory();
|
||||
let mut block_builder = TestBlockBuilder::eth().with_state();
|
||||
|
||||
let genesis = block_builder.get_executed_blocks(0..1).next().unwrap();
|
||||
let blocks = block_builder.get_executed_blocks(1..4).collect::<Vec<_>>();
|
||||
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&full_save_plan(std::slice::from_ref(&genesis).to_vec()),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&partial_save_plan(
|
||||
blocks.clone(),
|
||||
vec![
|
||||
SaveBlocksPlanStep::new(0..1, Some(1..3), true),
|
||||
SaveBlocksPlanStep::new(1..3, None, true),
|
||||
],
|
||||
),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider = factory.provider().unwrap();
|
||||
let anchor = blocks[1].recovered_block().hash();
|
||||
let err = OverlayBuilder::<EthPrimitives>::new(anchor, ChangesetCache::new())
|
||||
.with_lazy_overlay(Some(LazyOverlay::new(vec![blocks[2].clone()])))
|
||||
.build_overlay(&provider)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(err, ProviderError::InsufficientChangesets { .. }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn build_overlay_rejects_finish_anchor_without_trie_bridge() {
|
||||
let factory = create_test_provider_factory();
|
||||
let mut block_builder = TestBlockBuilder::eth().with_state();
|
||||
|
||||
let genesis = block_builder.get_executed_blocks(0..1).next().unwrap();
|
||||
let blocks = block_builder.get_executed_blocks(1..4).collect::<Vec<_>>();
|
||||
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&full_save_plan(std::slice::from_ref(&genesis).to_vec()),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
provider_rw
|
||||
.save_blocks(
|
||||
&partial_save_plan(
|
||||
blocks.clone(),
|
||||
vec![
|
||||
SaveBlocksPlanStep::new(0..1, Some(1..3), true),
|
||||
SaveBlocksPlanStep::new(1..3, None, true),
|
||||
],
|
||||
),
|
||||
SaveBlocksMode::Full,
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider = factory.provider().unwrap();
|
||||
let finish_anchor = blocks[2].recovered_block().hash();
|
||||
|
||||
let err = OverlayBuilder::<EthPrimitives>::new(finish_anchor, ChangesetCache::new())
|
||||
.with_lazy_overlay(None)
|
||||
.build_overlay(&provider)
|
||||
.unwrap_err();
|
||||
|
||||
assert!(matches!(err, ProviderError::InsufficientChangesets { .. }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -696,15 +696,14 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
|
||||
/// Updates the `self.reader` internal index.
|
||||
fn update_index(&self) -> ProviderResult<()> {
|
||||
let segment = self.writer.user_header().segment();
|
||||
|
||||
// We find the maximum block of the segment by checking this writer's last block.
|
||||
//
|
||||
// However if there's no block range (because there's no data), we try to calculate it by
|
||||
// subtracting 1 from the expected block start, resulting on the last block of the
|
||||
// previous file — but only if that file actually exists. If the previous file doesn't
|
||||
// exist (e.g. first-ever file for a segment starting past range boundary), there's
|
||||
// nothing to index.
|
||||
// previous file.
|
||||
//
|
||||
// If that expected block start is 0, then it means that there's no actual block data, and
|
||||
// there's no block data in static files.
|
||||
let segment_max_block = self
|
||||
.writer
|
||||
.user_header()
|
||||
@@ -712,18 +711,12 @@ impl<N: NodePrimitives> StaticFileProviderRW<N> {
|
||||
.as_ref()
|
||||
.map(|block_range| block_range.end())
|
||||
.or_else(|| {
|
||||
let expected_start = self.writer.user_header().expected_block_start();
|
||||
if expected_start <= self.reader().genesis_block_number() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let prev_block = expected_start - 1;
|
||||
let prev_range = self.reader().find_fixed_range(segment, prev_block);
|
||||
let prev_path = self.reader().directory().join(segment.filename(&prev_range));
|
||||
prev_path.exists().then_some(prev_block)
|
||||
(self.writer.user_header().expected_block_start() >
|
||||
self.reader().genesis_block_number())
|
||||
.then(|| self.writer.user_header().expected_block_start() - 1)
|
||||
});
|
||||
|
||||
self.reader().update_index(segment, segment_max_block)
|
||||
self.reader().update_index(self.writer.user_header().segment(), segment_max_block)
|
||||
}
|
||||
|
||||
/// Ensures that the writer is positioned at the specified block number.
|
||||
|
||||
@@ -809,78 +809,4 @@ mod tests {
|
||||
"Should have 7 blocks * 5 changes = 35 rows"
|
||||
);
|
||||
}
|
||||
|
||||
/// Opening a writer for a block past the first range boundary should succeed
|
||||
/// even when no previous static file exists for the segment.
|
||||
#[test]
|
||||
fn test_get_writer_no_previous_file() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
let provider = setup_test_provider(&static_dir, 100);
|
||||
|
||||
// Request a writer starting at block 250, which falls into range 200..=299.
|
||||
// No file exists for range 100..=199 (the "previous" range).
|
||||
// This must not panic or error.
|
||||
let mut writer = provider
|
||||
.get_writer(250, StaticFileSegment::AccountChangeSets)
|
||||
.expect("get_writer should succeed without previous file");
|
||||
|
||||
// The index should have no entry for AccountChangeSets yet (empty jar).
|
||||
assert!(
|
||||
provider.get_highest_static_file_block(StaticFileSegment::AccountChangeSets).is_none(),
|
||||
"Empty jar should not create an index entry"
|
||||
);
|
||||
|
||||
// Writing data requires padding from the range start (200) to block 250,
|
||||
// same as the migration code does.
|
||||
let writer_start = writer.next_block_number();
|
||||
for block in writer_start..250 {
|
||||
writer.append_account_changeset(vec![], block).unwrap();
|
||||
}
|
||||
let changeset = generate_test_changeset(250, 2);
|
||||
writer.append_account_changeset(changeset, 250).unwrap();
|
||||
writer.commit().unwrap();
|
||||
|
||||
assert_eq!(
|
||||
provider.get_highest_static_file_block(StaticFileSegment::AccountChangeSets).unwrap(),
|
||||
250,
|
||||
"After writing block 250, highest block should be 250"
|
||||
);
|
||||
}
|
||||
|
||||
/// When a previous file DOES exist, opening a new empty writer for the next
|
||||
/// range should still update the index to point at the previous file.
|
||||
#[test]
|
||||
fn test_get_writer_with_previous_file() {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
let provider = setup_test_provider(&static_dir, 100);
|
||||
|
||||
// Write blocks 0..=99 to fill the first file completely.
|
||||
{
|
||||
let mut writer = provider.get_writer(0, StaticFileSegment::AccountChangeSets).unwrap();
|
||||
for block in 0..100 {
|
||||
writer.append_account_changeset(generate_test_changeset(block, 1), block).unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
assert_eq!(
|
||||
provider.get_highest_static_file_block(StaticFileSegment::AccountChangeSets).unwrap(),
|
||||
99
|
||||
);
|
||||
|
||||
// Now get a writer for block 100 (next range 100..=199).
|
||||
// The previous file (0..=99) exists, so this should succeed.
|
||||
let writer = provider
|
||||
.get_writer(100, StaticFileSegment::AccountChangeSets)
|
||||
.expect("get_writer should succeed with previous file");
|
||||
|
||||
// The index should still reflect the previous file's max block.
|
||||
assert_eq!(
|
||||
provider.get_highest_static_file_block(StaticFileSegment::AccountChangeSets).unwrap(),
|
||||
99,
|
||||
"Index should still point at previous file's max block"
|
||||
);
|
||||
|
||||
drop(writer);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,70 +22,12 @@ pub trait BalStore: Send + Sync + 'static {
|
||||
/// The returned vector must align with `block_hashes`.
|
||||
fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult<Vec<Option<Bytes>>>;
|
||||
|
||||
/// Fetch BAL response entries for the given block hashes, stopping after the soft limit is
|
||||
/// exceeded.
|
||||
///
|
||||
/// Entries are returned in request order. Unavailable BALs are represented as an RLP-encoded
|
||||
/// empty list (`0xc0`). The limit is soft: the entry that exceeds the limit is included.
|
||||
fn get_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
) -> ProviderResult<Vec<Bytes>> {
|
||||
let mut out = Vec::new();
|
||||
self.append_by_hashes_with_limit(block_hashes, limit, &mut out)?;
|
||||
out.shrink_to_fit();
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// Extends the given vector with BAL response entries for the given hashes.
|
||||
///
|
||||
/// This adheres to the expected behavior of [`Self::get_by_hashes_with_limit`].
|
||||
fn append_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
out: &mut Vec<Bytes>,
|
||||
) -> ProviderResult<()> {
|
||||
let mut size = 0;
|
||||
for bal in self.get_by_hashes(block_hashes)? {
|
||||
let bal = bal.unwrap_or_else(|| Bytes::from_static(&[0xc0]));
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetch BALs for the requested range.
|
||||
///
|
||||
/// Implementations may stop at the first gap and return the contiguous prefix.
|
||||
fn get_by_range(&self, start: BlockNumber, count: u64) -> ProviderResult<Vec<Bytes>>;
|
||||
}
|
||||
|
||||
/// The limit to enforce for [`BalStore::get_by_hashes_with_limit`].
|
||||
#[derive(Debug, Clone, Copy, Eq, PartialEq)]
|
||||
pub enum GetBlockAccessListLimit {
|
||||
/// No limit, return all BALs.
|
||||
None,
|
||||
/// Enforce a size limit on the returned BALs, for example 2MB.
|
||||
ResponseSizeSoftLimit(usize),
|
||||
}
|
||||
|
||||
impl GetBlockAccessListLimit {
|
||||
/// Returns true if the given size exceeds the limit.
|
||||
#[inline]
|
||||
pub const fn exceeds(&self, size: usize) -> bool {
|
||||
match self {
|
||||
Self::None => false,
|
||||
Self::ResponseSizeSoftLimit(limit) => size > *limit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Clone-friendly façade around a BAL store implementation.
|
||||
#[derive(Clone)]
|
||||
pub struct BalStoreHandle {
|
||||
@@ -120,28 +62,6 @@ impl BalStoreHandle {
|
||||
self.inner.get_by_hashes(block_hashes)
|
||||
}
|
||||
|
||||
/// Fetch BAL response entries for the given block hashes, stopping after the soft limit is
|
||||
/// exceeded.
|
||||
#[inline]
|
||||
pub fn get_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
) -> ProviderResult<Vec<Bytes>> {
|
||||
self.inner.get_by_hashes_with_limit(block_hashes, limit)
|
||||
}
|
||||
|
||||
/// Extends the given vector with BAL response entries for the given hashes.
|
||||
#[inline]
|
||||
pub fn append_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
out: &mut Vec<Bytes>,
|
||||
) -> ProviderResult<()> {
|
||||
self.inner.append_by_hashes_with_limit(block_hashes, limit, out)
|
||||
}
|
||||
|
||||
/// Fetch BALs for the requested range.
|
||||
#[inline]
|
||||
pub fn get_by_range(&self, start: BlockNumber, count: u64) -> ProviderResult<Vec<Bytes>> {
|
||||
@@ -186,25 +106,6 @@ impl BalStore for NoopBalStore {
|
||||
Ok(block_hashes.iter().map(|_| None).collect())
|
||||
}
|
||||
|
||||
fn append_by_hashes_with_limit(
|
||||
&self,
|
||||
block_hashes: &[BlockHash],
|
||||
limit: GetBlockAccessListLimit,
|
||||
out: &mut Vec<Bytes>,
|
||||
) -> ProviderResult<()> {
|
||||
let mut size = 0;
|
||||
for _ in block_hashes {
|
||||
let bal = Bytes::from_static(&[0xc0]);
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn get_by_range(&self, _start: BlockNumber, _count: u64) -> ProviderResult<Vec<Bytes>> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
@@ -226,27 +127,4 @@ mod tests {
|
||||
assert_eq!(by_hash, vec![None, None]);
|
||||
assert!(by_range.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn noop_store_limited_lookup_returns_prefix() {
|
||||
let store = BalStoreHandle::default();
|
||||
let hashes = [B256::random(), B256::random(), B256::random()];
|
||||
|
||||
let limited = store
|
||||
.get_by_hashes_with_limit(&hashes, GetBlockAccessListLimit::ResponseSizeSoftLimit(1))
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(limited, vec![Bytes::from_static(&[0xc0]), Bytes::from_static(&[0xc0])]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_access_list_limit() {
|
||||
let limit_none = GetBlockAccessListLimit::None;
|
||||
assert!(!limit_none.exceeds(usize::MAX));
|
||||
|
||||
let size_limit_2mb = GetBlockAccessListLimit::ResponseSizeSoftLimit(2 * 1024 * 1024);
|
||||
assert!(!size_limit_2mb.exceeds(1024 * 1024));
|
||||
assert!(!size_limit_2mb.exceeds(2 * 1024 * 1024));
|
||||
assert!(size_limit_2mb.exceeds(3 * 1024 * 1024));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use core::ops::Not;
|
||||
use crate::{
|
||||
added_removed_keys::MultiAddedRemovedKeys,
|
||||
prefix_set::{PrefixSetMut, TriePrefixSetsMut},
|
||||
utils::{extend_sorted_vec, kway_merge_disjoint_sorted, kway_merge_sorted},
|
||||
utils::{extend_sorted_vec, kway_merge_sorted},
|
||||
KeyHasher, MultiProofTargets, Nibbles,
|
||||
};
|
||||
use alloc::{borrow::Cow, vec::Vec};
|
||||
@@ -691,100 +691,6 @@ impl HashedPostStateSorted {
|
||||
Self { accounts, storages }
|
||||
}
|
||||
|
||||
/// Merges the batch and removes any overlapping keys present in the mask.
|
||||
///
|
||||
/// Account keys are masked at the top level, while storage entries are only masked at the slot
|
||||
/// level unless the mask wipes the entire storage. For duplicate keys in the batch, later
|
||||
/// items take precedence over earlier ones. The order of the mask does not matter.
|
||||
pub fn disjointed_merge_batch<'a>(batch: Vec<&'a Self>, mask: Vec<&'a Self>) -> Self {
|
||||
let accounts = kway_merge_disjoint_sorted(
|
||||
batch.iter().map(|item| item.accounts.len()).sum(),
|
||||
batch.iter().rev().map(|item| item.accounts.as_slice()),
|
||||
mask.iter().map(|item| item.accounts.as_slice()),
|
||||
);
|
||||
|
||||
struct StorageAcc<'a> {
|
||||
wiped: bool,
|
||||
sealed: bool,
|
||||
slot_count: usize,
|
||||
slices: Vec<&'a [(B256, U256)]>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StorageMaskAcc<'a> {
|
||||
wiped: bool,
|
||||
slices: Vec<&'a [(B256, U256)]>,
|
||||
}
|
||||
|
||||
let mut storages = B256Map::with_capacity_and_hasher(
|
||||
batch.iter().map(|item| item.storages.len()).sum(),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
for item in batch.iter().rev() {
|
||||
for (hashed_address, storage) in &item.storages {
|
||||
let entry = storages.entry(*hashed_address).or_insert_with(|| StorageAcc {
|
||||
wiped: false,
|
||||
sealed: false,
|
||||
slot_count: 0,
|
||||
slices: Vec::new(),
|
||||
});
|
||||
|
||||
if entry.sealed {
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.slices.push(storage.storage_slots.as_slice());
|
||||
entry.slot_count += storage.storage_slots.len();
|
||||
if storage.wiped {
|
||||
entry.wiped = true;
|
||||
entry.sealed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut storage_masks: B256Map<StorageMaskAcc<'a>> = B256Map::with_capacity_and_hasher(
|
||||
mask.iter().map(|item| item.storages.len()).sum(),
|
||||
Default::default(),
|
||||
);
|
||||
for item in mask {
|
||||
for (hashed_address, storage) in &item.storages {
|
||||
let entry = storage_masks.entry(*hashed_address).or_default();
|
||||
if entry.wiped {
|
||||
continue;
|
||||
}
|
||||
if storage.wiped {
|
||||
entry.wiped = true;
|
||||
entry.slices.clear();
|
||||
} else {
|
||||
entry.slices.push(storage.storage_slots.as_slice());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let storages = storages
|
||||
.into_iter()
|
||||
.filter_map(|(hashed_address, entry)| {
|
||||
let storage_slots = match storage_masks.get(&hashed_address) {
|
||||
Some(mask_entry) if mask_entry.wiped => return None,
|
||||
Some(mask_entry) => kway_merge_disjoint_sorted(
|
||||
entry.slot_count,
|
||||
entry.slices,
|
||||
mask_entry.slices.iter().copied(),
|
||||
),
|
||||
None => kway_merge_sorted(entry.slices),
|
||||
};
|
||||
|
||||
(!storage_slots.is_empty() || entry.wiped).then_some((
|
||||
hashed_address,
|
||||
HashedStorageSorted { wiped: entry.wiped, storage_slots },
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self { accounts, storages }
|
||||
}
|
||||
|
||||
/// Clears all accounts and storage data.
|
||||
pub fn clear(&mut self) {
|
||||
self.accounts.clear();
|
||||
@@ -1628,152 +1534,6 @@ mod tests {
|
||||
assert_eq!(state.accounts.get(&addr1), Some(&None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hashed_post_state_sorted_disjointed_merge_batch() {
|
||||
fn account(nonce: u64) -> Account {
|
||||
Account { nonce, balance: U256::ZERO, bytecode_hash: None }
|
||||
}
|
||||
|
||||
let kept_account = B256::with_last_byte(1);
|
||||
let removed_account = B256::with_last_byte(2);
|
||||
let kept_storage = B256::with_last_byte(3);
|
||||
let removed_storage = B256::with_last_byte(4);
|
||||
let slot1 = B256::with_last_byte(11);
|
||||
let slot2 = B256::with_last_byte(12);
|
||||
|
||||
let older = HashedPostStateSorted::new(
|
||||
vec![(kept_account, Some(account(1))), (removed_account, Some(account(10)))],
|
||||
B256Map::from_iter([
|
||||
(
|
||||
kept_storage,
|
||||
HashedStorageSorted {
|
||||
wiped: false,
|
||||
storage_slots: vec![(slot1, U256::from(1))],
|
||||
},
|
||||
),
|
||||
(
|
||||
removed_storage,
|
||||
HashedStorageSorted {
|
||||
wiped: false,
|
||||
storage_slots: vec![(slot1, U256::from(2))],
|
||||
},
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
let newer = HashedPostStateSorted::new(
|
||||
vec![(kept_account, Some(account(2)))],
|
||||
B256Map::from_iter([(
|
||||
kept_storage,
|
||||
HashedStorageSorted {
|
||||
wiped: false,
|
||||
storage_slots: vec![(slot1, U256::from(3)), (slot2, U256::from(4))],
|
||||
},
|
||||
)]),
|
||||
);
|
||||
|
||||
let remove_a = HashedPostStateSorted::new(
|
||||
vec![(removed_account, None)],
|
||||
B256Map::from_iter([
|
||||
(
|
||||
kept_storage,
|
||||
HashedStorageSorted { wiped: false, storage_slots: vec![(slot2, U256::ZERO)] },
|
||||
),
|
||||
(removed_storage, HashedStorageSorted { wiped: true, storage_slots: vec![] }),
|
||||
]),
|
||||
);
|
||||
|
||||
let remove_b = HashedPostStateSorted::new(
|
||||
vec![(B256::with_last_byte(255), Some(account(99)))],
|
||||
B256Map::default(),
|
||||
);
|
||||
|
||||
let result = HashedPostStateSorted::disjointed_merge_batch(
|
||||
vec![&older, &newer],
|
||||
vec![&remove_b, &remove_a],
|
||||
);
|
||||
|
||||
assert_eq!(result.accounts, vec![(kept_account, Some(account(2)))]);
|
||||
assert_eq!(result.storages.len(), 1);
|
||||
assert_eq!(
|
||||
result.storages.get(&kept_storage),
|
||||
Some(&HashedStorageSorted {
|
||||
wiped: false,
|
||||
storage_slots: vec![(slot1, U256::from(3))],
|
||||
})
|
||||
);
|
||||
assert!(!result.storages.contains_key(&removed_storage));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hashed_post_state_sorted_disjointed_merge_batch_removes_overlapping_batch_key() {
|
||||
fn account(nonce: u64) -> Account {
|
||||
Account { nonce, balance: U256::ZERO, bytecode_hash: None }
|
||||
}
|
||||
|
||||
let overlapping_account = B256::with_last_byte(21);
|
||||
let overlapping_storage = B256::with_last_byte(22);
|
||||
let slot = B256::with_last_byte(23);
|
||||
|
||||
let older = HashedPostStateSorted::new(
|
||||
vec![(overlapping_account, Some(account(1)))],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let newer = HashedPostStateSorted::new(
|
||||
vec![(overlapping_account, Some(account(2)))],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(2))] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let remove = HashedPostStateSorted::new(
|
||||
vec![(overlapping_account, None)],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
HashedStorageSorted { wiped: true, storage_slots: vec![] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let result =
|
||||
HashedPostStateSorted::disjointed_merge_batch(vec![&older, &newer], vec![&remove]);
|
||||
|
||||
assert!(result.accounts.is_empty());
|
||||
assert!(result.storages.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hashed_post_state_sorted_disjointed_merge_batch_ignores_empty_storage_mask() {
|
||||
let storage = B256::with_last_byte(31);
|
||||
let slot = B256::with_last_byte(32);
|
||||
|
||||
let batch = HashedPostStateSorted::new(
|
||||
vec![],
|
||||
B256Map::from_iter([(
|
||||
storage,
|
||||
HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] },
|
||||
)]),
|
||||
);
|
||||
let mask = HashedPostStateSorted::new(
|
||||
vec![],
|
||||
B256Map::from_iter([(
|
||||
storage,
|
||||
HashedStorageSorted { wiped: false, storage_slots: vec![] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let result = HashedPostStateSorted::disjointed_merge_batch(vec![&batch], vec![&mask]);
|
||||
|
||||
assert_eq!(
|
||||
result.storages.get(&storage),
|
||||
Some(&HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] })
|
||||
);
|
||||
}
|
||||
|
||||
/// Test non-wiped storage merges both zero and non-zero valued slots
|
||||
#[test]
|
||||
fn test_hashed_storage_extend_from_sorted_non_wiped() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::{
|
||||
utils::{extend_sorted_vec, kway_merge_disjoint_sorted, kway_merge_sorted},
|
||||
utils::{extend_sorted_vec, kway_merge_sorted},
|
||||
BranchNodeCompact, HashBuilder, Nibbles,
|
||||
};
|
||||
use alloc::{
|
||||
@@ -710,101 +710,6 @@ impl TrieUpdatesSorted {
|
||||
|
||||
Self { account_nodes, storage_tries }
|
||||
}
|
||||
|
||||
/// Merges the batch and removes any overlapping keys present in the mask.
|
||||
///
|
||||
/// Account trie nodes are masked at the top level, while storage trie entries are only masked
|
||||
/// at the node level unless the mask deletes the entire storage trie. For duplicate keys in
|
||||
/// the batch, later items take precedence over earlier ones. The order of the mask does not
|
||||
/// matter.
|
||||
pub fn disjointed_merge_batch<'a>(batch: Vec<&'a Self>, mask: Vec<&'a Self>) -> Self {
|
||||
let account_nodes = kway_merge_disjoint_sorted(
|
||||
batch.iter().map(|item| item.account_nodes.len()).sum(),
|
||||
batch.iter().rev().map(|item| item.account_nodes.as_slice()),
|
||||
mask.iter().map(|item| item.account_nodes.as_slice()),
|
||||
);
|
||||
|
||||
struct StorageAcc<'a> {
|
||||
is_deleted: bool,
|
||||
sealed: bool,
|
||||
node_count: usize,
|
||||
slices: Vec<&'a [(Nibbles, Option<BranchNodeCompact>)]>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StorageMaskAcc<'a> {
|
||||
is_deleted: bool,
|
||||
slices: Vec<&'a [(Nibbles, Option<BranchNodeCompact>)]>,
|
||||
}
|
||||
|
||||
let mut storage_tries = B256Map::with_capacity_and_hasher(
|
||||
batch.iter().map(|item| item.storage_tries.len()).sum(),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
for item in batch.iter().rev() {
|
||||
for (hashed_address, storage_trie) in &item.storage_tries {
|
||||
let entry = storage_tries.entry(*hashed_address).or_insert_with(|| StorageAcc {
|
||||
is_deleted: false,
|
||||
sealed: false,
|
||||
node_count: 0,
|
||||
slices: Vec::new(),
|
||||
});
|
||||
|
||||
if entry.sealed {
|
||||
continue;
|
||||
}
|
||||
|
||||
entry.slices.push(storage_trie.storage_nodes.as_slice());
|
||||
entry.node_count += storage_trie.storage_nodes.len();
|
||||
if storage_trie.is_deleted {
|
||||
entry.is_deleted = true;
|
||||
entry.sealed = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut storage_masks: B256Map<StorageMaskAcc<'a>> = B256Map::with_capacity_and_hasher(
|
||||
mask.iter().map(|item| item.storage_tries.len()).sum(),
|
||||
Default::default(),
|
||||
);
|
||||
for item in mask {
|
||||
for (hashed_address, storage_trie) in &item.storage_tries {
|
||||
let entry = storage_masks.entry(*hashed_address).or_default();
|
||||
if entry.is_deleted {
|
||||
continue;
|
||||
}
|
||||
if storage_trie.is_deleted {
|
||||
entry.is_deleted = true;
|
||||
entry.slices.clear();
|
||||
} else {
|
||||
entry.slices.push(storage_trie.storage_nodes.as_slice());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let storage_tries = storage_tries
|
||||
.into_iter()
|
||||
.filter_map(|(hashed_address, entry)| {
|
||||
let storage_nodes = match storage_masks.get(&hashed_address) {
|
||||
Some(mask_entry) if mask_entry.is_deleted => return None,
|
||||
Some(mask_entry) => kway_merge_disjoint_sorted(
|
||||
entry.node_count,
|
||||
entry.slices,
|
||||
mask_entry.slices.iter().copied(),
|
||||
),
|
||||
None => kway_merge_sorted(entry.slices),
|
||||
};
|
||||
|
||||
(!storage_nodes.is_empty() || entry.is_deleted).then_some((
|
||||
hashed_address,
|
||||
StorageTrieUpdatesSorted { is_deleted: entry.is_deleted, storage_nodes },
|
||||
))
|
||||
})
|
||||
.collect();
|
||||
|
||||
Self::new(account_nodes, storage_tries)
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<Self> for TrieUpdatesSorted {
|
||||
@@ -1072,158 +977,6 @@ mod tests {
|
||||
assert_eq!(storage3.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x07]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trie_updates_sorted_disjointed_merge_batch() {
|
||||
let kept_node = Nibbles::from_nibbles_unchecked([0x01]);
|
||||
let removed_node = Nibbles::from_nibbles_unchecked([0x02]);
|
||||
let kept_storage = B256::from([3; 32]);
|
||||
let removed_storage = B256::from([4; 32]);
|
||||
let slot1 = Nibbles::from_nibbles_unchecked([0x0a]);
|
||||
let slot2 = Nibbles::from_nibbles_unchecked([0x0b]);
|
||||
|
||||
let older = TrieUpdatesSorted::new(
|
||||
vec![(kept_node, Some(BranchNodeCompact::default())), (removed_node, None)],
|
||||
B256Map::from_iter([
|
||||
(
|
||||
kept_storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot1, None)],
|
||||
},
|
||||
),
|
||||
(
|
||||
removed_storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot1, Some(BranchNodeCompact::default()))],
|
||||
},
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
let newer = TrieUpdatesSorted::new(
|
||||
vec![(kept_node, None)],
|
||||
B256Map::from_iter([(
|
||||
kept_storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot1, Some(BranchNodeCompact::default())), (slot2, None)],
|
||||
},
|
||||
)]),
|
||||
);
|
||||
|
||||
let remove_a = TrieUpdatesSorted::new(
|
||||
vec![(removed_node, Some(BranchNodeCompact::default()))],
|
||||
B256Map::from_iter([
|
||||
(
|
||||
kept_storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot2, Some(BranchNodeCompact::default()))],
|
||||
},
|
||||
),
|
||||
(
|
||||
removed_storage,
|
||||
StorageTrieUpdatesSorted { is_deleted: true, storage_nodes: vec![] },
|
||||
),
|
||||
]),
|
||||
);
|
||||
|
||||
let remove_b = TrieUpdatesSorted::new(
|
||||
vec![(Nibbles::from_nibbles_unchecked([0x0f]), Some(BranchNodeCompact::default()))],
|
||||
B256Map::default(),
|
||||
);
|
||||
|
||||
let result = TrieUpdatesSorted::disjointed_merge_batch(
|
||||
vec![&older, &newer],
|
||||
vec![&remove_b, &remove_a],
|
||||
);
|
||||
|
||||
assert_eq!(result.account_nodes, vec![(kept_node, None)]);
|
||||
assert_eq!(result.storage_tries.len(), 1);
|
||||
assert_eq!(
|
||||
result.storage_tries.get(&kept_storage),
|
||||
Some(&StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot1, Some(BranchNodeCompact::default()))],
|
||||
})
|
||||
);
|
||||
assert!(!result.storage_tries.contains_key(&removed_storage));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trie_updates_sorted_disjointed_merge_batch_removes_overlapping_batch_key() {
|
||||
let overlapping_node = Nibbles::from_nibbles_unchecked([0x03]);
|
||||
let overlapping_storage = B256::from([5; 32]);
|
||||
let slot = Nibbles::from_nibbles_unchecked([0x0c]);
|
||||
|
||||
let older = TrieUpdatesSorted::new(
|
||||
vec![(overlapping_node, Some(BranchNodeCompact::default()))],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))],
|
||||
},
|
||||
)]),
|
||||
);
|
||||
|
||||
let newer = TrieUpdatesSorted::new(
|
||||
vec![(overlapping_node, None)],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![(slot, None)] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let remove = TrieUpdatesSorted::new(
|
||||
vec![(overlapping_node, Some(BranchNodeCompact::default()))],
|
||||
B256Map::from_iter([(
|
||||
overlapping_storage,
|
||||
StorageTrieUpdatesSorted { is_deleted: true, storage_nodes: vec![] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let result = TrieUpdatesSorted::disjointed_merge_batch(vec![&older, &newer], vec![&remove]);
|
||||
|
||||
assert!(result.account_nodes.is_empty());
|
||||
assert!(result.storage_tries.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_trie_updates_sorted_disjointed_merge_batch_ignores_empty_storage_mask() {
|
||||
let storage = B256::from([6; 32]);
|
||||
let slot = Nibbles::from_nibbles_unchecked([0x0d]);
|
||||
|
||||
let batch = TrieUpdatesSorted::new(
|
||||
vec![],
|
||||
B256Map::from_iter([(
|
||||
storage,
|
||||
StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))],
|
||||
},
|
||||
)]),
|
||||
);
|
||||
let mask = TrieUpdatesSorted::new(
|
||||
vec![],
|
||||
B256Map::from_iter([(
|
||||
storage,
|
||||
StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![] },
|
||||
)]),
|
||||
);
|
||||
|
||||
let result = TrieUpdatesSorted::disjointed_merge_batch(vec![&batch], vec![&mask]);
|
||||
|
||||
assert_eq!(
|
||||
result.storage_tries.get(&storage),
|
||||
Some(&StorageTrieUpdatesSorted {
|
||||
is_deleted: false,
|
||||
storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))],
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
/// Test extending with storage tries adds both nodes and removed nodes correctly
|
||||
#[test]
|
||||
fn test_trie_updates_extend_from_sorted_with_storage_tries() {
|
||||
|
||||
@@ -26,51 +26,6 @@ where
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Merge sorted left slices into a sorted `Vec`, excluding keys present in any right slice.
|
||||
///
|
||||
/// Callers pass left slices in priority order (index 0 = highest priority), so the first
|
||||
/// left slice's value for a key takes precedence over later slices. Right slice order is ignored;
|
||||
/// the right-hand side only contributes keys to exclude.
|
||||
pub(crate) fn kway_merge_disjoint_sorted<'a, K, V>(
|
||||
capacity: usize,
|
||||
left_slices: impl IntoIterator<Item = &'a [(K, V)]>,
|
||||
right_slices: impl IntoIterator<Item = &'a [(K, V)]>,
|
||||
) -> Vec<(K, V)>
|
||||
where
|
||||
K: Ord + Clone + 'a,
|
||||
V: Clone + 'a,
|
||||
{
|
||||
let mut right_keys = right_slices
|
||||
.into_iter()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.iter().map(|(k, _)| k))
|
||||
.kmerge()
|
||||
.dedup()
|
||||
.peekable();
|
||||
|
||||
let mut out = Vec::with_capacity(capacity);
|
||||
for (_, key, value) in left_slices
|
||||
.into_iter()
|
||||
.filter(|s| !s.is_empty())
|
||||
.enumerate()
|
||||
.map(|(i, s)| s.iter().map(move |(k, v)| (i, k, v)))
|
||||
.kmerge_by(|(i1, k1, _), (i2, k2, _)| (k1, i1) < (k2, i2))
|
||||
.dedup_by(|(_, k1, _), (_, k2, _)| *k1 == *k2)
|
||||
{
|
||||
while right_keys.peek().is_some_and(|right_key| *right_key < key) {
|
||||
right_keys.next();
|
||||
}
|
||||
|
||||
if right_keys.peek().is_some_and(|right_key| *right_key == key) {
|
||||
continue;
|
||||
}
|
||||
|
||||
out.push((key.clone(), value.clone()));
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
/// Extend a sorted vector with another sorted vector using 2 pointer merge.
|
||||
/// Values from `other` take precedence for duplicate keys.
|
||||
pub(crate) fn extend_sorted_vec<K, V>(target: &mut Vec<(K, V)>, other: &[(K, V)])
|
||||
@@ -228,20 +183,4 @@ mod tests {
|
||||
let result: Vec<(i32, &str)> = kway_merge_sorted(Vec::<&[(i32, &str)]>::new());
|
||||
assert!(result.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_kway_merge_disjoint_sorted() {
|
||||
let left_old = vec![(1, "old"), (2, "drop"), (4, "keep")];
|
||||
let left_new = vec![(1, "new"), (3, "new_only")];
|
||||
let right_a = vec![(2, "ignored"), (5, "ignored")];
|
||||
let right_b = vec![(3, "ignored")];
|
||||
|
||||
let result = kway_merge_disjoint_sorted(
|
||||
left_old.len() + left_new.len(),
|
||||
[left_new.as_slice(), left_old.as_slice()],
|
||||
[right_a.as_slice(), right_b.as_slice()],
|
||||
);
|
||||
|
||||
assert_eq!(result, vec![(1, "new"), (4, "keep")]);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,8 +50,6 @@ rand = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
reth-chainspec.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-provider = { workspace = true, features = ["test-utils"] }
|
||||
reth-trie-db.workspace = true
|
||||
@@ -77,7 +75,5 @@ test-utils = [
|
||||
"reth-trie-db/test-utils",
|
||||
"reth-trie/test-utils",
|
||||
"reth-tasks/test-utils",
|
||||
"reth-chainspec/test-utils",
|
||||
"reth-trie-sparse?/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
|
||||
@@ -1151,9 +1151,7 @@ enum AccountWorkerJob {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_provider::test_utils::create_test_provider_factory_with_chain_spec;
|
||||
use std::sync::Arc;
|
||||
use reth_provider::test_utils::create_test_provider_factory;
|
||||
|
||||
fn test_ctx<Factory>(factory: Factory) -> ProofTaskCtx<Factory> {
|
||||
ProofTaskCtx::new(factory)
|
||||
@@ -1162,16 +1160,11 @@ mod tests {
|
||||
/// Ensures `ProofWorkerHandle::new` spawns workers correctly.
|
||||
#[test]
|
||||
fn spawn_proof_workers_creates_handle() {
|
||||
let chain_spec = Arc::new(ChainSpec::default());
|
||||
let anchor_hash = chain_spec.genesis_hash();
|
||||
let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec);
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let changeset_cache = reth_trie_db::ChangesetCache::new();
|
||||
let factory = reth_provider::providers::OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
reth_provider::providers::OverlayBuilder::<reth_ethereum_primitives::EthPrimitives>::new(
|
||||
anchor_hash,
|
||||
changeset_cache,
|
||||
),
|
||||
changeset_cache,
|
||||
);
|
||||
let ctx = test_ctx(factory);
|
||||
|
||||
|
||||
@@ -274,28 +274,18 @@ mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{keccak256, Address, U256};
|
||||
use rand::Rng;
|
||||
use reth_chainspec::{ChainSpec, EthChainSpec};
|
||||
use reth_ethereum_primitives::{Block, BlockBody};
|
||||
use reth_primitives_traits::{Account, RecoveredBlock, SealedBlock, StorageEntry};
|
||||
use reth_provider::{
|
||||
test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, ExecutionOutcome,
|
||||
HashingWriter,
|
||||
};
|
||||
use reth_primitives_traits::{Account, StorageEntry};
|
||||
use reth_provider::{test_utils::create_test_provider_factory, HashingWriter};
|
||||
use reth_trie::{test_utils, HashedPostState, HashedStorage};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn random_parallel_root() {
|
||||
let chain_spec = Arc::new(ChainSpec::default());
|
||||
let anchor_hash = chain_spec.genesis_hash();
|
||||
let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
|
||||
let factory = create_test_provider_factory();
|
||||
let changeset_cache = reth_trie_db::ChangesetCache::new();
|
||||
let overlay_builder = reth_provider::providers::OverlayBuilder::<
|
||||
reth_ethereum_primitives::EthPrimitives,
|
||||
>::new(anchor_hash, changeset_cache);
|
||||
let mut overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new(
|
||||
factory.clone(),
|
||||
overlay_builder.clone(),
|
||||
changeset_cache,
|
||||
);
|
||||
|
||||
let mut rng = rand::rng();
|
||||
@@ -320,20 +310,6 @@ mod tests {
|
||||
|
||||
{
|
||||
let provider_rw = factory.provider_rw().unwrap();
|
||||
let genesis_block = RecoveredBlock::new_sealed(
|
||||
SealedBlock::<Block>::seal_parts(
|
||||
chain_spec.genesis_header().clone(),
|
||||
BlockBody::default(),
|
||||
),
|
||||
vec![],
|
||||
);
|
||||
provider_rw
|
||||
.append_blocks_with_state(
|
||||
vec![genesis_block],
|
||||
&ExecutionOutcome::default(),
|
||||
Default::default(),
|
||||
)
|
||||
.unwrap();
|
||||
provider_rw
|
||||
.insert_account_for_hashing(
|
||||
state.iter().map(|(address, (account, _))| (*address, Some(*account))),
|
||||
@@ -386,10 +362,8 @@ mod tests {
|
||||
}
|
||||
|
||||
let prefix_sets = hashed_state.construct_prefix_sets();
|
||||
overlay_factory = reth_provider::providers::OverlayStateProviderFactory::new(
|
||||
factory,
|
||||
overlay_builder.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted()))),
|
||||
);
|
||||
overlay_factory =
|
||||
overlay_factory.with_hashed_state_overlay(Some(Arc::new(hashed_state.into_sorted())));
|
||||
|
||||
assert_eq!(
|
||||
ParallelStateRoot::new(overlay_factory, prefix_sets.freeze(), runtime)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user