mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
262 Commits
staging
...
emma/fix-c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f04db57d3a | ||
|
|
09adb83922 | ||
|
|
c12b6d4c90 | ||
|
|
7a78044587 | ||
|
|
f88538e033 | ||
|
|
63dff64b8a | ||
|
|
233590cefd | ||
|
|
40962ef6fc | ||
|
|
2f121b099b | ||
|
|
0470050c05 | ||
|
|
cbc416b82a | ||
|
|
3fddefbd38 | ||
|
|
f97a6530c1 | ||
|
|
80e3e1c79d | ||
|
|
ee37c25a4b | ||
|
|
c01f9688e2 | ||
|
|
815a75833e | ||
|
|
59c4e24296 | ||
|
|
d5b5caa439 | ||
|
|
47f1999654 | ||
|
|
3ac5637bd1 | ||
|
|
4cec99ed13 | ||
|
|
2f73835483 | ||
|
|
ed20a40649 | ||
|
|
080a9cfc10 | ||
|
|
c4cd5c9b7b | ||
|
|
ce2a194fb7 | ||
|
|
6dcab51c97 | ||
|
|
4db23809cc | ||
|
|
f84d5e6d7f | ||
|
|
e63b6239d7 | ||
|
|
660a0dee90 | ||
|
|
f92c9b4370 | ||
|
|
f0e2522294 | ||
|
|
7103088adc | ||
|
|
663765af5c | ||
|
|
20cfb2d517 | ||
|
|
0bdf6e2f2e | ||
|
|
85abd41824 | ||
|
|
70fb03a530 | ||
|
|
96fce4dc4f | ||
|
|
728c7acd08 | ||
|
|
626c82db33 | ||
|
|
624fcbd345 | ||
|
|
aed47bc3f8 | ||
|
|
7680c1e4f6 | ||
|
|
93cb4068d2 | ||
|
|
2fba05dc67 | ||
|
|
ea143d4d31 | ||
|
|
fddb7dad10 | ||
|
|
af6d674cac | ||
|
|
de5688a76e | ||
|
|
d4cb91f0a5 | ||
|
|
d122c7b49c | ||
|
|
aed9014e1e | ||
|
|
d340114d52 | ||
|
|
7fc22f7b5b | ||
|
|
c8c5f8886d | ||
|
|
2f3c8d7d03 | ||
|
|
a90f8be67b | ||
|
|
7faca05344 | ||
|
|
2827b0aca0 | ||
|
|
d3bb2faf28 | ||
|
|
ef292ffa00 | ||
|
|
ea98d37bb3 | ||
|
|
f2b3201187 | ||
|
|
d1cbf6ca5a | ||
|
|
56bb47709c | ||
|
|
3703255d5d | ||
|
|
b431caf806 | ||
|
|
21dadb71c3 | ||
|
|
98c45a4245 | ||
|
|
ac2cc7b4e2 | ||
|
|
3931affcf2 | ||
|
|
93b7ae9286 | ||
|
|
7e7717bdaa | ||
|
|
815037e27d | ||
|
|
80bf5532ac | ||
|
|
028e99191a | ||
|
|
dc35fc8251 | ||
|
|
285c325d71 | ||
|
|
ca47a7e9f9 | ||
|
|
6d718d0c21 | ||
|
|
949111c953 | ||
|
|
742eb56949 | ||
|
|
4af4836ec1 | ||
|
|
3bc71e7ec0 | ||
|
|
03fbb6cafe | ||
|
|
b09b097a0b | ||
|
|
0fffdcdd23 | ||
|
|
bc33eb764a | ||
|
|
190157636e | ||
|
|
8e3bc6567c | ||
|
|
45b961c7b3 | ||
|
|
94818d7676 | ||
|
|
4c2a9a9b4a | ||
|
|
76c37f0f80 | ||
|
|
0275ff35fd | ||
|
|
3f011c8328 | ||
|
|
beac28dbb2 | ||
|
|
bce100c6c8 | ||
|
|
40e99a4a4f | ||
|
|
1ff88e43cd | ||
|
|
d23c244cd1 | ||
|
|
3de9259026 | ||
|
|
d24f0b1e05 | ||
|
|
bb1b9ec611 | ||
|
|
70cab0d163 | ||
|
|
e530b1f6a1 | ||
|
|
ff5d375526 | ||
|
|
d1a92afb57 | ||
|
|
0517c12c90 | ||
|
|
237eb1675c | ||
|
|
b6bcd7e6bd | ||
|
|
48122300d7 | ||
|
|
13f214f160 | ||
|
|
f17592670d | ||
|
|
c225132b81 | ||
|
|
dcc5d9ec30 | ||
|
|
6cd56b645b | ||
|
|
794dbff26e | ||
|
|
fcfbed0bbc | ||
|
|
70bcd475fe | ||
|
|
cd6e895a97 | ||
|
|
6552a3a9ab | ||
|
|
6a91089542 | ||
|
|
a9a1e504b4 | ||
|
|
e280f25885 | ||
|
|
37c4f908fa | ||
|
|
a157be3f3b | ||
|
|
e0eb306b2b | ||
|
|
7f4f3f1eb9 | ||
|
|
8970f82aaf | ||
|
|
8529da976f | ||
|
|
8fa539225b | ||
|
|
93d546a36d | ||
|
|
5c83eb0b06 | ||
|
|
cd32e3cc05 | ||
|
|
26470cadfc | ||
|
|
506ab806e4 | ||
|
|
c2e846093e | ||
|
|
5df22b12d8 | ||
|
|
ff9700bb3b | ||
|
|
85d35fa6c0 | ||
|
|
47544d9a7e | ||
|
|
ef33961aff | ||
|
|
0e01a694a7 | ||
|
|
ee19320ee8 | ||
|
|
9251997c1f | ||
|
|
302993b45a | ||
|
|
8d97ab63c6 | ||
|
|
251f83ab0b | ||
|
|
e6e0dde903 | ||
|
|
b1b51261af | ||
|
|
2ae5ef475e | ||
|
|
8861e2724f | ||
|
|
734ec4ffe6 | ||
|
|
cbcdf8dac0 | ||
|
|
826e387c87 | ||
|
|
1c40188993 | ||
|
|
49a2df0d7a | ||
|
|
a1d1b6def6 | ||
|
|
56bbb3ce2c | ||
|
|
5b1010322c | ||
|
|
a195b777eb | ||
|
|
5045e6ef8b | ||
|
|
b49cadb346 | ||
|
|
aeb2c6e731 | ||
|
|
477fed7a11 | ||
|
|
59993b974a | ||
|
|
9ecef47aff | ||
|
|
0ba685386d | ||
|
|
6ff4f947c8 | ||
|
|
719bbc2543 | ||
|
|
a9a6044bc5 | ||
|
|
6f9a3242ef | ||
|
|
e89bf483bc | ||
|
|
61038449c8 | ||
|
|
48b2cd970f | ||
|
|
fb90051010 | ||
|
|
a0a622a155 | ||
|
|
8db352dfd2 | ||
|
|
117b212e2e | ||
|
|
df9e3669aa | ||
|
|
0464cddfb0 | ||
|
|
e21a174737 | ||
|
|
e972d9d8c7 | ||
|
|
7f00ebfafe | ||
|
|
883e9ae8cc | ||
|
|
a1e4132c2d | ||
|
|
4ecb0d5680 | ||
|
|
5b8808e5fd | ||
|
|
2eec519bf9 | ||
|
|
02513ecf3b | ||
|
|
10c6bdb5ff | ||
|
|
20ae9ac405 | ||
|
|
881500e592 | ||
|
|
8db125daff | ||
|
|
bf2071f773 | ||
|
|
ee5ec069cd | ||
|
|
8722277d6e | ||
|
|
57148eac9f | ||
|
|
74abad29ad | ||
|
|
997af404a5 | ||
|
|
314a92e93c | ||
|
|
f0c4be108b | ||
|
|
9265e8e46c | ||
|
|
7594e1513a | ||
|
|
7f5acc2723 | ||
|
|
60d0430c2b | ||
|
|
d49f828998 | ||
|
|
2f78bcd7b5 | ||
|
|
f60febfa62 | ||
|
|
317f858bd4 | ||
|
|
11acd97982 | ||
|
|
f5cf90227b | ||
|
|
0dd47af250 | ||
|
|
0142769191 | ||
|
|
e1dc93e24f | ||
|
|
33ac869a85 | ||
|
|
ec982f8686 | ||
|
|
47cef33a0d | ||
|
|
9529de4cf2 | ||
|
|
5a9dd02301 | ||
|
|
d71a0c0c7b | ||
|
|
2be3788481 | ||
|
|
adbec3218d | ||
|
|
2e5560b444 | ||
|
|
1f3fd5da2e | ||
|
|
3ab7cb98aa | ||
|
|
d3088e171c | ||
|
|
2c443a3dcb | ||
|
|
4b444069a5 | ||
|
|
25d371817a | ||
|
|
4b0fa8a330 | ||
|
|
df22d38224 | ||
|
|
e4ec836a46 | ||
|
|
d3c42fc718 | ||
|
|
8171cee927 | ||
|
|
61cfcd8195 | ||
|
|
b646f4559c | ||
|
|
564ffa5868 | ||
|
|
12891dd171 | ||
|
|
c1015022f5 | ||
|
|
e3fe6326bc | ||
|
|
e3d520b24f | ||
|
|
9f29939ea1 | ||
|
|
10881d1c73 | ||
|
|
408593467b | ||
|
|
8caf8cdf11 | ||
|
|
1e8030ef28 | ||
|
|
f72c503d6f | ||
|
|
42890e6e7f | ||
|
|
e30e441ada | ||
|
|
121160d248 | ||
|
|
7ff78ca082 | ||
|
|
d7f56d509c | ||
|
|
3300e404cf | ||
|
|
77cb99fc78 | ||
|
|
66169c7e7c | ||
|
|
4f5fafc8f3 | ||
|
|
0b8e6c6ed3 |
10
.changelog/brave-dogs-fly.md
Normal file
10
.changelog/brave-dogs-fly.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
reth-engine-primitives: patch
|
||||
reth-engine-tree: patch
|
||||
reth-node-core: patch
|
||||
reth-trie-parallel: minor
|
||||
---
|
||||
|
||||
Removed legacy proof calculation system and V2-specific configuration flags.
|
||||
|
||||
Removed the legacy (non-V2) proof calculation code paths, simplified multiproof task architecture by removing the dual-mode system, and cleaned up V2-specific CLI flags (`--engine.disable-proof-v2`, `--engine.disable-trie-cache`) that are no longer needed. The codebase now exclusively uses V2 proofs with the sparse trie cache.
|
||||
5
.changelog/calm-clams-buzz.md
Normal file
5
.changelog/calm-clams-buzz.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Refactored sparse trie node state tracking to use RLP nodes instead of hashes. Replaced `Option<B256>` hash fields with `SparseNodeState` enum that tracks either dirty nodes or cached RLP nodes with optional database storage flags. Added debug assertions to validate leaf path lengths and improved pruning logic to use node paths directly instead of path-hash tuples.
|
||||
5
.changelog/cool-suns-rest.md
Normal file
5
.changelog/cool-suns-rest.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-transaction-pool: minor
|
||||
---
|
||||
|
||||
Added support for optional custom stateless and stateful validation hooks in `EthTransactionValidator` via `set_additional_stateless_validation` and `set_additional_stateful_validation` methods. Also implemented a manual `Debug` impl to handle the non-`Debug` function pointer fields.
|
||||
5
.changelog/dull-koalas-play.md
Normal file
5
.changelog/dull-koalas-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Added recording of `SetRoot` operation in `ParallelSparseTrie::set_root` when the `trie-debug` feature is enabled.
|
||||
6
.changelog/evil-fish-smile.md
Normal file
6
.changelog/evil-fish-smile.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth-rpc-convert: minor
|
||||
reth-storage-rpc-provider: minor
|
||||
---
|
||||
|
||||
Replaced the separate `TryFromBlockResponse`, `TryFromReceiptResponse`, and `TryFromTransactionResponse` traits with a unified `RpcResponseConverter` trait and default `EthRpcConverter` implementation. Removed the `op-alloy-network` dependency and refactored `RpcBlockchainProvider` to store a dynamic converter instance instead of relying on per-type trait bounds.
|
||||
5
.changelog/fast-seals-play.md
Normal file
5
.changelog/fast-seals-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-transaction-pool: minor
|
||||
---
|
||||
|
||||
Added `consensus_ref` method to `PoolTransaction` trait for borrowing consensus transactions without cloning.
|
||||
4
.changelog/fast-waves-smile.md
Normal file
4
.changelog/fast-waves-smile.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved nightly Docker build failure Slack notification with more detailed formatting and context.
|
||||
5
.changelog/fix-simulate-revert-code.md
Normal file
5
.changelog/fix-simulate-revert-code.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-rpc-eth-types: patch
|
||||
---
|
||||
|
||||
Updated `eth_simulateV1` revert error code from `-32000` to `3` to be consistent with `eth_call`, per [execution-apis#748](https://github.com/ethereum/execution-apis/pull/748).
|
||||
5
.changelog/keen-geese-bake.md
Normal file
5
.changelog/keen-geese-bake.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Added sub-phase timing histograms to the sparse trie event loop, tracking channel wait, proof coalescing, multiproof reveal, and trie update durations separately.
|
||||
4
.changelog/lively-clouds-bake.md
Normal file
4
.changelog/lively-clouds-bake.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved documentation overview page with better structure and clarity.
|
||||
5
.changelog/lively-foxes-play.md
Normal file
5
.changelog/lively-foxes-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-node-events: patch
|
||||
---
|
||||
|
||||
Updated consensus engine log message to be more accurate about received updates.
|
||||
6
.changelog/merry-koalas-nod.md
Normal file
6
.changelog/merry-koalas-nod.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth-rpc-eth-api: minor
|
||||
reth-rpc-server-types: minor
|
||||
---
|
||||
|
||||
Added `eth_getStorageValues` RPC method for batch storage slot retrieval across multiple addresses.
|
||||
9
.changelog/nice-trees-drink.md
Normal file
9
.changelog/nice-trees-drink.md
Normal file
@@ -0,0 +1,9 @@
|
||||
---
|
||||
reth-network-api: minor
|
||||
reth-network-types: minor
|
||||
reth-network: minor
|
||||
reth-node-core: minor
|
||||
reth: minor
|
||||
---
|
||||
|
||||
Added optional ENR fork ID enforcement to filter out peers from incompatible networks during peer discovery, controlled by the `--enforce-enr-fork-id` CLI flag.
|
||||
5
.changelog/odd-donkeys-chirp.md
Normal file
5
.changelog/odd-donkeys-chirp.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Fixed `compare_trie_updates` to return `bool` indicating whether differences were found, and updated the caller to properly use the return value instead of treating all successful comparisons as having no differences.
|
||||
5
.changelog/odd-snails-play.md
Normal file
5
.changelog/odd-snails-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-node-core: minor
|
||||
---
|
||||
|
||||
Added `with_dev_block_time` helper method to `NodeConfig` for configuring dev miner block production interval.
|
||||
5
.changelog/proud-crabs-bark.md
Normal file
5
.changelog/proud-crabs-bark.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-db-api: patch
|
||||
---
|
||||
|
||||
Changed `StoredNibblesSubKey` encoding to use a stack-allocated `[u8; 65]` array instead of a heap-allocated `Vec<u8>`, avoiding unnecessary heap allocation.
|
||||
5
.changelog/proud-wolves-spin.md
Normal file
5
.changelog/proud-wolves-spin.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-storage-api: patch
|
||||
---
|
||||
|
||||
Added `Arc` to `auto_impl` derive for storage-api traits to support automatic `Arc` wrapper implementations.
|
||||
8
.changelog/quick-mice-jump.md
Normal file
8
.changelog/quick-mice-jump.md
Normal file
@@ -0,0 +1,8 @@
|
||||
---
|
||||
reth: patch
|
||||
reth-engine-tree: patch
|
||||
reth-node-builder: patch
|
||||
reth-trie-sparse: minor
|
||||
---
|
||||
|
||||
Added `trie-debug` feature for recording sparse trie mutations to aid in debugging state root mismatches.
|
||||
5
.changelog/rich-lakes-bark.md
Normal file
5
.changelog/rich-lakes-bark.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-provider: patch
|
||||
---
|
||||
|
||||
Fixed sender pruning during block reorg to skip when sender_recovery is fully pruned, preventing a fatal crash when no sender data exists in static files.
|
||||
7
.changelog/safe-waves-read.md
Normal file
7
.changelog/safe-waves-read.md
Normal file
@@ -0,0 +1,7 @@
|
||||
---
|
||||
reth-network-types: minor
|
||||
reth-network: minor
|
||||
reth-node-core: patch
|
||||
---
|
||||
|
||||
Added `PersistedPeerInfo` struct to persist richer peer metadata (kind, fork ID, reputation) to disk. Updated `PeersConfig::with_basic_nodes_from_file` to support both the new `PersistedPeerInfo` format and the legacy `Vec<NodeRecord>` format with automatic conversion, and updated `write_peers_to_file` to exclude backed-off and banned peers.
|
||||
5
.changelog/tall-stars-shout.md
Normal file
5
.changelog/tall-stars-shout.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-network: minor
|
||||
---
|
||||
|
||||
Added `fork_id` as a tiebreaker in peer selection when reputations are equal, preferring peers with a discovered `fork_id` as it indicates fork compatibility. Added a test to verify the tiebreaker behavior.
|
||||
5
.changelog/tidy-stars-cry.md
Normal file
5
.changelog/tidy-stars-cry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Fixed a bug where trie nodes could appear in both `updated_nodes` and `removed_nodes` simultaneously by removing entries from `removed_nodes` when a node is inserted as updated.
|
||||
6
.changelog/warm-donkeys-dry.md
Normal file
6
.changelog/warm-donkeys-dry.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth-engine-local: minor
|
||||
reth-node-builder: minor
|
||||
---
|
||||
|
||||
Added trigger-based `MiningMode` variant that allows blocks to be built on-demand via custom streams, and exposed `with_mining_mode` method on `DebugNodeLauncherFuture` to override default mining configuration.
|
||||
5
.changelog/warm-pandas-cook.md
Normal file
5
.changelog/warm-pandas-cook.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-transaction-pool: patch
|
||||
---
|
||||
|
||||
Fixed a bug where transactions from the same sender were added to the pending subpool out of nonce order. Ensured `process_updates` runs before `add_new_transaction` so that lower-nonce promotions are enqueued before the newly inserted higher-nonce transaction, preserving correct ordering for live `BestTransactions` iterators.
|
||||
@@ -1,6 +1,6 @@
|
||||
[profile.default]
|
||||
retries = { backoff = "exponential", count = 2, delay = "2s", jitter = true }
|
||||
slow-timeout = { period = "30s", terminate-after = 4 }
|
||||
slow-timeout = { period = "30s", terminate-after = 2 }
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = "test(general_state_tests)"
|
||||
|
||||
4
.github/CODEOWNERS
vendored
4
.github/CODEOWNERS
vendored
@@ -1,7 +1,7 @@
|
||||
* @gakonst
|
||||
crates/chain-state/ @fgimenez @mattsse
|
||||
crates/chainspec/ @Rjected @joshieDo @mattsse
|
||||
crates/cli/ @mattsse
|
||||
crates/cli/ @mattsse @Rjected
|
||||
crates/config/ @shekhirin @mattsse @Rjected
|
||||
crates/consensus/ @mattsse @Rjected
|
||||
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
|
||||
@@ -38,7 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tasks/ @mattsse @DaniPopes
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
|
||||
99
.github/scripts/bench-reth-build.sh
vendored
Executable file
99
.github/scripts/bench-reth-build.sh
vendored
Executable file
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds (or fetches from cache) reth binaries for benchmarking.
|
||||
#
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
|
||||
#
|
||||
# baseline — build/fetch the baseline binary at <commit> (merge-base)
|
||||
# source-dir must be checked out at <commit>
|
||||
# feature — build/fetch the candidate binary + reth-bench at <commit>
|
||||
# source-dir must be checked out at <commit>
|
||||
# optional branch-sha is the PR head commit for cache key
|
||||
#
|
||||
# Outputs:
|
||||
# baseline: <source-dir>/target/profiling/reth
|
||||
# feature: <source-dir>/target/profiling/reth, reth-bench installed to cargo bin
|
||||
#
|
||||
# Required: mc (MinIO client) with a configured alias
|
||||
set -euo pipefail
|
||||
|
||||
MC="mc"
|
||||
MODE="$1"
|
||||
SOURCE_DIR="$2"
|
||||
COMMIT="$3"
|
||||
|
||||
# Verify a cached reth binary was built from the expected commit.
|
||||
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
|
||||
verify_binary() {
|
||||
local binary="$1" expected_commit="$2"
|
||||
local version binary_sha
|
||||
version=$("$binary" --version 2>/dev/null) || return 1
|
||||
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
|
||||
if [ -z "$binary_sha" ]; then
|
||||
echo "Warning: could not extract commit SHA from version output"
|
||||
return 1
|
||||
fi
|
||||
if [ "$binary_sha" = "$expected_commit" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
baseline|main)
|
||||
BUCKET="minio/reth-binaries/${COMMIT}"
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat "${BUCKET}/reth" &>/dev/null; then
|
||||
echo "Cache hit for baseline (${COMMIT}), downloading binary..."
|
||||
$MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/reth"
|
||||
if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached baseline binary is stale, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building baseline (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
cargo build --profile profiling --bin reth
|
||||
$MC cp target/profiling/reth "${BUCKET}/reth"
|
||||
fi
|
||||
;;
|
||||
|
||||
feature|branch)
|
||||
BRANCH_SHA="${4:-$COMMIT}"
|
||||
BUCKET="minio/reth-binaries/${BRANCH_SHA}"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
|
||||
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
$MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
|
||||
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/reth" /home/ubuntu/.cargo/bin/reth-bench
|
||||
if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached feature binary is stale, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building feature (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
rustup show active-toolchain || rustup default stable
|
||||
make profiling
|
||||
make install-reth-bench
|
||||
$MC cp target/profiling/reth "${BUCKET}/reth"
|
||||
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
260
.github/scripts/bench-reth-charts.py
vendored
Normal file
260
.github/scripts/bench-reth-charts.py
vendored
Normal file
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate benchmark charts from reth-bench CSV output.
|
||||
|
||||
Usage:
|
||||
bench-engine-charts.py <combined_csv> --output-dir <dir> [--baseline <baseline_csv>]
|
||||
|
||||
Generates three PNG charts:
|
||||
1. newPayload latency + Ggas/s per block (+ latency diff when baseline present)
|
||||
2. Wait breakdown (persistence, execution cache, sparse trie) per block
|
||||
3. Scatter plot of gas used vs latency
|
||||
|
||||
When --baseline is provided, charts overlay both datasets for comparison.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
print("matplotlib is required: pip install matplotlib", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
GIGAGAS = 1_000_000_000
|
||||
|
||||
|
||||
def parse_combined_csv(path: str) -> list[dict]:
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"new_payload_latency_us": int(row["new_payload_latency"]),
|
||||
"persistence_wait_us": int(row["persistence_wait"])
|
||||
if row.get("persistence_wait")
|
||||
else None,
|
||||
"execution_cache_wait_us": int(row.get("execution_cache_wait", 0)),
|
||||
"sparse_trie_wait_us": int(row.get("sparse_trie_wait", 0)),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def plot_latency_and_throughput(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "baseline", feature_name: str = "feature",
|
||||
):
|
||||
num_plots = 3 if baseline else 2
|
||||
fig, axes = plt.subplots(num_plots, 1, figsize=(12, 4 * num_plots), sharex=True)
|
||||
ax1, ax2 = axes[0], axes[1]
|
||||
|
||||
feat_x = [r["block_number"] for r in feature]
|
||||
feat_lat = [r["new_payload_latency_us"] / 1_000 for r in feature]
|
||||
feat_ggas = []
|
||||
for r in feature:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
feat_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
|
||||
|
||||
if baseline:
|
||||
base_x = [r["block_number"] for r in baseline]
|
||||
base_lat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
|
||||
base_ggas = []
|
||||
for r in baseline:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
base_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
|
||||
l, = ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
ax1.axhline(np.median(base_lat), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
|
||||
l, = ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
ax2.axhline(np.median(base_ggas), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
|
||||
|
||||
l, = ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
|
||||
ax1.axhline(np.median(feat_lat), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
|
||||
ax1.set_ylabel("Latency (ms)")
|
||||
ax1.set_title("newPayload Latency per Block")
|
||||
ax1.grid(True, alpha=0.3)
|
||||
ax1.legend()
|
||||
|
||||
l, = ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
|
||||
ax2.axhline(np.median(feat_ggas), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
|
||||
ax2.set_ylabel("Ggas/s")
|
||||
ax2.set_title("Execution Throughput per Block")
|
||||
ax2.grid(True, alpha=0.3)
|
||||
ax2.legend()
|
||||
|
||||
if baseline:
|
||||
ax3 = axes[2]
|
||||
base_by_block = {r["block_number"]: r["new_payload_latency_us"] for r in baseline}
|
||||
blocks, diffs = [], []
|
||||
for r in feature:
|
||||
bn = r["block_number"]
|
||||
if bn in base_by_block and base_by_block[bn] > 0:
|
||||
pct = (r["new_payload_latency_us"] - base_by_block[bn]) / base_by_block[bn] * 100
|
||||
blocks.append(bn)
|
||||
diffs.append(pct)
|
||||
if blocks:
|
||||
colors = ["green" if d <= 0 else "red" for d in diffs]
|
||||
ax3.bar(blocks, diffs, width=1.0, color=colors, alpha=0.7, edgecolor="none")
|
||||
ax3.axhline(0, color="black", linewidth=0.5)
|
||||
ax3.set_ylabel("Δ Latency (%)")
|
||||
ax3.set_title("Per-Block newPayload Latency Change (feature vs baseline)")
|
||||
ax3.grid(True, alpha=0.3, axis="y")
|
||||
|
||||
axes[-1].set_xlabel("Block Number")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plot_wait_breakdown(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "baseline", feature_name: str = "feature",
|
||||
):
|
||||
series = [
|
||||
("Persistence Wait", "persistence_wait_us"),
|
||||
("State Cache Wait", "execution_cache_wait_us"),
|
||||
("Trie Cache Wait", "sparse_trie_wait_us"),
|
||||
]
|
||||
|
||||
fig, axes = plt.subplots(len(series), 1, figsize=(12, 3 * len(series)), sharex=True)
|
||||
for ax, (label, key) in zip(axes, series):
|
||||
if baseline:
|
||||
bx = [r["block_number"] for r in baseline if r[key] is not None]
|
||||
by = [r[key] / 1_000 for r in baseline if r[key] is not None]
|
||||
if bx:
|
||||
ax.plot(bx, by, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
|
||||
fx = [r["block_number"] for r in feature if r[key] is not None]
|
||||
fy = [r[key] / 1_000 for r in feature if r[key] is not None]
|
||||
if fx:
|
||||
ax.plot(fx, fy, linewidth=0.8, label=feature_name)
|
||||
|
||||
ax.set_ylabel("ms")
|
||||
ax.set_title(label)
|
||||
ax.grid(True, alpha=0.3)
|
||||
if baseline:
|
||||
ax.legend()
|
||||
|
||||
axes[-1].set_xlabel("Block Number")
|
||||
fig.suptitle("Wait Time Breakdown per Block", fontsize=14, y=1.01)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def _add_regression(ax, x, y, color, label):
|
||||
"""Add a linear regression line to the axes."""
|
||||
if len(x) < 2:
|
||||
return
|
||||
xa, ya = np.array(x), np.array(y)
|
||||
m, b = np.polyfit(xa, ya, 1)
|
||||
x_range = np.linspace(xa.min(), xa.max(), 100)
|
||||
ax.plot(x_range, m * x_range + b, color=color, linewidth=1.5, alpha=0.8,
|
||||
label=label)
|
||||
|
||||
|
||||
def plot_gas_vs_latency(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "baseline", feature_name: str = "feature",
|
||||
):
|
||||
fig, ax = plt.subplots(figsize=(8, 6))
|
||||
|
||||
if baseline:
|
||||
bgas = [r["gas_used"] / 1_000_000 for r in baseline]
|
||||
blat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
|
||||
ax.scatter(bgas, blat, s=8, alpha=0.5)
|
||||
_add_regression(ax, bgas, blat, "tab:blue", baseline_name)
|
||||
|
||||
fgas = [r["gas_used"] / 1_000_000 for r in feature]
|
||||
flat = [r["new_payload_latency_us"] / 1_000 for r in feature]
|
||||
ax.scatter(fgas, flat, s=8, alpha=0.6)
|
||||
_add_regression(ax, fgas, flat, "tab:orange", feature_name)
|
||||
|
||||
ax.set_xlabel("Gas Used (Mgas)")
|
||||
ax.set_ylabel("newPayload Latency (ms)")
|
||||
ax.set_title("Gas Used vs Latency")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def merge_csvs(paths: list[str]) -> list[dict]:
|
||||
"""Parse and merge multiple CSVs, averaging values for duplicate blocks."""
|
||||
by_block: dict[int, list[dict]] = {}
|
||||
for path in paths:
|
||||
for row in parse_combined_csv(path):
|
||||
by_block.setdefault(row["block_number"], []).append(row)
|
||||
|
||||
merged = []
|
||||
for bn in sorted(by_block):
|
||||
rows = by_block[bn]
|
||||
if len(rows) == 1:
|
||||
merged.append(rows[0])
|
||||
else:
|
||||
avg = {"block_number": bn}
|
||||
for key in ("gas_used", "new_payload_latency_us"):
|
||||
avg[key] = int(sum(r[key] for r in rows) / len(rows))
|
||||
for key in ("persistence_wait_us", "execution_cache_wait_us", "sparse_trie_wait_us"):
|
||||
vals = [r[key] for r in rows if r[key] is not None]
|
||||
avg[key] = int(sum(vals) / len(vals)) if vals else None
|
||||
merged.append(avg)
|
||||
return merged
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate benchmark charts")
|
||||
parser.add_argument(
|
||||
"--feature", nargs="+", required=True,
|
||||
help="Path(s) to feature combined_latency.csv",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir", required=True, help="Output directory for PNG charts"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline", nargs="+", help="Path(s) to baseline combined_latency.csv"
|
||||
)
|
||||
parser.add_argument("--baseline-name", default="baseline", help="Label for baseline")
|
||||
parser.add_argument("--feature-name", "--branch-name", default="feature", help="Label for feature")
|
||||
args = parser.parse_args()
|
||||
|
||||
feature = merge_csvs(args.feature)
|
||||
if not feature:
|
||||
print("No results found in feature CSV(s)", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
baseline = None
|
||||
if args.baseline:
|
||||
baseline = merge_csvs(args.baseline)
|
||||
if not baseline:
|
||||
print(
|
||||
"Warning: no results in baseline CSV(s), skipping comparison",
|
||||
file=sys.stderr,
|
||||
)
|
||||
baseline = None
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bname = args.baseline_name
|
||||
fname = args.feature_name
|
||||
plot_latency_and_throughput(feature, baseline, out_dir / "latency_throughput.png", bname, fname)
|
||||
plot_wait_breakdown(feature, baseline, out_dir / "wait_breakdown.png", bname, fname)
|
||||
plot_gas_vs_latency(feature, baseline, out_dir / "gas_vs_latency.png", bname, fname)
|
||||
|
||||
print(f"Charts written to {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
144
.github/scripts/bench-reth-run.sh
vendored
Executable file
144
.github/scripts/bench-reth-run.sh
vendored
Executable file
@@ -0,0 +1,144 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Runs a single reth-bench cycle: mount snapshot → start node → warmup →
|
||||
# benchmark → stop node → recover snapshot.
|
||||
#
|
||||
# Usage: bench-reth-run.sh <label> <binary> <output-dir>
|
||||
#
|
||||
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
|
||||
set -euo pipefail
|
||||
|
||||
LABEL="$1"
|
||||
BINARY="$2"
|
||||
OUTPUT_DIR="$3"
|
||||
DATADIR="$SCHELK_MOUNT/datadir"
|
||||
mkdir -p "$OUTPUT_DIR"
|
||||
LOG="${OUTPUT_DIR}/node.log"
|
||||
|
||||
cleanup() {
|
||||
kill "$TAIL_PID" 2>/dev/null || true
|
||||
if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
|
||||
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
|
||||
# Send SIGINT to the inner reth process by exact name (not -f which
|
||||
# would also match samply's cmdline containing "reth"). Samply will
|
||||
# capture reth's exit and save the profile.
|
||||
sudo pkill -INT -x reth 2>/dev/null || true
|
||||
# Wait for samply to finish writing the profile and exit
|
||||
for i in $(seq 1 120); do
|
||||
sudo pgrep -x samply > /dev/null 2>&1 || break
|
||||
if [ $((i % 10)) -eq 0 ]; then
|
||||
echo "Waiting for samply to finish writing profile... (${i}s)"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
if sudo pgrep -x samply > /dev/null 2>&1; then
|
||||
echo "Samply still running after 120s, sending SIGTERM..."
|
||||
sudo pkill -x samply 2>/dev/null || true
|
||||
fi
|
||||
else
|
||||
sudo kill "$RETH_PID"
|
||||
for i in $(seq 1 30); do
|
||||
sudo kill -0 "$RETH_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
fi
|
||||
sudo kill -9 "$RETH_PID" 2>/dev/null || true
|
||||
sleep 1
|
||||
fi
|
||||
# Fix ownership of reth-created files (reth runs as root)
|
||||
sudo chown -R "$(id -un):$(id -gn)" "$OUTPUT_DIR" 2>/dev/null || true
|
||||
if mountpoint -q "$SCHELK_MOUNT"; then
|
||||
sudo umount -l "$SCHELK_MOUNT" || true
|
||||
sudo schelk recover -y || true
|
||||
fi
|
||||
}
|
||||
TAIL_PID=
|
||||
trap cleanup EXIT
|
||||
|
||||
# Mount
|
||||
sudo schelk mount -y
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
echo "=== Cache state after drop ==="
|
||||
free -h
|
||||
grep Cached /proc/meminfo
|
||||
|
||||
# Start reth
|
||||
# CPU layout: core 0 = OS/IRQs/reth-bench/aux, cores 1+ = reth node
|
||||
RETH_BENCH="$(which reth-bench)"
|
||||
ONLINE=$(nproc --all)
|
||||
MAX_RETH=$(( ONLINE - 1 ))
|
||||
if [ "${BENCH_CORES:-0}" -gt 0 ] && [ "$BENCH_CORES" -lt "$MAX_RETH" ]; then
|
||||
MAX_RETH=$BENCH_CORES
|
||||
fi
|
||||
RETH_CPUS="1-${MAX_RETH}"
|
||||
|
||||
RETH_ARGS=(
|
||||
node
|
||||
--datadir "$DATADIR"
|
||||
--log.file.directory "$OUTPUT_DIR/reth-logs"
|
||||
--engine.accept-execution-requests-hash
|
||||
--http
|
||||
--http.port 8545
|
||||
--ws
|
||||
--ws.api all
|
||||
--authrpc.port 8551
|
||||
--disable-discovery
|
||||
--no-persist-peers
|
||||
)
|
||||
|
||||
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
|
||||
RETH_ARGS+=(--log.samply)
|
||||
SAMPLY="$(which samply)"
|
||||
sudo taskset -c "$RETH_CPUS" nice -n -20 \
|
||||
"$SAMPLY" record --save-only --presymbolicate --rate 10000 \
|
||||
--output "$OUTPUT_DIR/samply-profile.json.gz" \
|
||||
-- "$BINARY" "${RETH_ARGS[@]}" \
|
||||
> "$LOG" 2>&1 &
|
||||
else
|
||||
sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
|
||||
> "$LOG" 2>&1 &
|
||||
fi
|
||||
|
||||
RETH_PID=$!
|
||||
stdbuf -oL tail -f "$LOG" | sed -u "s/^/[reth] /" &
|
||||
TAIL_PID=$!
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://127.0.0.1:8545 -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
> /dev/null 2>&1; then
|
||||
echo "reth (${LABEL}) is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo "::error::reth (${LABEL}) failed to start within 60s"
|
||||
cat "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Run reth-bench with high priority but as the current user so output
|
||||
# files are not root-owned (avoids EACCES on next checkout).
|
||||
BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
|
||||
|
||||
# Warmup
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
|
||||
--reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
|
||||
|
||||
# Benchmark
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "$BENCH_BLOCKS" \
|
||||
--reth-new-payload \
|
||||
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
|
||||
|
||||
# cleanup runs via trap
|
||||
127
.github/scripts/bench-reth-snapshot.sh
vendored
Executable file
127
.github/scripts/bench-reth-snapshot.sh
vendored
Executable file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Downloads the latest nightly snapshot into the schelk volume with
|
||||
# progress reporting to the GitHub PR comment.
|
||||
#
|
||||
# Skips the download if the local ETag marker matches the remote one.
|
||||
#
|
||||
# Usage: bench-reth-snapshot.sh [--check]
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 1 if not.
|
||||
#
|
||||
# Required env:
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# GITHUB_TOKEN – token for GitHub API calls (only for download)
|
||||
# BENCH_COMMENT_ID – PR comment ID to update (optional)
|
||||
# BENCH_REPO – owner/repo (e.g. paradigmxyz/reth)
|
||||
# BENCH_JOB_URL – link to the Actions job
|
||||
# BENCH_ACTOR – user who triggered the benchmark
|
||||
# BENCH_CONFIG – config summary line
|
||||
set -euo pipefail
|
||||
|
||||
BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
|
||||
DATADIR="$SCHELK_MOUNT/datadir"
|
||||
ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
|
||||
|
||||
# Get remote metadata via JSON for reliable parsing
|
||||
MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
|
||||
REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
|
||||
if [ -z "$REMOTE_ETAG" ]; then
|
||||
echo "::warning::Failed to get ETag from mc stat, will re-download"
|
||||
REMOTE_ETAG="unknown-$(date +%s)"
|
||||
fi
|
||||
|
||||
LOCAL_ETAG=""
|
||||
[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
|
||||
|
||||
if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
|
||||
echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Get compressed size for progress tracking
|
||||
TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
|
||||
if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
|
||||
echo "::error::Failed to get snapshot size from mc stat"
|
||||
exit 1
|
||||
fi
|
||||
echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
|
||||
|
||||
# Prepare mount
|
||||
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
||||
sudo schelk mount -y
|
||||
sudo rm -rf "$DATADIR"
|
||||
sudo mkdir -p "$DATADIR"
|
||||
|
||||
update_comment() {
|
||||
local pct="$1"
|
||||
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
|
||||
local status="Building binaries & downloading snapshot… ${pct}%"
|
||||
local body
|
||||
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
|
||||
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
|
||||
curl -sf -X PATCH \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
|
||||
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
# Track compressed bytes flowing through the pipe
|
||||
DL_BYTES_FILE=$(mktemp)
|
||||
echo 0 > "$DL_BYTES_FILE"
|
||||
|
||||
# Start progress reporter in background
|
||||
(
|
||||
while true; do
|
||||
sleep 10
|
||||
CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$TOTAL_BYTES" -gt 0 ]; then
|
||||
PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
|
||||
[ "$PCT" -gt 100 ] && PCT=100
|
||||
echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
|
||||
update_comment "$PCT"
|
||||
fi
|
||||
done
|
||||
) &
|
||||
PROGRESS_PID=$!
|
||||
trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
|
||||
|
||||
# Download and extract; python byte counter tracks compressed bytes received
|
||||
mc cat "$BUCKET" | python3 -c "
|
||||
import sys
|
||||
count = 0
|
||||
while True:
|
||||
data = sys.stdin.buffer.read(1048576)
|
||||
if not data:
|
||||
break
|
||||
count += len(data)
|
||||
sys.stdout.buffer.write(data)
|
||||
with open('$DL_BYTES_FILE', 'w') as f:
|
||||
f.write(str(count))
|
||||
" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
|
||||
|
||||
# Stop progress reporter
|
||||
kill $PROGRESS_PID 2>/dev/null || true
|
||||
wait $PROGRESS_PID 2>/dev/null || true
|
||||
|
||||
update_comment "100"
|
||||
echo "Snapshot download complete"
|
||||
|
||||
# Promote the new snapshot to become the schelk baseline (virgin volume).
|
||||
# This copies changed blocks from scratch → virgin so that future
|
||||
# `schelk recover` calls restore to this new state.
|
||||
sync
|
||||
sudo schelk promote -y
|
||||
|
||||
# Save ETag marker
|
||||
echo "$REMOTE_ETAG" > "$ETAG_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (ETag: ${REMOTE_ETAG})"
|
||||
553
.github/scripts/bench-reth-summary.py
vendored
Executable file
553
.github/scripts/bench-reth-summary.py
vendored
Executable file
@@ -0,0 +1,553 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse reth-bench CSV output and generate a summary JSON + markdown comparison.
|
||||
|
||||
Usage:
|
||||
bench-reth-summary.py <combined_csv> <gas_csv> \
|
||||
--output-summary <summary.json> \
|
||||
--output-markdown <comment.md> \
|
||||
--baseline-csv <baseline_combined.csv> \
|
||||
[--repo <owner/repo>] \
|
||||
[--baseline-ref <sha>] \
|
||||
[--feature-name <name>] \
|
||||
[--feature-sha <sha>]
|
||||
|
||||
Generates a paired statistical comparison between baseline and feature.
|
||||
Matches blocks by number and computes per-block diffs to cancel out gas
|
||||
variance. Fails if baseline or feature CSV is missing or empty.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import sys
|
||||
|
||||
GIGAGAS = 1_000_000_000
|
||||
T_CRITICAL = 1.96 # two-tailed 95% confidence
|
||||
BOOTSTRAP_ITERATIONS = 10_000
|
||||
|
||||
|
||||
def _opt_int(row: dict, key: str) -> int | None:
|
||||
"""Return int value for a CSV field, or None if missing/empty."""
|
||||
v = row.get(key)
|
||||
if v is None or v == "":
|
||||
return None
|
||||
return int(v)
|
||||
|
||||
|
||||
def parse_combined_csv(path: str) -> list[dict]:
|
||||
"""Parse combined_latency.csv into a list of per-block dicts."""
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"gas_limit": int(row["gas_limit"]),
|
||||
"transaction_count": int(row["transaction_count"]),
|
||||
"new_payload_latency_us": int(row["new_payload_latency"]),
|
||||
"fcu_latency_us": int(row["fcu_latency"]),
|
||||
"total_latency_us": int(row["total_latency"]),
|
||||
"persistence_wait_us": _opt_int(row, "persistence_wait"),
|
||||
"execution_cache_wait_us": _opt_int(row, "execution_cache_wait"),
|
||||
"sparse_trie_wait_us": _opt_int(row, "sparse_trie_wait"),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def parse_gas_csv(path: str) -> list[dict]:
|
||||
"""Parse total_gas.csv into a list of per-block dicts."""
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"time_us": int(row["time"]),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def stddev(values: list[float], mean: float) -> float:
|
||||
if len(values) < 2:
|
||||
return 0.0
|
||||
return math.sqrt(sum((v - mean) ** 2 for v in values) / (len(values) - 1))
|
||||
|
||||
|
||||
def percentile(sorted_vals: list[float], pct: int) -> float:
|
||||
if not sorted_vals:
|
||||
return 0.0
|
||||
idx = int(len(sorted_vals) * pct / 100)
|
||||
idx = min(idx, len(sorted_vals) - 1)
|
||||
return sorted_vals[idx]
|
||||
|
||||
|
||||
def compute_stats(combined: list[dict]) -> dict:
|
||||
"""Compute per-run statistics from parsed CSV data."""
|
||||
n = len(combined)
|
||||
if n == 0:
|
||||
return {}
|
||||
|
||||
latencies_ms = [r["new_payload_latency_us"] / 1_000 for r in combined]
|
||||
sorted_lat = sorted(latencies_ms)
|
||||
mean_lat = sum(latencies_ms) / n
|
||||
std_lat = stddev(latencies_ms, mean_lat)
|
||||
|
||||
mgas_s_values = []
|
||||
for r in combined:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
if lat_s > 0:
|
||||
mgas_s_values.append(r["gas_used"] / lat_s / 1_000_000)
|
||||
mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_ms": mean_lat,
|
||||
"stddev_ms": std_lat,
|
||||
"p50_ms": percentile(sorted_lat, 50),
|
||||
"p90_ms": percentile(sorted_lat, 90),
|
||||
"p99_ms": percentile(sorted_lat, 99),
|
||||
"mean_mgas_s": mean_mgas_s,
|
||||
}
|
||||
|
||||
|
||||
def compute_wait_stats(combined: list[dict], field: str) -> dict:
|
||||
"""Compute mean/p50/p95 for a wait time field (in ms)."""
|
||||
values_ms = []
|
||||
for r in combined:
|
||||
v = r.get(field)
|
||||
if v is not None:
|
||||
values_ms.append(v / 1_000)
|
||||
if not values_ms:
|
||||
return {}
|
||||
n = len(values_ms)
|
||||
mean_val = sum(values_ms) / n
|
||||
sorted_vals = sorted(values_ms)
|
||||
return {
|
||||
"mean_ms": mean_val,
|
||||
"p50_ms": percentile(sorted_vals, 50),
|
||||
"p95_ms": percentile(sorted_vals, 95),
|
||||
}
|
||||
|
||||
|
||||
def _paired_data(
|
||||
baseline: list[dict], feature: list[dict]
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float]]:
|
||||
"""Match blocks and return paired latencies and per-block diffs.
|
||||
|
||||
Returns:
|
||||
pairs: list of (baseline_ms, feature_ms) tuples
|
||||
lat_diffs_ms: list of feature − baseline latency diffs in ms
|
||||
mgas_diffs: list of feature − baseline Mgas/s diffs
|
||||
"""
|
||||
baseline_by_block = {r["block_number"]: r for r in baseline}
|
||||
feature_by_block = {r["block_number"]: r for r in feature}
|
||||
common_blocks = sorted(set(baseline_by_block) & set(feature_by_block))
|
||||
|
||||
pairs = []
|
||||
lat_diffs_ms = []
|
||||
mgas_diffs = []
|
||||
for bn in common_blocks:
|
||||
b = baseline_by_block[bn]
|
||||
f = feature_by_block[bn]
|
||||
b_ms = b["new_payload_latency_us"] / 1_000
|
||||
f_ms = f["new_payload_latency_us"] / 1_000
|
||||
pairs.append((b_ms, f_ms))
|
||||
lat_diffs_ms.append(f_ms - b_ms)
|
||||
b_lat_s = b["new_payload_latency_us"] / 1_000_000
|
||||
f_lat_s = f["new_payload_latency_us"] / 1_000_000
|
||||
if b_lat_s > 0 and f_lat_s > 0:
|
||||
mgas_diffs.append(
|
||||
f["gas_used"] / f_lat_s / 1_000_000
|
||||
- b["gas_used"] / b_lat_s / 1_000_000
|
||||
)
|
||||
return pairs, lat_diffs_ms, mgas_diffs
|
||||
|
||||
|
||||
def compute_paired_stats(
|
||||
baseline_runs: list[list[dict]],
|
||||
feature_runs: list[list[dict]],
|
||||
) -> dict:
|
||||
"""Compute paired statistics between baseline and feature runs.
|
||||
|
||||
Each pair (baseline_runs[i], feature_runs[i]) produces per-block diffs.
|
||||
All diffs are pooled for the final CI.
|
||||
"""
|
||||
all_pairs = []
|
||||
all_lat_diffs = []
|
||||
all_mgas_diffs = []
|
||||
blocks_per_pair = []
|
||||
for baseline, feature in zip(baseline_runs, feature_runs):
|
||||
pairs, lat_diffs, mgas_diffs = _paired_data(baseline, feature)
|
||||
all_pairs.extend(pairs)
|
||||
all_lat_diffs.extend(lat_diffs)
|
||||
all_mgas_diffs.extend(mgas_diffs)
|
||||
blocks_per_pair.append(len(pairs))
|
||||
|
||||
if not all_lat_diffs:
|
||||
return {}
|
||||
|
||||
n = len(all_lat_diffs)
|
||||
mean_diff = sum(all_lat_diffs) / n
|
||||
std_diff = stddev(all_lat_diffs, mean_diff)
|
||||
se = std_diff / math.sqrt(n) if n > 0 else 0.0
|
||||
ci = T_CRITICAL * se
|
||||
|
||||
# Bootstrap CI on difference-of-percentiles (resample paired blocks)
|
||||
base_lats = sorted([p[0] for p in all_pairs])
|
||||
feature_lats = sorted([p[1] for p in all_pairs])
|
||||
p50_diff = percentile(feature_lats, 50) - percentile(base_lats, 50)
|
||||
p90_diff = percentile(feature_lats, 90) - percentile(base_lats, 90)
|
||||
p99_diff = percentile(feature_lats, 99) - percentile(base_lats, 99)
|
||||
|
||||
rng = random.Random(42)
|
||||
p50_boot, p90_boot, p99_boot = [], [], []
|
||||
for _ in range(BOOTSTRAP_ITERATIONS):
|
||||
sample = rng.choices(all_pairs, k=n)
|
||||
b_sorted = sorted(p[0] for p in sample)
|
||||
f_sorted = sorted(p[1] for p in sample)
|
||||
p50_boot.append(percentile(f_sorted, 50) - percentile(b_sorted, 50))
|
||||
p90_boot.append(percentile(f_sorted, 90) - percentile(b_sorted, 90))
|
||||
p99_boot.append(percentile(f_sorted, 99) - percentile(b_sorted, 99))
|
||||
p50_boot.sort()
|
||||
p90_boot.sort()
|
||||
p99_boot.sort()
|
||||
lo = int(BOOTSTRAP_ITERATIONS * 0.025)
|
||||
hi = int(BOOTSTRAP_ITERATIONS * 0.975)
|
||||
|
||||
mean_mgas_diff = sum(all_mgas_diffs) / len(all_mgas_diffs) if all_mgas_diffs else 0.0
|
||||
std_mgas_diff = stddev(all_mgas_diffs, mean_mgas_diff) if len(all_mgas_diffs) > 1 else 0.0
|
||||
mgas_se = std_mgas_diff / math.sqrt(len(all_mgas_diffs)) if all_mgas_diffs else 0.0
|
||||
mgas_ci = T_CRITICAL * mgas_se
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_diff_ms": mean_diff,
|
||||
"ci_ms": ci,
|
||||
"p50_diff_ms": p50_diff,
|
||||
"p50_ci_ms": (p50_boot[hi] - p50_boot[lo]) / 2,
|
||||
"p90_diff_ms": p90_diff,
|
||||
"p90_ci_ms": (p90_boot[hi] - p90_boot[lo]) / 2,
|
||||
"p99_diff_ms": p99_diff,
|
||||
"p99_ci_ms": (p99_boot[hi] - p99_boot[lo]) / 2,
|
||||
"mean_mgas_diff": mean_mgas_diff,
|
||||
"mgas_ci": mgas_ci,
|
||||
"blocks": max(blocks_per_pair),
|
||||
}
|
||||
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
if seconds >= 60:
|
||||
return f"{seconds / 60:.1f}min"
|
||||
return f"{seconds}s"
|
||||
|
||||
|
||||
def format_gas(gas: int) -> str:
|
||||
if gas >= GIGAGAS:
|
||||
return f"{gas / GIGAGAS:.1f}G"
|
||||
if gas >= 1_000_000:
|
||||
return f"{gas / 1_000_000:.1f}M"
|
||||
return f"{gas:,}"
|
||||
|
||||
|
||||
|
||||
def fmt_ms(v: float) -> str:
|
||||
return f"{v:.2f}ms"
|
||||
|
||||
|
||||
def fmt_mgas(v: float) -> str:
|
||||
return f"{v:.2f}"
|
||||
|
||||
|
||||
def significance(pct: float, ci_pct: float, lower_is_better: bool) -> str:
|
||||
"""Return significance label: 'good', 'bad', or 'neutral'."""
|
||||
significant = abs(pct) > ci_pct
|
||||
if not significant:
|
||||
return "neutral"
|
||||
elif (pct < 0) == lower_is_better:
|
||||
return "good"
|
||||
else:
|
||||
return "bad"
|
||||
|
||||
|
||||
def change_str(pct: float, ci_pct: float, lower_is_better: bool) -> str:
|
||||
"""Format change% with paired CI significance.
|
||||
|
||||
Significant if the CI doesn't cross zero (i.e. |pct| > ci_pct).
|
||||
"""
|
||||
sig = significance(pct, ci_pct, lower_is_better)
|
||||
emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[sig]
|
||||
return f"{pct:+.2f}% {emoji} (±{ci_pct:.2f}%)"
|
||||
|
||||
|
||||
def compute_changes(
|
||||
baseline_stats: dict, feature_stats: dict, paired_stats: dict
|
||||
) -> dict:
|
||||
"""Pre-compute change percentages and significance for each metric."""
|
||||
def pct(base: float, feat: float) -> float:
|
||||
return (feat - base) / base * 100.0 if base > 0 else 0.0
|
||||
|
||||
def ci_pct(ci_ms: float, base_ms: float) -> float:
|
||||
return ci_ms / base_ms * 100.0 if base_ms > 0 else 0.0
|
||||
|
||||
metrics = [
|
||||
("mean", "mean_ms", "ci_ms", "mean_ms", True),
|
||||
("p50", "p50_ms", "p50_ci_ms", "p50_ms", True),
|
||||
("p90", "p90_ms", "p90_ci_ms", "p90_ms", True),
|
||||
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
|
||||
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
|
||||
]
|
||||
changes = {}
|
||||
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
|
||||
p = pct(baseline_stats[stat_key], feature_stats[stat_key])
|
||||
c = ci_pct(paired_stats[ci_key], baseline_stats[base_key])
|
||||
changes[name] = {
|
||||
"pct": round(p, 4),
|
||||
"ci_pct": round(c, 4),
|
||||
"sig": significance(p, c, lower_is_better),
|
||||
}
|
||||
return changes
|
||||
|
||||
|
||||
def generate_comparison_table(
|
||||
run1: dict,
|
||||
run2: dict,
|
||||
paired: dict,
|
||||
repo: str,
|
||||
baseline_ref: str,
|
||||
baseline_name: str,
|
||||
feature_name: str,
|
||||
feature_sha: str,
|
||||
) -> str:
|
||||
"""Generate a markdown comparison table between baseline and feature."""
|
||||
n = paired["blocks"]
|
||||
|
||||
def pct(base: float, feat: float) -> float:
|
||||
return (feat - base) / base * 100.0 if base > 0 else 0.0
|
||||
|
||||
mean_pct = pct(run1["mean_ms"], run2["mean_ms"])
|
||||
gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"])
|
||||
|
||||
p50_pct = pct(run1["p50_ms"], run2["p50_ms"])
|
||||
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
|
||||
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
|
||||
|
||||
# Bootstrap CIs as % of baseline percentile
|
||||
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
|
||||
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
|
||||
p99_ci_pct = paired["p99_ci_ms"] / run1["p99_ms"] * 100.0 if run1["p99_ms"] > 0 else 0.0
|
||||
|
||||
# CI as a percentage of baseline mean
|
||||
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
|
||||
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
|
||||
|
||||
base_url = f"https://github.com/{repo}/commit"
|
||||
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
|
||||
feature_label = f"[`{feature_name}`]({base_url}/{feature_sha})"
|
||||
|
||||
lines = [
|
||||
f"| Metric | {baseline_label} | {feature_label} | Change |",
|
||||
"|--------|------|--------|--------|",
|
||||
f"| Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, lat_ci_pct, lower_is_better=True)} |",
|
||||
f"| StdDev | {fmt_ms(run1['stddev_ms'])} | {fmt_ms(run2['stddev_ms'])} | |",
|
||||
f"| P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, lower_is_better=True)} |",
|
||||
f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, lower_is_better=True)} |",
|
||||
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
|
||||
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
|
||||
"",
|
||||
f"*{n} blocks*",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_wait_time_table(
|
||||
title: str,
|
||||
baseline_stats: dict,
|
||||
feature_stats: dict,
|
||||
baseline_label: str,
|
||||
feature_label: str,
|
||||
) -> str:
|
||||
"""Generate a markdown table for a wait time metric."""
|
||||
if not baseline_stats or not feature_stats:
|
||||
return ""
|
||||
lines = [
|
||||
f"### {title}",
|
||||
"",
|
||||
f"| Metric | {baseline_label} | {feature_label} |",
|
||||
"|--------|------|--------|",
|
||||
f"| Mean | {fmt_ms(baseline_stats['mean_ms'])} | {fmt_ms(feature_stats['mean_ms'])} |",
|
||||
f"| P50 | {fmt_ms(baseline_stats['p50_ms'])} | {fmt_ms(feature_stats['p50_ms'])} |",
|
||||
f"| P95 | {fmt_ms(baseline_stats['p95_ms'])} | {fmt_ms(feature_stats['p95_ms'])} |",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_markdown(
|
||||
summary: dict, comparison_table: str,
|
||||
wait_time_tables: list[str] | None = None,
|
||||
behind_baseline: int = 0, repo: str = "", baseline_ref: str = "", baseline_name: str = "",
|
||||
) -> str:
|
||||
"""Generate a markdown comment body."""
|
||||
lines = ["## Benchmark Results", ""]
|
||||
if behind_baseline > 0:
|
||||
s = "s" if behind_baseline > 1 else ""
|
||||
diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}"
|
||||
lines.append(f"> ⚠️ Feature is [**{behind_baseline} commit{s} behind `{baseline_name}`**]({diff_link}). Consider rebasing for accurate results.")
|
||||
lines.append("")
|
||||
lines.append(comparison_table)
|
||||
if wait_time_tables:
|
||||
lines.append("")
|
||||
lines.append("<details>")
|
||||
lines.append("<summary>Wait Time Breakdown</summary>")
|
||||
lines.append("")
|
||||
for table in wait_time_tables:
|
||||
if table:
|
||||
lines.append(table)
|
||||
lines.append("")
|
||||
lines.append("</details>")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Parse reth-bench ABBA results")
|
||||
parser.add_argument(
|
||||
"--baseline-csv", nargs="+", required=True,
|
||||
help="Baseline combined_latency.csv files (A1, A2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--feature-csv", "--branch-csv", nargs="+", required=True,
|
||||
help="Feature combined_latency.csv files (B1, B2)",
|
||||
)
|
||||
parser.add_argument("--gas-csv", required=True, help="Path to total_gas.csv")
|
||||
parser.add_argument(
|
||||
"--output-summary", required=True, help="Output JSON summary path"
|
||||
)
|
||||
parser.add_argument("--output-markdown", required=True, help="Output markdown path")
|
||||
parser.add_argument(
|
||||
"--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)"
|
||||
)
|
||||
parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA")
|
||||
parser.add_argument("--baseline-name", default=None, help="Baseline display name")
|
||||
parser.add_argument("--feature-name", "--branch-name", default=None, help="Feature branch name")
|
||||
parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
|
||||
parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.baseline_csv) != len(args.feature_csv):
|
||||
print("Must provide equal number of baseline and feature CSVs", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
baseline_runs = []
|
||||
feature_runs = []
|
||||
for path in args.baseline_csv:
|
||||
data = parse_combined_csv(path)
|
||||
if not data:
|
||||
print(f"No results in {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
baseline_runs.append(data)
|
||||
for path in args.feature_csv:
|
||||
data = parse_combined_csv(path)
|
||||
if not data:
|
||||
print(f"No results in {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
feature_runs.append(data)
|
||||
|
||||
gas = parse_gas_csv(args.gas_csv)
|
||||
|
||||
all_baseline = [r for run in baseline_runs for r in run]
|
||||
all_feature = [r for run in feature_runs for r in run]
|
||||
|
||||
baseline_stats = compute_stats(all_baseline)
|
||||
feature_stats = compute_stats(all_feature)
|
||||
paired_stats = compute_paired_stats(baseline_runs, feature_runs)
|
||||
|
||||
if not paired_stats:
|
||||
print("No common blocks between baseline and feature runs", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
baseline_ref = args.baseline_ref or "main"
|
||||
baseline_name = args.baseline_name or "baseline"
|
||||
feature_name = args.feature_name or "feature"
|
||||
feature_sha = args.feature_ref or "unknown"
|
||||
|
||||
comparison_table = generate_comparison_table(
|
||||
baseline_stats,
|
||||
feature_stats,
|
||||
paired_stats,
|
||||
repo=args.repo,
|
||||
baseline_ref=baseline_ref,
|
||||
baseline_name=baseline_name,
|
||||
feature_name=feature_name,
|
||||
feature_sha=feature_sha,
|
||||
)
|
||||
print(f"Generated comparison ({paired_stats['n']} paired blocks, "
|
||||
f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
|
||||
|
||||
base_url = f"https://github.com/{args.repo}/commit"
|
||||
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
|
||||
feature_label = f"[`{feature_name}`]({base_url}/{feature_sha})"
|
||||
|
||||
wait_fields = [
|
||||
("persistence_wait_us", "Persistence Wait"),
|
||||
("sparse_trie_wait_us", "Trie Cache Update Wait"),
|
||||
("execution_cache_wait_us", "Execution Cache Update Wait"),
|
||||
]
|
||||
wait_time_tables = []
|
||||
wait_time_data = {}
|
||||
for field, title in wait_fields:
|
||||
b_stats = compute_wait_stats(all_baseline, field)
|
||||
f_stats = compute_wait_stats(all_feature, field)
|
||||
if b_stats and f_stats:
|
||||
wait_time_data[field] = {
|
||||
"title": title,
|
||||
"baseline": b_stats,
|
||||
"feature": f_stats,
|
||||
}
|
||||
table = generate_wait_time_table(title, b_stats, f_stats, baseline_label, feature_label)
|
||||
if table:
|
||||
wait_time_tables.append(table)
|
||||
|
||||
summary = {
|
||||
"blocks": paired_stats["blocks"],
|
||||
"baseline": {
|
||||
"name": baseline_name,
|
||||
"ref": baseline_ref,
|
||||
"stats": baseline_stats,
|
||||
},
|
||||
"feature": {
|
||||
"name": feature_name,
|
||||
"ref": feature_sha,
|
||||
"stats": feature_stats,
|
||||
},
|
||||
"paired": paired_stats,
|
||||
"changes": compute_changes(baseline_stats, feature_stats, paired_stats),
|
||||
"wait_times": wait_time_data,
|
||||
}
|
||||
with open(args.output_summary, "w") as f:
|
||||
json.dump(summary, f, indent=2)
|
||||
print(f"Summary written to {args.output_summary}")
|
||||
|
||||
markdown = generate_markdown(
|
||||
summary, comparison_table,
|
||||
wait_time_tables=wait_time_tables,
|
||||
behind_baseline=args.behind_baseline,
|
||||
repo=args.repo,
|
||||
baseline_ref=baseline_ref,
|
||||
baseline_name=baseline_name,
|
||||
)
|
||||
|
||||
with open(args.output_markdown, "w") as f:
|
||||
f.write(markdown)
|
||||
print(f"Markdown written to {args.output_markdown}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
342
.github/scripts/bench-slack-notify.js
vendored
Normal file
342
.github/scripts/bench-slack-notify.js
vendored
Normal file
@@ -0,0 +1,342 @@
|
||||
// Sends Slack notifications for reth-bench results.
|
||||
//
|
||||
// Reads from environment:
|
||||
// SLACK_BENCH_BOT_TOKEN – Slack Bot User OAuth Token (xoxb-...)
|
||||
// SLACK_BENCH_CHANNEL – Public channel ID for significant improvements
|
||||
// BENCH_WORK_DIR – Directory containing summary.json
|
||||
// BENCH_PR – PR number (may be empty)
|
||||
// BENCH_ACTOR – GitHub user who triggered the bench
|
||||
// BENCH_JOB_URL – URL to the Actions job page
|
||||
// BENCH_SAMPLY – 'true' if samply profiling was enabled
|
||||
//
|
||||
// Usage from actions/github-script:
|
||||
// const notify = require('./.github/scripts/bench-slack-notify.js');
|
||||
// await notify.success({ core, context });
|
||||
// await notify.failure({ core, context, failedStep: '...' });
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const SLACK_API = 'https://slack.com/api/chat.postMessage';
|
||||
|
||||
function loadSlackUsers(repoRoot) {
|
||||
try {
|
||||
const raw = fs.readFileSync(path.join(repoRoot, '.github', 'scripts', 'bench-slack-users.json'), 'utf8');
|
||||
const data = JSON.parse(raw);
|
||||
// Filter out non-user-ID entries (like _comment)
|
||||
const users = {};
|
||||
for (const [k, v] of Object.entries(data)) {
|
||||
if (!k.startsWith('_') && typeof v === 'string' && v.startsWith('U')) {
|
||||
users[k] = v;
|
||||
}
|
||||
}
|
||||
return users;
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
async function postToSlack(token, channel, blocks, text, core, threadTs) {
|
||||
const payload = { channel, blocks, text, unfurl_links: false };
|
||||
if (threadTs) payload.thread_ts = threadTs;
|
||||
const resp = await fetch(SLACK_API, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(payload),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.ok) {
|
||||
core.warning(`Slack API error (channel ${channel}): ${JSON.stringify(data)}`);
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function cell(text) {
|
||||
const s = String(text);
|
||||
return { type: 'raw_text', text: s || ' ' };
|
||||
}
|
||||
|
||||
function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
|
||||
const b = summary.baseline.stats;
|
||||
const f = summary.feature.stats;
|
||||
const c = summary.changes;
|
||||
|
||||
const sigEmoji = { good: '\u2705', bad: '\u274c', neutral: '\u26aa' };
|
||||
|
||||
function fmtMs(v) { return v.toFixed(2) + 'ms'; }
|
||||
function fmtMgas(v) { return v.toFixed(2); }
|
||||
function fmtChange(ch) {
|
||||
if (!ch.pct && !ch.ci_pct) return ' ';
|
||||
const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
|
||||
const ciStr = ch.ci_pct ? ` (\u00b1${ch.ci_pct.toFixed(2)}%)` : '';
|
||||
return `${pctStr}${ciStr} ${sigEmoji[ch.sig]}`;
|
||||
}
|
||||
|
||||
// Overall result for header
|
||||
const vals = Object.values(c);
|
||||
const hasBad = vals.some(v => v.sig === 'bad');
|
||||
const hasGood = vals.some(v => v.sig === 'good');
|
||||
let headerEmoji, headerResult;
|
||||
if (hasBad && hasGood) {
|
||||
headerEmoji = ':warning:';
|
||||
headerResult = 'Mixed Results';
|
||||
} else if (hasBad) {
|
||||
headerEmoji = ':x:';
|
||||
headerResult = 'Regression';
|
||||
} else if (hasGood) {
|
||||
headerEmoji = ':white_check_mark:';
|
||||
headerResult = 'Improvement';
|
||||
} else {
|
||||
headerEmoji = ':white_circle:';
|
||||
headerResult = 'No Difference';
|
||||
}
|
||||
|
||||
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
|
||||
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
|
||||
|
||||
// Meta line
|
||||
const metaParts = [];
|
||||
if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`);
|
||||
metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`);
|
||||
|
||||
// Baseline/feature lines with samply profile links
|
||||
let baselineLine = `*Baseline:* ${baselineLink}`;
|
||||
const bl1 = samplyUrls['baseline-1'];
|
||||
const bl2 = samplyUrls['baseline-2'];
|
||||
if (bl1) baselineLine += ` | <${bl1}|Samply 1>`;
|
||||
if (bl2) baselineLine += ` | <${bl2}|Samply 2>`;
|
||||
|
||||
let featureLine = `*Feature:* ${featureLink}`;
|
||||
const fl1 = samplyUrls['feature-1'];
|
||||
const fl2 = samplyUrls['feature-2'];
|
||||
if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
|
||||
if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
|
||||
|
||||
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
|
||||
const cores = process.env.BENCH_CORES || '0';
|
||||
const countsParts = [];
|
||||
if (warmup) countsParts.push(`*Warmup:* ${warmup}`);
|
||||
countsParts.push(`*Blocks:* ${summary.blocks}`);
|
||||
if (cores !== '0') countsParts.push(`*Cores:* ${cores}`);
|
||||
const countsLine = countsParts.join(' | ');
|
||||
|
||||
const sectionText = [metaParts.join(' | '), '', baselineLine, featureLine, countsLine].join('\n');
|
||||
|
||||
// Action buttons
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
const buttons = [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
|
||||
url: diffUrl,
|
||||
action_id: 'diff_button',
|
||||
},
|
||||
];
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `${headerEmoji} ${headerResult}`, emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: sectionText },
|
||||
},
|
||||
{
|
||||
type: 'table',
|
||||
column_settings: [
|
||||
{ align: 'left' },
|
||||
{ align: 'right' },
|
||||
{ align: 'right' },
|
||||
{ align: 'right' },
|
||||
],
|
||||
rows: [
|
||||
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
|
||||
[cell('Mean'), cell(fmtMs(b.mean_ms)), cell(fmtMs(f.mean_ms)), cell(fmtChange(c.mean))],
|
||||
[cell('StdDev'), cell(fmtMs(b.stddev_ms)), cell(fmtMs(f.stddev_ms)), cell(' ')],
|
||||
[cell('P50'), cell(fmtMs(b.p50_ms)), cell(fmtMs(f.p50_ms)), cell(fmtChange(c.p50))],
|
||||
[cell('P90'), cell(fmtMs(b.p90_ms)), cell(fmtMs(f.p90_ms)), cell(fmtChange(c.p90))],
|
||||
[cell('P99'), cell(fmtMs(b.p99_ms)), cell(fmtMs(f.p99_ms)), cell(fmtChange(c.p99))],
|
||||
[cell('Mgas/s'), cell(fmtMgas(b.mean_mgas_s)), cell(fmtMgas(f.mean_mgas_s)), cell(fmtChange(c.mgas_s))],
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: buttons,
|
||||
},
|
||||
];
|
||||
|
||||
// Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
|
||||
const threadBlocks = [];
|
||||
const waitTimes = summary.wait_times || {};
|
||||
const waitKeys = Object.keys(waitTimes);
|
||||
if (waitKeys.length > 0) {
|
||||
const waitRows = [
|
||||
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
|
||||
];
|
||||
for (const key of waitKeys) {
|
||||
const wt = waitTimes[key];
|
||||
waitRows.push([cell(wt.title), cell(fmtMs(wt.baseline.mean_ms)), cell(fmtMs(wt.feature.mean_ms))]);
|
||||
}
|
||||
threadBlocks.push({
|
||||
type: 'table',
|
||||
column_settings: [
|
||||
{ align: 'left' },
|
||||
{ align: 'right' },
|
||||
{ align: 'right' },
|
||||
],
|
||||
rows: waitRows,
|
||||
});
|
||||
}
|
||||
|
||||
return { blocks, threadBlocks };
|
||||
}
|
||||
|
||||
function buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep }) {
|
||||
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
|
||||
const actorMention = actorSlackId ? `<@${actorSlackId}>` : `@${actor}`;
|
||||
const parts = [
|
||||
prNumber ? `*<${prUrl}|PR #${prNumber}>*` : '',
|
||||
`by ${actorMention}`,
|
||||
`failed while *${failedStep}*`,
|
||||
].filter(Boolean);
|
||||
|
||||
const buttons = [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
];
|
||||
|
||||
return [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: ':rotating_light: Bench Failed', emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: parts.join(' | ') },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: buttons,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function success({ core, context }) {
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
if (!token) {
|
||||
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
core.warning('Could not read summary.json for Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const prNumber = process.env.BENCH_PR;
|
||||
const actor = process.env.BENCH_ACTOR;
|
||||
const jobUrl = process.env.BENCH_JOB_URL ||
|
||||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
// Load samply profile URLs (files exist when samply profiling was enabled)
|
||||
const samplyUrls = {};
|
||||
for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
|
||||
try {
|
||||
const url = fs.readFileSync(
|
||||
path.join(process.env.BENCH_WORK_DIR, run, 'samply-profile-url.txt'), 'utf8'
|
||||
).trim();
|
||||
if (url) samplyUrls[run] = url;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
|
||||
const actorSlackId = slackUsers[actor];
|
||||
|
||||
const { blocks, threadBlocks } = buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls });
|
||||
const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`;
|
||||
|
||||
async function sendWithThread(ch) {
|
||||
const res = await postToSlack(token, ch, blocks, text, core);
|
||||
if (res.ok && res.ts && threadBlocks.length > 0) {
|
||||
for (const tb of threadBlocks) {
|
||||
await postToSlack(token, ch, [tb], 'Wait time breakdown', core, res.ts);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Post to public channel if any metric shows significant improvement or regression
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
let postedToChannel = false;
|
||||
if (channel) {
|
||||
const changes = summary.changes || {};
|
||||
const hasImprovement = Object.values(changes).some(c => c.sig === 'good');
|
||||
if (hasImprovement) {
|
||||
await sendWithThread(channel);
|
||||
postedToChannel = true;
|
||||
} else {
|
||||
core.info('No significant improvement, skipping public channel notification');
|
||||
}
|
||||
}
|
||||
|
||||
// DM the actor only when results were not posted to the public channel
|
||||
if (!postedToChannel) {
|
||||
if (actorSlackId) {
|
||||
await sendWithThread(actorSlackId);
|
||||
} else {
|
||||
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
|
||||
}
|
||||
} else {
|
||||
core.info(`Results posted to channel, skipping DM to ${actor}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function failure({ core, context, failedStep }) {
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
if (!token) {
|
||||
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const prNumber = process.env.BENCH_PR;
|
||||
const actor = process.env.BENCH_ACTOR;
|
||||
const jobUrl = process.env.BENCH_JOB_URL ||
|
||||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
|
||||
const actorSlackId = slackUsers[actor];
|
||||
|
||||
const blocks = buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep });
|
||||
const text = `Bench failed while ${failedStep}`;
|
||||
|
||||
// Always DM the actor
|
||||
if (actorSlackId) {
|
||||
await postToSlack(token, actorSlackId, blocks, text, core);
|
||||
} else {
|
||||
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
|
||||
}
|
||||
|
||||
// Only DM for failures, don't post to public channel
|
||||
}
|
||||
|
||||
module.exports = { success, failure };
|
||||
13
.github/scripts/bench-slack-users.json
vendored
Normal file
13
.github/scripts/bench-slack-users.json
vendored
Normal file
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"_comment": "Maps GitHub usernames to Slack user IDs. Find yours: Slack profile > ··· > Copy member ID.",
|
||||
"shekhirin": "U09FAL2UMLJ",
|
||||
"mattsse": "U09FQNPMRT3",
|
||||
"klkvr": "U09FAK95FC2",
|
||||
"joshieDo": "U09LHN6GYAU",
|
||||
"mediocregopher": "U09FF75KMQU",
|
||||
"yongkangc": "U09FB0ECTD4",
|
||||
"gakonst": "U092SEPDM40",
|
||||
"Rjected": "U09F6SCKRGT",
|
||||
"DaniPopes": "U09FAT8EK2A",
|
||||
"emmajam": "U0A34UN92HW"
|
||||
}
|
||||
27
.github/scripts/bench-update-status.js
vendored
Normal file
27
.github/scripts/bench-update-status.js
vendored
Normal file
@@ -0,0 +1,27 @@
|
||||
// Updates the reth-bench PR comment with current status.
|
||||
//
|
||||
// Reads from environment:
|
||||
// BENCH_COMMENT_ID – GitHub comment ID to update
|
||||
// BENCH_JOB_URL – URL to the Actions job page
|
||||
// BENCH_CONFIG – Config line (blocks, warmup, refs)
|
||||
// BENCH_ACTOR – User who triggered the benchmark
|
||||
//
|
||||
// Usage from actions/github-script:
|
||||
// const s = require('./.github/scripts/bench-update-status.js');
|
||||
// await s({github, context, status: 'Building baseline binary...'});
|
||||
|
||||
function buildBody(status) {
|
||||
return `cc @${process.env.BENCH_ACTOR}\n\n🚀 Benchmark started! [View job](${process.env.BENCH_JOB_URL})\n\n⏳ **Status:** ${status}\n\n${process.env.BENCH_CONFIG}`;
|
||||
}
|
||||
|
||||
async function updateStatus({ github, context, status }) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
|
||||
body: buildBody(status),
|
||||
});
|
||||
}
|
||||
|
||||
updateStatus.buildBody = buildBody;
|
||||
module.exports = updateStatus;
|
||||
28
.github/scripts/hive/expected_failures.yaml
vendored
28
.github/scripts/hive/expected_failures.yaml
vendored
@@ -59,10 +59,6 @@ engine-auth: [ ]
|
||||
#
|
||||
# System contract tests (already fixed and deployed):
|
||||
#
|
||||
# tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout and test_invalid_log_length
|
||||
# System contract is already fixed and deployed; tests cover scenarios where contract is
|
||||
# malformed which can't happen retroactively. No point in adding checks.
|
||||
#
|
||||
# tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment
|
||||
# tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment
|
||||
# Post-fork system contract deployment tests. Should fix for spec compliance but not realistic
|
||||
@@ -71,32 +67,8 @@ eels/consume-engine:
|
||||
- tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test_engine-zero_nonce]-reth
|
||||
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
|
||||
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
|
||||
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
|
||||
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_False]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_True]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
|
||||
11
.github/scripts/hive/ignored_tests.yaml
vendored
11
.github/scripts/hive/ignored_tests.yaml
vendored
@@ -11,17 +11,16 @@
|
||||
#
|
||||
# When a test should no longer be ignored, remove it from this list.
|
||||
|
||||
# flaky
|
||||
engine-withdrawals:
|
||||
- Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
|
||||
- Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
|
||||
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
|
||||
- Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
|
||||
engine-cancun:
|
||||
- Transaction Re-Org, New Payload on Revert Back (Cancun) (reth)
|
||||
- Transaction Re-Org, Re-Org to Different Block (Cancun) (reth)
|
||||
- Transaction Re-Org, Re-Org Out (Cancun) (reth)
|
||||
- Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth)
|
||||
# Hive test infra bug: geth sidecar switched to PathScheme for state storage, which has
|
||||
# strict trie integrity requirements incompatible with inserting intentionally invalid blocks.
|
||||
# Affects all clients, not just reth. Tracked: https://github.com/ethereum/hive/issues/1382
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
|
||||
- Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth)
|
||||
engine-api:
|
||||
- Transaction Re-Org, Re-Org Out (Paris) (reth)
|
||||
|
||||
8
.github/scripts/hive/run_simulator.sh
vendored
8
.github/scripts/hive/run_simulator.sh
vendored
@@ -6,8 +6,14 @@ cd hivetests/
|
||||
sim="${1}"
|
||||
limit="${2}"
|
||||
|
||||
# Use lower parallelism for eels tests to avoid OOM-killing the runner
|
||||
parallelism=16
|
||||
if [[ "${sim}" == *"eels"* ]]; then
|
||||
parallelism=4
|
||||
fi
|
||||
|
||||
run_hive() {
|
||||
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 16 --client reth 2>&1 | tee /tmp/log || true
|
||||
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism "${parallelism}" --client reth 2>&1 | tee /tmp/log || true
|
||||
}
|
||||
|
||||
check_log() {
|
||||
|
||||
1015
.github/workflows/bench.yml
vendored
1015
.github/workflows/bench.yml
vendored
File diff suppressed because it is too large
Load Diff
2
.github/workflows/book.yml
vendored
2
.github/workflows/book.yml
vendored
@@ -15,7 +15,7 @@ env:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: depot-ubuntu-latest-8
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
2
.github/workflows/check-alloy.yml
vendored
2
.github/workflows/check-alloy.yml
vendored
@@ -25,7 +25,7 @@ env:
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
2
.github/workflows/compact.yml
vendored
2
.github/workflows/compact.yml
vendored
@@ -18,7 +18,7 @@ env:
|
||||
name: compact-codec
|
||||
jobs:
|
||||
compact-codec:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
matrix:
|
||||
bin:
|
||||
|
||||
33
.github/workflows/docker-test.yml
vendored
33
.github/workflows/docker-test.yml
vendored
@@ -15,7 +15,6 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -31,10 +30,22 @@ jobs:
|
||||
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Detect fork
|
||||
id: fork
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Depot build (upstream only)
|
||||
- name: Set up Depot CLI
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/setup-action@v1
|
||||
|
||||
- name: Build reth image
|
||||
- name: Build reth image (Depot)
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/bake-action@v1
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
@@ -46,6 +57,24 @@ jobs:
|
||||
targets: ${{ inputs.hive_target }}
|
||||
push: false
|
||||
|
||||
# Docker build (forks)
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build reth image (Docker)
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/bake-action@v6
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
with:
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ inputs.hive_target }}
|
||||
push: false
|
||||
set: |
|
||||
*.dockerfile=Dockerfile
|
||||
|
||||
- name: Upload reth image
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
||||
27
.github/workflows/docker.yml
vendored
27
.github/workflows/docker.yml
vendored
@@ -70,18 +70,27 @@ jobs:
|
||||
# Add 'latest' tag for non-RC releases
|
||||
if [[ ! "$VERSION" =~ -rc ]]; then
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "ethereum_set<<EOF"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:${VERSION}"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:latest"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
# git-sha build
|
||||
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
@@ -97,7 +106,7 @@ jobs:
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
${{ steps.params.outputs.ethereum_set }}
|
||||
|
||||
- name: Verify image architectures
|
||||
env:
|
||||
@@ -117,6 +126,18 @@ jobs:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
SLACK_COLOR: danger
|
||||
SLACK_ICON_EMOJI: ":rotating_light:"
|
||||
SLACK_USERNAME: "GitHub Actions"
|
||||
SLACK_TITLE: ":rotating_light: Nightly Docker Build Failed"
|
||||
SLACK_MESSAGE: |
|
||||
The scheduled nightly Docker build failed.
|
||||
|
||||
*Commit:* `${{ github.sha }}`
|
||||
*Branch:* `${{ github.ref_name }}`
|
||||
*Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
|
||||
|
||||
*Action required:* Re-run the workflow or investigate the build failure.
|
||||
SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
|
||||
MSG_MINIMAL: true
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
4
.github/workflows/e2e.yml
vendored
4
.github/workflows/e2e.yml
vendored
@@ -20,7 +20,7 @@ concurrency:
|
||||
jobs:
|
||||
test:
|
||||
name: e2e-testsuite
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 90
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
|
||||
5
.github/workflows/hive.yml
vendored
5
.github/workflows/hive.yml
vendored
@@ -32,7 +32,7 @@ jobs:
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Checkout hive tests
|
||||
@@ -188,7 +188,8 @@ jobs:
|
||||
- build-reth-edge
|
||||
- prepare-hive
|
||||
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
|
||||
2
.github/workflows/integration.yml
vendored
2
.github/workflows/integration.yml
vendored
@@ -24,7 +24,7 @@ jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
|
||||
2
.github/workflows/kurtosis.yml
vendored
2
.github/workflows/kurtosis.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
name: run kurtosis
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
needs:
|
||||
- build-reth
|
||||
steps:
|
||||
|
||||
26
.github/workflows/lint.yml
vendored
26
.github/workflows/lint.yml
vendored
@@ -13,7 +13,7 @@ env:
|
||||
jobs:
|
||||
clippy-binaries:
|
||||
name: clippy binaries / ${{ matrix.type }}
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
|
||||
clippy:
|
||||
name: clippy
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
RUSTFLAGS: -D warnings
|
||||
|
||||
wasm:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
.github/scripts/check_wasm.sh
|
||||
|
||||
riscv:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2, 3]
|
||||
@@ -117,14 +117,14 @@ jobs:
|
||||
|
||||
msrv:
|
||||
name: MSRV
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: "1.88" # MSRV
|
||||
toolchain: "1.93" # MSRV
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
|
||||
docs:
|
||||
name: docs
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
|
||||
fmt:
|
||||
name: fmt
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -167,7 +167,7 @@ jobs:
|
||||
|
||||
udeps:
|
||||
name: udeps
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -182,7 +182,7 @@ jobs:
|
||||
|
||||
book:
|
||||
name: book
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -240,7 +240,7 @@ jobs:
|
||||
# Checks that selected crates can compile with power set of features
|
||||
features:
|
||||
name: features
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -264,7 +264,7 @@ jobs:
|
||||
|
||||
# Check crates correctly propagate features
|
||||
feature-propagation:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
8
.github/workflows/release.yml
vendored
8
.github/workflows/release.yml
vendored
@@ -74,7 +74,7 @@ jobs:
|
||||
profile: maxperf
|
||||
allow_fail: false
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
os: ubuntu-24.04
|
||||
os: ubuntu-24.04-arm
|
||||
profile: maxperf
|
||||
allow_fail: false
|
||||
- target: x86_64-apple-darwin
|
||||
@@ -85,10 +85,6 @@ jobs:
|
||||
os: macos-14
|
||||
profile: maxperf
|
||||
allow_fail: false
|
||||
- target: riscv64gc-unknown-linux-gnu
|
||||
os: ubuntu-24.04
|
||||
profile: maxperf
|
||||
allow_fail: true
|
||||
build:
|
||||
- command: build
|
||||
binary: reth
|
||||
@@ -102,7 +98,7 @@ jobs:
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
cargo install cross --locked --git https://github.com/cross-rs/cross
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
4
.github/workflows/stage.yml
vendored
4
.github/workflows/stage.yml
vendored
@@ -23,7 +23,7 @@ jobs:
|
||||
name: stage-run-test
|
||||
# Only run stage commands test in merge groups
|
||||
if: github.event_name == 'merge_group'
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
cache-on-failure: true
|
||||
- name: Build reth
|
||||
run: |
|
||||
cargo install --path bin/reth
|
||||
cargo install --locked --path bin/reth
|
||||
- name: Run headers stage
|
||||
run: |
|
||||
reth stage run headers --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
|
||||
|
||||
2
.github/workflows/sync-era.yml
vendored
2
.github/workflows/sync-era.yml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
2
.github/workflows/sync.yml
vendored
2
.github/workflows/sync.yml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
6
.github/workflows/unit.yml
vendored
6
.github/workflows/unit.yml
vendored
@@ -20,7 +20,7 @@ concurrency:
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.type }} / ${{ matrix.storage }}
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
state:
|
||||
name: Ethereum state tests
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
|
||||
doc:
|
||||
name: doc tests
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
|
||||
162
CLAUDE.md
162
CLAUDE.md
@@ -172,10 +172,97 @@ Before submitting changes, ensure:
|
||||
2. **Clippy**: No warnings
|
||||
3. **Tests Pass**: All unit and integration tests
|
||||
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
|
||||
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
|
||||
5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
|
||||
6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
|
||||
|
||||
### CLI Reference Docs (`book` CI Job)
|
||||
|
||||
The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
|
||||
|
||||
When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
|
||||
|
||||
```bash
|
||||
make update-book-cli
|
||||
```
|
||||
|
||||
This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
|
||||
|
||||
The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
|
||||
|
||||
### Opening PRs against <https://github.com/paradigmxyz/reth>
|
||||
|
||||
#### Titles
|
||||
|
||||
Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
|
||||
|
||||
```
|
||||
<type>(<scope>): <short description>
|
||||
```
|
||||
|
||||
**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
|
||||
|
||||
**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
|
||||
|
||||
Examples:
|
||||
- `fix(rpc): correct gas estimation for ERC-20 transfers`
|
||||
- `perf: batch trie updates to reduce cursor overhead`
|
||||
- `feat(engine): add new_payload_interval metric`
|
||||
|
||||
#### Descriptions
|
||||
|
||||
Keep it short. Say what changed and why — nothing more.
|
||||
|
||||
**Do:**
|
||||
- Write 1–3 sentences summarizing the change
|
||||
- Explain _why_ if the diff doesn't make it obvious
|
||||
- Link related issues or EIPs
|
||||
- Include benchmark numbers for perf changes
|
||||
|
||||
**Don't:**
|
||||
- List every file changed — that's what the diff is for
|
||||
- Repeat the title in the body
|
||||
- Add "Files changed" or "Changes" sections
|
||||
- Write walls of text that go stale when the diff is updated
|
||||
- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
|
||||
|
||||
**Template:**
|
||||
|
||||
```
|
||||
Closes #<issue>
|
||||
|
||||
<what changed, 1-3 sentences>
|
||||
|
||||
<why, if not obvious from the diff>
|
||||
```
|
||||
|
||||
**Good example:**
|
||||
|
||||
```
|
||||
Closes #16800
|
||||
|
||||
Adds fallback for external IP resolution so node startup doesn't fail
|
||||
when STUN is unreachable. Falls back to the configured default.
|
||||
```
|
||||
|
||||
**Bad example:**
|
||||
|
||||
```
|
||||
## Summary
|
||||
This PR introduces comprehensive improvements to the IP resolution system.
|
||||
|
||||
## Changes
|
||||
- Modified `crates/net/discv4/src/lib.rs` to add fallback
|
||||
- Modified `crates/net/discv4/src/config.rs` to add default IP
|
||||
- Added tests in `crates/net/discv4/src/tests/ip.rs`
|
||||
|
||||
## Files Changed
|
||||
- crates/net/discv4/src/lib.rs
|
||||
- crates/net/discv4/src/config.rs
|
||||
- crates/net/discv4/src/tests/ip.rs
|
||||
```
|
||||
|
||||
#### Labels and CI
|
||||
|
||||
Label PRs appropriately, first check the available labels and then apply the relevant ones:
|
||||
* when changes are RPC related, add A-rpc label
|
||||
* when changes are docs related, add C-docs label
|
||||
@@ -313,6 +400,74 @@ GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst);
|
||||
Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful?
|
||||
|
||||
|
||||
#### Rust Style Guides
|
||||
|
||||
##### Type Ordering in Files
|
||||
|
||||
When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers.
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
/// The primary type of this file (matches filename).
|
||||
pub struct PayloadProcessor { ... }
|
||||
|
||||
impl PayloadProcessor { ... }
|
||||
|
||||
// Followed by public auxiliary types that support the primary type
|
||||
|
||||
/// Configuration for the processor.
|
||||
pub struct PayloadProcessorConfig { ... }
|
||||
|
||||
/// Result type returned by processor operations.
|
||||
pub struct ProcessorResult { ... }
|
||||
|
||||
// Followed by public traits related to the primary type
|
||||
|
||||
pub trait ProcessorExt { ... }
|
||||
|
||||
// Followed by private helper types
|
||||
|
||||
struct InternalState { ... }
|
||||
|
||||
// Followed by private helper functions
|
||||
|
||||
fn validate_input() { ... }
|
||||
```
|
||||
|
||||
❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)):
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
// ❌ BAD - new auxiliary struct added before the file's main type
|
||||
pub struct CacheWaitDurations { ... }
|
||||
|
||||
// ❌ BAD - new trait added before the file's main type
|
||||
pub trait WaitForCaches { ... }
|
||||
|
||||
// The file's primary type is buried below unrelated additions
|
||||
pub struct PayloadProcessor { ... }
|
||||
```
|
||||
|
||||
✅ **Good**: New types go **after** the primary type:
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
// ✅ The file's primary type stays at the top
|
||||
pub struct PayloadProcessor { ... }
|
||||
|
||||
impl PayloadProcessor { ... }
|
||||
|
||||
// ✅ Auxiliary types follow the primary type
|
||||
pub struct CacheWaitDurations { ... }
|
||||
|
||||
pub trait WaitForCaches { ... }
|
||||
|
||||
impl WaitForCaches for PayloadProcessor { ... }
|
||||
```
|
||||
|
||||
### Example Contribution Workflow
|
||||
|
||||
Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
@@ -387,5 +542,8 @@ cargo build --release
|
||||
cargo check --workspace --all-features
|
||||
|
||||
# Check documentation
|
||||
cargo docs --document-private-items
|
||||
cargo docs --document-private-items
|
||||
|
||||
# Regenerate CLI reference docs (after CLI changes)
|
||||
make update-book-cli
|
||||
```
|
||||
|
||||
1170
Cargo.lock
generated
1170
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
69
Cargo.toml
69
Cargo.toml
@@ -1,7 +1,7 @@
|
||||
[workspace.package]
|
||||
version = "1.10.2"
|
||||
version = "1.11.1"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
homepage = "https://paradigmxyz.github.io/reth"
|
||||
repository = "https://github.com/paradigmxyz/reth"
|
||||
@@ -27,7 +27,6 @@ members = [
|
||||
"crates/engine/invalid-block-hooks/",
|
||||
"crates/engine/local",
|
||||
"crates/engine/primitives/",
|
||||
"crates/engine/service",
|
||||
"crates/engine/tree/",
|
||||
"crates/engine/util/",
|
||||
"crates/era",
|
||||
@@ -139,6 +138,7 @@ members = [
|
||||
"examples/exex-subscription",
|
||||
"examples/exex-test",
|
||||
"examples/full-contract-state",
|
||||
"examples/migrate-trie-to-packed",
|
||||
"examples/manual-p2p/",
|
||||
"examples/network-txpool/",
|
||||
"examples/network/",
|
||||
@@ -349,7 +349,6 @@ reth-ecies = { path = "crates/net/ecies" }
|
||||
reth-engine-local = { path = "crates/engine/local" }
|
||||
reth-engine-primitives = { path = "crates/engine/primitives", default-features = false }
|
||||
reth-engine-tree = { path = "crates/engine/tree" }
|
||||
reth-engine-service = { path = "crates/engine/service" }
|
||||
reth-engine-util = { path = "crates/engine/util" }
|
||||
reth-era = { path = "crates/era" }
|
||||
reth-era-downloader = { path = "crates/era-downloader" }
|
||||
@@ -399,7 +398,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives = { path = "crates/primitives", default-features = false }
|
||||
reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
|
||||
reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
@@ -461,33 +460,33 @@ alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.6.3", default-features = false }
|
||||
alloy-contract = { version = "1.6.3", default-features = false }
|
||||
alloy-eips = { version = "1.6.3", default-features = false }
|
||||
alloy-genesis = { version = "1.6.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.6.3", default-features = false }
|
||||
alloy-network = { version = "1.6.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.6.3", default-features = false }
|
||||
alloy-provider = { version = "1.6.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.6.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.6.3", default-features = false }
|
||||
alloy-serde = { version = "1.6.3", default-features = false }
|
||||
alloy-signer = { version = "1.6.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.6.3", default-features = false }
|
||||
alloy-transport = { version = "1.6.3" }
|
||||
alloy-transport-http = { version = "1.6.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.6.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.6.3", default-features = false }
|
||||
alloy-consensus = { version = "1.7.3", default-features = false }
|
||||
alloy-contract = { version = "1.7.3", default-features = false }
|
||||
alloy-eips = { version = "1.7.3", default-features = false }
|
||||
alloy-genesis = { version = "1.7.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.7.3", default-features = false }
|
||||
alloy-network = { version = "1.7.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.7.3", default-features = false }
|
||||
alloy-provider = { version = "1.7.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.7.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.7.3", default-features = false }
|
||||
alloy-serde = { version = "1.7.3", default-features = false }
|
||||
alloy-signer = { version = "1.7.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.7.3", default-features = false }
|
||||
alloy-transport = { version = "1.7.3" }
|
||||
alloy-transport-http = { version = "1.7.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.7.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.7.3", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.27.2", default-features = false }
|
||||
@@ -530,16 +529,17 @@ notify = { version = "8.0.0", default-features = false, features = ["macos_fseve
|
||||
nybbles = { version = "0.4.8", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
parking_lot = "0.12"
|
||||
quanta = "0.12"
|
||||
paste = "1.0"
|
||||
rand = "0.9"
|
||||
rayon = "1.7"
|
||||
thread-priority = "3.0.0"
|
||||
rustc-hash = { version = "2.0", default-features = false }
|
||||
schnellru = "0.2"
|
||||
serde = { version = "1.0", default-features = false }
|
||||
serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros"] }
|
||||
sha2 = { version = "0.10", default-features = false }
|
||||
shellexpand = "3.0.0"
|
||||
shlex = "1.3"
|
||||
smallvec = "1"
|
||||
strum = { version = "0.27", default-features = false }
|
||||
@@ -652,7 +652,7 @@ ethereum_ssz_derive = "0.10.1"
|
||||
jemalloc_pprof = { version = "0.8", default-features = false }
|
||||
tikv-jemalloc-ctl = "0.6"
|
||||
tikv-jemallocator = "0.6"
|
||||
tracy-client = "0.18.0"
|
||||
tracy-client = { version = "0.18.0", features = ["demangle"] }
|
||||
snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
|
||||
|
||||
aes = "0.8.1"
|
||||
@@ -665,6 +665,7 @@ cipher = "0.4.3"
|
||||
comfy-table = "7.0"
|
||||
concat-kdf = "0.1.0"
|
||||
crossbeam-channel = "0.5.13"
|
||||
crossbeam-utils = "0.8"
|
||||
crossterm = "0.29.0"
|
||||
csv = "1.3.0"
|
||||
ctrlc = "3.4"
|
||||
|
||||
@@ -19,10 +19,11 @@ pre-build = [
|
||||
image = "ubuntu:24.04"
|
||||
pre-build = [
|
||||
"apt update",
|
||||
"apt install --yes gcc gcc-riscv64-linux-gnu libclang-dev make",
|
||||
"apt install --yes gcc gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libclang-dev make",
|
||||
]
|
||||
env.passthrough = [
|
||||
"CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER=riscv64-linux-gnu-gcc",
|
||||
"CXX_riscv64gc_unknown_linux_gnu=riscv64-linux-gnu-g++",
|
||||
]
|
||||
|
||||
[build.env]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker.io/docker/dockerfile:1.7-labs
|
||||
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1.93 AS chef
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Usage:
|
||||
# reth: --build-arg BINARY=reth
|
||||
|
||||
FROM rust:1 AS builder
|
||||
FROM rust:1.93 AS builder
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
|
||||
4
Makefile
4
Makefile
@@ -80,7 +80,7 @@ build-native-%:
|
||||
#
|
||||
# These commands require that:
|
||||
#
|
||||
# - `cross` is installed (`cargo install cross`).
|
||||
# - `cross` is installed (`cargo install --locked cross`).
|
||||
# - Docker is running.
|
||||
# - The current user is in the `docker` group.
|
||||
#
|
||||
@@ -261,7 +261,7 @@ lint-typos: ensure-typos
|
||||
|
||||
ensure-typos:
|
||||
@if ! command -v typos &> /dev/null; then \
|
||||
echo "typos not found. Please install it by running the command 'cargo install typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
|
||||
echo "typos not found. Please install it by running the command 'cargo install --locked typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ When updating this, also update:
|
||||
- .github/workflows/lint.yml
|
||||
-->
|
||||
|
||||
The Minimum Supported Rust Version (MSRV) of this project is [1.88.0](https://blog.rust-lang.org/2025/06/26/Rust-1.88.0/).
|
||||
The Minimum Supported Rust Version (MSRV) of this project is [1.93.0](https://blog.rust-lang.org/2026/01/22/Rust-1.93.0/).
|
||||
|
||||
See the docs for detailed instructions on how to [build from source](https://reth.rs/installation/source/).
|
||||
|
||||
|
||||
@@ -45,9 +45,6 @@ serde_json.workspace = true
|
||||
# Time handling
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
|
||||
# Path manipulation
|
||||
shellexpand.workspace = true
|
||||
|
||||
# CSV handling
|
||||
csv.workspace = true
|
||||
|
||||
|
||||
@@ -289,11 +289,7 @@ impl Args {
|
||||
/// Get the JWT secret path - either provided or derived from datadir
|
||||
pub(crate) fn jwt_secret_path(&self) -> PathBuf {
|
||||
match &self.jwt_secret {
|
||||
Some(path) => {
|
||||
let jwt_secret_str = path.to_string_lossy();
|
||||
let expanded = shellexpand::tilde(&jwt_secret_str);
|
||||
PathBuf::from(expanded.as_ref())
|
||||
}
|
||||
Some(path) => path.clone(),
|
||||
None => {
|
||||
// Use the same logic as reth: <datadir>/<chain>/jwt.hex
|
||||
let chain_path = self.datadir.clone().resolve_datadir(self.chain);
|
||||
@@ -308,10 +304,9 @@ impl Args {
|
||||
chain_path.data_dir().to_path_buf()
|
||||
}
|
||||
|
||||
/// Get the expanded output directory path
|
||||
/// Get the output directory path
|
||||
pub(crate) fn output_dir_path(&self) -> PathBuf {
|
||||
let expanded = shellexpand::tilde(&self.output_dir);
|
||||
PathBuf::from(expanded.as_ref())
|
||||
PathBuf::from(&self.output_dir)
|
||||
}
|
||||
|
||||
/// Get the effective warmup blocks value - either specified or defaults to blocks
|
||||
|
||||
@@ -29,6 +29,10 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) next_block: u64,
|
||||
/// Whether the chain is an OP rollup.
|
||||
pub(crate) is_optimism: bool,
|
||||
/// Whether to use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
pub(crate) use_reth_namespace: bool,
|
||||
/// Whether to fetch and replay RLP-encoded blocks.
|
||||
pub(crate) rlp_blocks: bool,
|
||||
}
|
||||
|
||||
impl BenchContext {
|
||||
@@ -140,6 +144,16 @@ impl BenchContext {
|
||||
};
|
||||
|
||||
let next_block = first_block.header.number + 1;
|
||||
Ok(Self { auth_provider, block_provider, benchmark_mode, next_block, is_optimism })
|
||||
let rlp_blocks = bench_args.rlp_blocks;
|
||||
let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
is_optimism,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,9 @@ use crate::{
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
|
||||
valid_payload::{
|
||||
call_forkchoice_updated_with_reth, call_new_payload_with_reth, payload_to_new_payload,
|
||||
},
|
||||
};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -19,6 +21,7 @@ use reth_chainspec::ChainSpec;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
|
||||
use reth_rpc_api::RethNewPayloadInput;
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
@@ -47,6 +50,14 @@ pub struct Command {
|
||||
/// Output directory for benchmark results and generated payloads.
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: PathBuf,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
/// directly, waits for persistence and cache updates to complete before processing,
|
||||
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
@@ -138,6 +149,9 @@ impl Command {
|
||||
);
|
||||
}
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
}
|
||||
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
@@ -163,7 +177,7 @@ impl Command {
|
||||
// Regenerate the payload from the modified block, but keep the original sidecar
|
||||
// which contains the actual execution requests data (not just the hash)
|
||||
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
|
||||
let (version, params) = payload_to_new_payload(
|
||||
let (version, params, execution_data) = payload_to_new_payload(
|
||||
payload,
|
||||
sidecar,
|
||||
false,
|
||||
@@ -171,27 +185,45 @@ impl Command {
|
||||
Some(new_payload_version),
|
||||
)?;
|
||||
|
||||
let (version, params) = if self.reth_new_payload {
|
||||
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?)
|
||||
} else {
|
||||
(Some(version), params)
|
||||
};
|
||||
|
||||
// Save payload to file with version info for replay
|
||||
let payload_path =
|
||||
self.output.join(format!("payload_block_{}.json", block.header.number));
|
||||
let file =
|
||||
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
|
||||
let file = GasRampPayloadFile {
|
||||
version: version.map(|v| v as u8),
|
||||
block_hash,
|
||||
params: params.clone(),
|
||||
};
|
||||
let payload_json = serde_json::to_string_pretty(&file)?;
|
||||
std::fs::write(&payload_path, &payload_json)?;
|
||||
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
|
||||
|
||||
call_new_payload(&provider, version, params).await?;
|
||||
let _ = call_new_payload_with_reth(&provider, version, params).await?;
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: block_hash,
|
||||
finalized_block_hash: block_hash,
|
||||
};
|
||||
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
|
||||
call_forkchoice_updated_with_reth(&provider, version, forkchoice_state).await?;
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
blocks_processed += 1;
|
||||
|
||||
let progress = match mode {
|
||||
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
|
||||
format!("{pct:.1}%")
|
||||
}
|
||||
};
|
||||
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
|
||||
156
bin/reth-bench/src/bench/metrics_scraper.rs
Normal file
156
bin/reth-bench/src/bench/metrics_scraper.rs
Normal file
@@ -0,0 +1,156 @@
|
||||
//! Prometheus metrics scraper for reth-bench.
|
||||
//!
|
||||
//! Scrapes a node's Prometheus metrics endpoint after each block to record
|
||||
//! execution and state root durations with block-level granularity.
|
||||
|
||||
use csv::Writer;
|
||||
use eyre::Context;
|
||||
use reqwest::Client;
|
||||
use serde::Serialize;
|
||||
use std::{path::Path, time::Duration};
|
||||
use tracing::info;
|
||||
|
||||
/// Suffix for the metrics CSV output file.
|
||||
pub(crate) const METRICS_OUTPUT_SUFFIX: &str = "metrics.csv";
|
||||
|
||||
/// A single row of scraped prometheus metrics for one block.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct MetricsRow {
|
||||
/// The block number.
|
||||
pub(crate) block_number: u64,
|
||||
/// EVM execution duration in seconds (from `sync_execution_execution_duration` gauge).
|
||||
pub(crate) execution_duration_secs: Option<f64>,
|
||||
/// State root computation duration in seconds (from
|
||||
/// `sync_block_validation_state_root_duration` gauge).
|
||||
pub(crate) state_root_duration_secs: Option<f64>,
|
||||
}
|
||||
|
||||
/// Scrapes a Prometheus metrics endpoint after each block to collect
|
||||
/// execution and state root durations.
|
||||
pub(crate) struct MetricsScraper {
|
||||
/// The full URL of the Prometheus metrics endpoint.
|
||||
url: String,
|
||||
/// Reusable HTTP client.
|
||||
client: Client,
|
||||
/// Collected metrics rows, one per block.
|
||||
rows: Vec<MetricsRow>,
|
||||
}
|
||||
|
||||
impl MetricsScraper {
|
||||
/// Creates a new scraper if a URL is provided.
|
||||
pub(crate) fn maybe_new(url: Option<String>) -> Option<Self> {
|
||||
url.map(|url| {
|
||||
info!(target: "reth-bench", %url, "Prometheus metrics scraping enabled");
|
||||
let client = Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.expect("failed to build reqwest client");
|
||||
Self { url, client, rows: Vec::new() }
|
||||
})
|
||||
}
|
||||
|
||||
/// Scrapes the metrics endpoint and records values for the given block.
|
||||
pub(crate) async fn scrape_after_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
let text = self
|
||||
.client
|
||||
.get(&self.url)
|
||||
.send()
|
||||
.await
|
||||
.wrap_err("failed to fetch metrics endpoint")?
|
||||
.error_for_status()
|
||||
.wrap_err("metrics endpoint returned error status")?
|
||||
.text()
|
||||
.await
|
||||
.wrap_err("failed to read metrics response body")?;
|
||||
|
||||
let execution = parse_gauge(&text, "sync_execution_execution_duration");
|
||||
let state_root = parse_gauge(&text, "sync_block_validation_state_root_duration");
|
||||
|
||||
self.rows.push(MetricsRow {
|
||||
block_number,
|
||||
execution_duration_secs: execution,
|
||||
state_root_duration_secs: state_root,
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Writes collected metrics to a CSV file in the output directory.
|
||||
pub(crate) fn write_csv(&self, output_dir: &Path) -> eyre::Result<()> {
|
||||
let path = output_dir.join(METRICS_OUTPUT_SUFFIX);
|
||||
info!(target: "reth-bench", "Writing scraped metrics to file: {:?}", path);
|
||||
let mut writer = Writer::from_path(&path)?;
|
||||
for row in &self.rows {
|
||||
writer.serialize(row)?;
|
||||
}
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a Prometheus gauge value from exposition-format text.
|
||||
///
|
||||
/// Searches for lines starting with `name` followed by either a space or `{`
|
||||
/// (for labeled metrics), then parses the numeric value. Returns the last
|
||||
/// matching sample to handle metrics emitted with multiple label sets.
|
||||
fn parse_gauge(text: &str, name: &str) -> Option<f64> {
|
||||
let mut result = None;
|
||||
for line in text.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() || line.starts_with('#') {
|
||||
continue;
|
||||
}
|
||||
|
||||
if !line.starts_with(name) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Ensure we match the full metric name, not a prefix of another metric.
|
||||
let rest = &line[name.len()..];
|
||||
if !rest.starts_with(' ') && !rest.starts_with('{') {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Format: `metric_name{labels} value [timestamp]` or `metric_name value [timestamp]`
|
||||
// Value is always the second whitespace-separated token.
|
||||
let mut parts = line.split_whitespace();
|
||||
if let Some(value_str) = parts.nth(1) &&
|
||||
let Ok(v) = value_str.parse::<f64>()
|
||||
{
|
||||
result = Some(v);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_parse_gauge_simple() {
|
||||
let text = r#"# HELP sync_execution_execution_duration Duration of execution
|
||||
# TYPE sync_execution_execution_duration gauge
|
||||
sync_execution_execution_duration 0.123456
|
||||
"#;
|
||||
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), Some(0.123456));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gauge_missing() {
|
||||
let text = "some_other_metric 1.0\n";
|
||||
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gauge_with_labels() {
|
||||
let text = "sync_block_validation_state_root_duration{instance=\"node1\"} 0.5\n";
|
||||
assert_eq!(parse_gauge(text, "sync_block_validation_state_root_duration"), Some(0.5));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_gauge_prefix_no_false_match() {
|
||||
let text =
|
||||
"sync_execution_execution_duration_total 99.0\nsync_execution_execution_duration 0.5\n";
|
||||
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), Some(0.5));
|
||||
}
|
||||
}
|
||||
@@ -12,6 +12,7 @@ pub(crate) mod helpers;
|
||||
pub use generate_big_block::{
|
||||
RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
|
||||
};
|
||||
pub(crate) mod metrics_scraper;
|
||||
mod new_payload_fcu;
|
||||
mod new_payload_only;
|
||||
mod output;
|
||||
|
||||
@@ -13,6 +13,7 @@ use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::parse_duration,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
@@ -20,9 +21,11 @@ use crate::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
|
||||
valid_payload::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
},
|
||||
};
|
||||
use alloy_provider::Provider;
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use clap::Parser;
|
||||
use eyre::{Context, OptionExt};
|
||||
@@ -30,7 +33,7 @@ use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// `reth benchmark new-payload-fcu` command
|
||||
#[derive(Debug, Parser)]
|
||||
@@ -150,9 +153,18 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
|
||||
|
||||
if use_reth_namespace {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
}
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -175,6 +187,21 @@ impl Command {
|
||||
}
|
||||
};
|
||||
|
||||
let rlp = if rlp_blocks {
|
||||
let rlp = match block_provider.debug_get_raw_block(next_block.into()).await {
|
||||
Ok(rlp) => rlp,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
|
||||
let _ = error_sender
|
||||
.send(eyre::eyre!("Failed to fetch raw block {next_block}: {e}"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
Some(rlp)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let head_block_hash = block.header.hash;
|
||||
let safe_block_hash = block_provider
|
||||
.get_block_by_number(block.header.number.saturating_sub(32).into());
|
||||
@@ -196,7 +223,7 @@ impl Command {
|
||||
|
||||
next_block += 1;
|
||||
if let Err(e) = sender
|
||||
.send((block, head_block_hash, safe_block_hash, finalized_block_hash))
|
||||
.send((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
|
||||
.await
|
||||
{
|
||||
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
|
||||
@@ -206,10 +233,11 @@ impl Command {
|
||||
});
|
||||
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
while let Some((block, head, safe, finalized)) = {
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = receiver.recv().await;
|
||||
total_wait_time += wait_start.elapsed();
|
||||
@@ -228,16 +256,40 @@ impl Command {
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(block, is_optimism)?;
|
||||
let (version, params) =
|
||||
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
|
||||
let start = Instant::now();
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency: np_latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
|
||||
call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
@@ -249,8 +301,19 @@ impl Command {
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
|
||||
if let Some(scraper) = metrics_scraper.as_mut() &&
|
||||
let Err(err) = scraper.scrape_after_block(block_number).await
|
||||
{
|
||||
warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
|
||||
}
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
@@ -277,6 +340,10 @@ impl Command {
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
if let (Some(path), Some(scraper)) = (&self.benchmark.output, &metrics_scraper) {
|
||||
scraper.write_csv(path)?;
|
||||
}
|
||||
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
|
||||
NEW_PAYLOAD_OUTPUT_SUFFIX,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_new_payload},
|
||||
valid_payload::{block_to_new_payload, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_provider::Provider;
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use clap::Parser;
|
||||
use csv::Writer;
|
||||
use eyre::{Context, OptionExt};
|
||||
@@ -49,9 +50,18 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
|
||||
|
||||
if use_reth_namespace {
|
||||
info!("Using reth_newPayload endpoint");
|
||||
}
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -74,20 +84,33 @@ impl Command {
|
||||
}
|
||||
};
|
||||
|
||||
let rlp = if rlp_blocks {
|
||||
let Ok(rlp) = block_provider.debug_get_raw_block(next_block.into()).await
|
||||
else {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}");
|
||||
let _ = error_sender
|
||||
.send(eyre::eyre!("Failed to fetch raw block {next_block}"));
|
||||
break;
|
||||
};
|
||||
Some(rlp)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
next_block += 1;
|
||||
if let Err(e) = sender.send(block).await {
|
||||
if let Err(e) = sender.send((block, rlp)).await {
|
||||
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// put results in a summary vec so they can be printed at the end
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
while let Some(block) = {
|
||||
while let Some((block, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = receiver.recv().await;
|
||||
total_wait_time += wait_start.elapsed();
|
||||
@@ -99,13 +122,34 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let (version, params) = block_to_new_payload(block, is_optimism)?;
|
||||
let (version, params) =
|
||||
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
|
||||
|
||||
let start = Instant::now();
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
info!(target: "reth-bench", %new_payload_result);
|
||||
let latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
blocks_processed += 1;
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %new_payload_result);
|
||||
|
||||
// current duration since the start of the benchmark minus the time
|
||||
// waiting for blocks
|
||||
@@ -115,6 +159,12 @@ impl Command {
|
||||
let row =
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((row, new_payload_result));
|
||||
|
||||
if let Some(scraper) = metrics_scraper.as_mut() &&
|
||||
let Err(err) = scraper.scrape_after_block(block_number).await
|
||||
{
|
||||
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
|
||||
}
|
||||
}
|
||||
|
||||
// Check if the spawned task encountered an error
|
||||
@@ -145,6 +195,10 @@ impl Command {
|
||||
}
|
||||
writer.flush()?;
|
||||
|
||||
if let Some(scraper) = &metrics_scraper {
|
||||
scraper.write_csv(&path)?;
|
||||
}
|
||||
|
||||
info!(target: "reth-bench", "Finished writing benchmark output files to {:?}.", path);
|
||||
}
|
||||
|
||||
|
||||
@@ -22,7 +22,10 @@ pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct GasRampPayloadFile {
|
||||
/// Engine API version (1-5).
|
||||
pub(crate) version: u8,
|
||||
///
|
||||
/// `None` indicates that `reth_newPayload` should be used.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) version: Option<u8>,
|
||||
/// The block hash for FCU.
|
||||
pub(crate) block_hash: B256,
|
||||
/// The params to pass to newPayload.
|
||||
@@ -37,6 +40,12 @@ pub(crate) struct NewPayloadResult {
|
||||
pub(crate) gas_used: u64,
|
||||
/// The latency of the `newPayload` call.
|
||||
pub(crate) latency: Duration,
|
||||
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
|
||||
pub(crate) persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for execution cache lock.
|
||||
pub(crate) execution_cache_wait: Duration,
|
||||
/// Time spent waiting for sparse trie lock.
|
||||
pub(crate) sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
impl NewPayloadResult {
|
||||
@@ -67,9 +76,12 @@ impl Serialize for NewPayloadResult {
|
||||
{
|
||||
// convert the time to microseconds
|
||||
let time = self.latency.as_micros();
|
||||
let mut state = serializer.serialize_struct("NewPayloadResult", 2)?;
|
||||
let mut state = serializer.serialize_struct("NewPayloadResult", 5)?;
|
||||
state.serialize_field("gas_used", &self.gas_used)?;
|
||||
state.serialize_field("latency", &time)?;
|
||||
state.serialize_field("persistence_wait", &self.persistence_wait.map(|d| d.as_micros()))?;
|
||||
state.serialize_field("execution_cache_wait", &self.execution_cache_wait.as_micros())?;
|
||||
state.serialize_field("sparse_trie_wait", &self.sparse_trie_wait.as_micros())?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -126,7 +138,7 @@ impl Serialize for CombinedResult {
|
||||
let fcu_latency = self.fcu_latency.as_micros();
|
||||
let new_payload_latency = self.new_payload_result.latency.as_micros();
|
||||
let total_latency = self.total_latency.as_micros();
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 10)?;
|
||||
|
||||
// flatten the new payload result because this is meant for CSV writing
|
||||
state.serialize_field("block_number", &self.block_number)?;
|
||||
@@ -136,6 +148,18 @@ impl Serialize for CombinedResult {
|
||||
state.serialize_field("new_payload_latency", &new_payload_latency)?;
|
||||
state.serialize_field("fcu_latency", &fcu_latency)?;
|
||||
state.serialize_field("total_latency", &total_latency)?;
|
||||
state.serialize_field(
|
||||
"persistence_wait",
|
||||
&self.new_payload_result.persistence_wait.map(|d| d.as_micros()),
|
||||
)?;
|
||||
state.serialize_field(
|
||||
"execution_cache_wait",
|
||||
&self.new_payload_result.execution_cache_wait.as_micros(),
|
||||
)?;
|
||||
state.serialize_field(
|
||||
"sparse_trie_wait",
|
||||
&self.new_payload_result.sparse_trie_wait.as_micros(),
|
||||
)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@ use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
helpers::parse_duration,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
|
||||
TotalGasOutput, TotalGasRow,
|
||||
@@ -23,17 +24,21 @@ use crate::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload},
|
||||
valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
|
||||
ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use reth_rpc_api::RethNewPayloadInput;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{Duration, Instant},
|
||||
@@ -124,6 +129,22 @@ pub struct Command {
|
||||
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
|
||||
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
|
||||
ws_rpc_url: Option<String>,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
/// directly, waits for persistence and cache updates to complete before processing,
|
||||
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
|
||||
/// Optional Prometheus metrics endpoint to scrape after each block.
|
||||
///
|
||||
/// When provided, reth-bench will fetch metrics from this URL after each
|
||||
/// payload, recording per-block execution and state root durations.
|
||||
/// Results are written to `metrics.csv` in the output directory.
|
||||
#[arg(long = "metrics-url", value_name = "URL", verbatim_doc_comment)]
|
||||
metrics_url: Option<String>,
|
||||
}
|
||||
|
||||
/// A loaded payload ready for execution.
|
||||
@@ -141,7 +162,9 @@ struct GasRampPayload {
|
||||
/// Block number from filename.
|
||||
block_number: u64,
|
||||
/// Engine API version for newPayload.
|
||||
version: EngineApiMessageVersion,
|
||||
///
|
||||
/// `None` indicates that `reth_newPayload` should be used.
|
||||
version: Option<EngineApiMessageVersion>,
|
||||
/// The file contents.
|
||||
file: GasRampPayloadFile,
|
||||
}
|
||||
@@ -163,6 +186,9 @@ impl Command {
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options
|
||||
// When both are set: wait at least wait_time, and also wait for persistence if needed
|
||||
@@ -190,6 +216,8 @@ impl Command {
|
||||
(None, false) => None,
|
||||
};
|
||||
|
||||
let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
|
||||
|
||||
// Set up authenticated engine provider
|
||||
let jwt =
|
||||
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
|
||||
@@ -248,14 +276,19 @@ impl Command {
|
||||
"Executing gas ramp payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
|
||||
let _ = call_new_payload_with_reth(
|
||||
&auth_provider,
|
||||
payload.version,
|
||||
payload.file.params.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: payload.file.block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
|
||||
call_forkchoice_updated_with_reth(&auth_provider, payload.version, fcu_state).await?;
|
||||
|
||||
info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
|
||||
|
||||
@@ -303,20 +336,50 @@ impl Command {
|
||||
"Sending newPayload"
|
||||
);
|
||||
|
||||
let status = auth_provider
|
||||
.new_payload_v4(
|
||||
execution_payload.clone(),
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
let (version, params) = if self.reth_new_payload {
|
||||
let reth_data = ExecutionData {
|
||||
payload: execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
parent_beacon_block_root: B256::ZERO,
|
||||
},
|
||||
PraguePayloadFields {
|
||||
requests: envelope.execution_requests.clone().into(),
|
||||
},
|
||||
),
|
||||
};
|
||||
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(reth_data),))?)
|
||||
} else {
|
||||
(
|
||||
Some(EngineApiMessageVersion::V4),
|
||||
serde_json::to_value((
|
||||
execution_payload.clone(),
|
||||
Vec::<B256>::new(),
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
))?,
|
||||
)
|
||||
.await?;
|
||||
};
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency: np_latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
@@ -324,12 +387,12 @@ impl Command {
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
|
||||
debug!(target: "reth-bench", method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, fcu_state).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
@@ -341,7 +404,14 @@ impl Command {
|
||||
};
|
||||
|
||||
let current_duration = total_benchmark_duration.elapsed();
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
let progress = format!("{}/{}", i + 1, payloads.len());
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
|
||||
if let Some(scraper) = metrics_scraper.as_mut() &&
|
||||
let Err(err) = scraper.scrape_after_block(block_number).await
|
||||
{
|
||||
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
|
||||
}
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
@@ -351,7 +421,6 @@ impl Command {
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((gas_row, combined_result));
|
||||
|
||||
debug!(target: "reth-bench", ?status, ?fcu_result, "Payload executed successfully");
|
||||
parent_hash = block_hash;
|
||||
}
|
||||
|
||||
@@ -366,6 +435,10 @@ impl Command {
|
||||
write_benchmark_results(path, &gas_output_results, &combined_results)?;
|
||||
}
|
||||
|
||||
if let (Some(path), Some(scraper)) = (&self.output, &metrics_scraper) {
|
||||
scraper.write_csv(path)?;
|
||||
}
|
||||
|
||||
let gas_output =
|
||||
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
|
||||
info!(
|
||||
@@ -476,13 +549,18 @@ impl Command {
|
||||
let file: GasRampPayloadFile = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
|
||||
let version = match file.version {
|
||||
1 => EngineApiMessageVersion::V1,
|
||||
2 => EngineApiMessageVersion::V2,
|
||||
3 => EngineApiMessageVersion::V3,
|
||||
4 => EngineApiMessageVersion::V4,
|
||||
5 => EngineApiMessageVersion::V5,
|
||||
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
|
||||
let version = if let Some(version) = file.version {
|
||||
match version {
|
||||
1 => EngineApiMessageVersion::V1,
|
||||
2 => EngineApiMessageVersion::V2,
|
||||
3 => EngineApiMessageVersion::V3,
|
||||
4 => EngineApiMessageVersion::V4,
|
||||
5 => EngineApiMessageVersion::V5,
|
||||
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
|
||||
}
|
||||
.into()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
info!(
|
||||
|
||||
@@ -20,6 +20,19 @@ impl BenchMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total number of blocks in the benchmark, if known.
|
||||
///
|
||||
/// For [`BenchMode::Range`] this is the length of the range.
|
||||
/// For [`BenchMode::Continuous`] the total is unbounded, so `None` is returned.
|
||||
pub const fn total_blocks(&self) -> Option<u64> {
|
||||
match self {
|
||||
Self::Continuous(_) => None,
|
||||
Self::Range(range) => {
|
||||
Some(range.end().saturating_sub(*range.start()).saturating_add(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a [`BenchMode`] from optional `from` and `to` fields.
|
||||
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
|
||||
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,
|
||||
|
||||
@@ -3,15 +3,18 @@
|
||||
//! before sending additional calls.
|
||||
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
|
||||
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
|
||||
ExecutionData, ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar,
|
||||
ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
|
||||
};
|
||||
use alloy_transport::TransportResult;
|
||||
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use reth_rpc_api::RethNewPayloadInput;
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, error};
|
||||
|
||||
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
|
||||
@@ -161,10 +164,21 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
///
|
||||
/// Returns `(version, versioned_params, execution_data)`.
|
||||
pub(crate) fn block_to_new_payload(
|
||||
block: AnyRpcBlock,
|
||||
is_optimism: bool,
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
|
||||
rlp: Option<Bytes>,
|
||||
reth_new_payload: bool,
|
||||
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
|
||||
if let Some(rlp) = rlp {
|
||||
return Ok((
|
||||
None,
|
||||
serde_json::to_value((RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),))?,
|
||||
));
|
||||
}
|
||||
let block = block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
@@ -176,16 +190,29 @@ pub(crate) fn block_to_new_payload(
|
||||
|
||||
// Convert to execution payload
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
|
||||
let (version, params, execution_data) =
|
||||
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
|
||||
|
||||
if reth_new_payload {
|
||||
Ok((None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?))
|
||||
} else {
|
||||
Ok((Some(version), params))
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an execution payload and sidecar into versioned engine API params and an
|
||||
/// [`ExecutionData`].
|
||||
///
|
||||
/// Returns `(version, versioned_params, execution_data)`.
|
||||
pub(crate) fn payload_to_new_payload(
|
||||
payload: ExecutionPayload,
|
||||
sidecar: ExecutionPayloadSidecar,
|
||||
is_optimism: bool,
|
||||
withdrawals_root: Option<B256>,
|
||||
target_version: Option<EngineApiMessageVersion>,
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar.cancun().unwrap();
|
||||
@@ -244,7 +271,7 @@ pub(crate) fn payload_to_new_payload(
|
||||
}
|
||||
};
|
||||
|
||||
Ok((version, params))
|
||||
Ok((version, params, execution_data))
|
||||
}
|
||||
|
||||
/// Calls the correct `engine_newPayload` method depending on the given [`ExecutionPayload`] and its
|
||||
@@ -252,32 +279,85 @@ pub(crate) fn payload_to_new_payload(
|
||||
///
|
||||
/// # Panics
|
||||
/// If the given payload is a V3 payload, but a parent beacon block root is provided as `None`.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
|
||||
provider: P,
|
||||
version: EngineApiMessageVersion,
|
||||
version: Option<EngineApiMessageVersion>,
|
||||
params: serde_json::Value,
|
||||
) -> TransportResult<()> {
|
||||
let method = version.method_name();
|
||||
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
|
||||
call_new_payload_with_reth(provider, version, params).await
|
||||
}
|
||||
|
||||
/// Response from `reth_newPayload` endpoint, which includes server-measured latency.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RethPayloadStatus {
|
||||
latency_us: u64,
|
||||
#[serde(default)]
|
||||
persistence_wait_us: Option<u64>,
|
||||
#[serde(default)]
|
||||
execution_cache_wait_us: u64,
|
||||
#[serde(default)]
|
||||
sparse_trie_wait_us: u64,
|
||||
}
|
||||
|
||||
/// Server-side timing breakdown from `reth_newPayload` endpoint.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub(crate) struct NewPayloadTimingBreakdown {
|
||||
/// Server-side execution latency.
|
||||
pub(crate) latency: Duration,
|
||||
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
|
||||
pub(crate) persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for execution cache lock.
|
||||
pub(crate) execution_cache_wait: Duration,
|
||||
/// Time spent waiting for sparse trie lock.
|
||||
pub(crate) sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
/// Calls either `engine_newPayload*` or `reth_newPayload` depending on whether
|
||||
/// `version` is provided.
|
||||
///
|
||||
/// When `version` is `None`, uses `reth_newPayload` endpoint with provided params.
|
||||
///
|
||||
/// Returns the server-reported timing breakdown when using the reth namespace, or `None` for
|
||||
/// the standard engine namespace.
|
||||
pub(crate) async fn call_new_payload_with_reth<N: Network, P: Provider<N>>(
|
||||
provider: P,
|
||||
version: Option<EngineApiMessageVersion>,
|
||||
params: serde_json::Value,
|
||||
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
|
||||
let method = version.map(|v| v.method_name()).unwrap_or("reth_newPayload");
|
||||
|
||||
debug!(target: "reth-bench", method, "Sending newPayload");
|
||||
|
||||
let mut status: PayloadStatus = provider.client().request(method, ¶ms).await?;
|
||||
let resp = loop {
|
||||
let resp: serde_json::Value = provider.client().request(method, ¶ms).await?;
|
||||
let status = PayloadStatus::deserialize(&resp)?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_valid() {
|
||||
break resp;
|
||||
}
|
||||
if status.is_invalid() {
|
||||
error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
|
||||
format!("Invalid {method}: {status:?}"),
|
||||
))))
|
||||
return Err(eyre::eyre!("Invalid {method}: {status:?}"));
|
||||
}
|
||||
if status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
"invalid range: no canonical state found for parent of requested block",
|
||||
))
|
||||
return Err(eyre::eyre!(
|
||||
"invalid range: no canonical state found for parent of requested block"
|
||||
));
|
||||
}
|
||||
status = provider.client().request(method, ¶ms).await?;
|
||||
};
|
||||
|
||||
if version.is_some() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(())
|
||||
|
||||
let resp: RethPayloadStatus = serde_json::from_value(resp)?;
|
||||
|
||||
Ok(Some(NewPayloadTimingBreakdown {
|
||||
latency: Duration::from_micros(resp.latency_us),
|
||||
persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
|
||||
execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
|
||||
sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
|
||||
@@ -304,3 +384,47 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Calls either `reth_forkchoiceUpdated` or the standard `engine_forkchoiceUpdated*` depending
|
||||
/// on `use_reth`.
|
||||
///
|
||||
/// When `use_reth` is true, uses the `reth_forkchoiceUpdated` endpoint which sends a regular FCU
|
||||
/// with no payload attributes.
|
||||
pub(crate) async fn call_forkchoice_updated_with_reth<
|
||||
N: Network,
|
||||
P: Provider<N> + EngineApiValidWaitExt<N>,
|
||||
>(
|
||||
provider: P,
|
||||
message_version: Option<EngineApiMessageVersion>,
|
||||
forkchoice_state: ForkchoiceState,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
if let Some(message_version) = message_version {
|
||||
call_forkchoice_updated(provider, message_version, forkchoice_state, None).await
|
||||
} else {
|
||||
let method = "reth_forkchoiceUpdated";
|
||||
let reth_params = serde_json::to_value((forkchoice_state,))
|
||||
.expect("ForkchoiceState serialization cannot fail");
|
||||
|
||||
debug!(target: "reth-bench", method, "Sending forkchoiceUpdated");
|
||||
|
||||
loop {
|
||||
let resp: ForkchoiceUpdated = provider.client().request(method, &reth_params).await?;
|
||||
|
||||
if resp.is_valid() {
|
||||
break Ok(resp)
|
||||
}
|
||||
|
||||
if resp.is_invalid() {
|
||||
error!(target: "reth-bench", ?resp, "Invalid {method}");
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
|
||||
std::io::Error::other(format!("Invalid {method}: {resp:?}")),
|
||||
)))
|
||||
}
|
||||
if resp.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
"invalid range: no canonical state found for parent of requested block",
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,7 +30,8 @@ workspace = true
|
||||
# reth
|
||||
reth-ethereum-cli.workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
reth-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-db = { workspace = true, features = ["mdbx"] }
|
||||
reth-provider.workspace = true
|
||||
reth-revm.workspace = true
|
||||
@@ -110,7 +111,6 @@ dev = ["reth-ethereum-cli/dev"]
|
||||
|
||||
asm-keccak = [
|
||||
"reth-node-core/asm-keccak",
|
||||
"reth-primitives/asm-keccak",
|
||||
"reth-ethereum-cli/asm-keccak",
|
||||
"reth-node-ethereum/asm-keccak",
|
||||
"alloy-primitives/asm-keccak",
|
||||
@@ -190,6 +190,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
|
||||
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
|
||||
edge = ["rocksdb"]
|
||||
|
||||
|
||||
@@ -124,9 +124,11 @@ pub mod providers {
|
||||
pub use reth_provider::*;
|
||||
}
|
||||
|
||||
/// Re-exported from `reth_primitives`.
|
||||
/// Re-exported primitives.
|
||||
#[allow(ambiguous_glob_reexports)]
|
||||
pub mod primitives {
|
||||
pub use reth_primitives::*;
|
||||
pub use reth_ethereum_primitives::*;
|
||||
pub use reth_primitives_traits::*;
|
||||
}
|
||||
|
||||
/// Re-exported from `reth_ethereum_consensus`.
|
||||
|
||||
@@ -132,6 +132,6 @@ impl<H: BlockHeader> EthChainSpec for ChainSpec<H> {
|
||||
}
|
||||
|
||||
fn final_paris_total_difficulty(&self) -> Option<U256> {
|
||||
self.paris_block_and_final_difficulty.map(|(_, final_difficulty)| final_difficulty)
|
||||
self.get_final_paris_total_difficulty()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -855,15 +855,9 @@ impl From<Genesis> for ChainSpec {
|
||||
// those networks we use the activation
|
||||
// blocks of those networks
|
||||
match genesis.config.chain_id {
|
||||
1 => {
|
||||
if ttd == MAINNET_PARIS_TTD {
|
||||
return Some(MAINNET_PARIS_BLOCK)
|
||||
}
|
||||
}
|
||||
11155111 => {
|
||||
if ttd == SEPOLIA_PARIS_TTD {
|
||||
return Some(SEPOLIA_PARIS_BLOCK)
|
||||
}
|
||||
1 if ttd == MAINNET_PARIS_TTD => return Some(MAINNET_PARIS_BLOCK),
|
||||
11155111 if ttd == SEPOLIA_PARIS_TTD => {
|
||||
return Some(SEPOLIA_PARIS_BLOCK)
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
|
||||
@@ -18,6 +18,5 @@ alloy-genesis.workspace = true
|
||||
|
||||
# misc
|
||||
clap.workspace = true
|
||||
shellexpand.workspace = true
|
||||
eyre.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -73,7 +73,7 @@ pub trait ChainSpecParser: Clone + Send + Sync + 'static {
|
||||
/// A helper to parse a [`Genesis`](alloy_genesis::Genesis) as argument or from disk.
|
||||
pub fn parse_genesis(s: &str) -> eyre::Result<alloy_genesis::Genesis> {
|
||||
// try to read json from path first
|
||||
let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) {
|
||||
let raw = match fs::read_to_string(PathBuf::from(s)) {
|
||||
Ok(raw) => raw,
|
||||
Err(io_err) => {
|
||||
// valid json may start with "\n", but must contain "{"
|
||||
|
||||
@@ -43,7 +43,7 @@ reth-node-metrics.workspace = true
|
||||
reth-ethereum-primitives = { workspace = true, optional = true }
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-prune-types = { workspace = true, optional = true }
|
||||
reth-prune-types.workspace = true
|
||||
reth-revm.workspace = true
|
||||
reth-stages.workspace = true
|
||||
reth-stages-types = { workspace = true, optional = true }
|
||||
@@ -53,7 +53,7 @@ reth-tasks.workspace = true
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-db = { workspace = true, features = ["metrics"] }
|
||||
reth-trie-common.workspace = true
|
||||
reth-trie-common = { workspace = true, optional = true }
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-discv4.workspace = true
|
||||
reth-discv5.workspace = true
|
||||
@@ -113,6 +113,7 @@ arbitrary = [
|
||||
"dep:proptest",
|
||||
"dep:arbitrary",
|
||||
"dep:proptest-arbitrary-interop",
|
||||
"dep:reth-trie-common",
|
||||
"reth-db-api/arbitrary",
|
||||
"reth-eth-wire/arbitrary",
|
||||
"reth-db/arbitrary",
|
||||
@@ -123,11 +124,11 @@ arbitrary = [
|
||||
"reth-codecs/test-utils",
|
||||
"reth-prune-types/test-utils",
|
||||
"reth-stages-types/test-utils",
|
||||
"reth-trie-common/test-utils",
|
||||
"reth-trie-common?/test-utils",
|
||||
"reth-codecs/arbitrary",
|
||||
"reth-prune-types?/arbitrary",
|
||||
"reth-prune-types/arbitrary",
|
||||
"reth-stages-types?/arbitrary",
|
||||
"reth-trie-common/arbitrary",
|
||||
"reth-trie-common?/arbitrary",
|
||||
"alloy-consensus/arbitrary",
|
||||
"reth-primitives-traits/arbitrary",
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs, StorageArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs, StorageArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -67,74 +67,37 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
pub storage: StorageArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
|
||||
/// `RocksDB` CLI args.
|
||||
/// Returns the effective storage settings derived from `--storage.v2`.
|
||||
///
|
||||
/// The base storage mode is determined by `--storage.v2`:
|
||||
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
|
||||
///
|
||||
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
|
||||
/// - Otherwise: uses [`StorageSettings::base()`] defaults
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
|
||||
|
||||
// Apply static files overrides (only when explicitly set)
|
||||
if let Some(v) = self.static_files.receipts {
|
||||
s = s.with_receipts_in_static_files(v);
|
||||
if self.storage.v2 {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::base()
|
||||
}
|
||||
if let Some(v) = self.static_files.transaction_senders {
|
||||
s = s.with_transaction_senders_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.account_changesets {
|
||||
s = s.with_account_changesets_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.storage_changesets {
|
||||
s = s.with_storage_changesets_in_static_files(v);
|
||||
}
|
||||
|
||||
// Apply rocksdb overrides
|
||||
// --rocksdb.all sets all rocksdb flags to true
|
||||
if self.rocksdb.all {
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true)
|
||||
.with_account_history_in_rocksdb(true);
|
||||
}
|
||||
|
||||
// Individual rocksdb flags override --rocksdb.all when explicitly set
|
||||
if let Some(v) = self.rocksdb.tx_hash {
|
||||
s = s.with_transaction_hash_numbers_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.storages_history {
|
||||
s = s.with_storages_history_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.account_history {
|
||||
s = s.with_account_history_in_rocksdb(v);
|
||||
}
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
/// [`Environment`].
|
||||
///
|
||||
/// Internally builds a [`reth_tasks::Runtime`] attached to the current tokio handle for
|
||||
/// parallel storage I/O.
|
||||
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
|
||||
/// The provided `runtime` is used for parallel storage I/O.
|
||||
pub fn init<N: CliNodeTypes>(
|
||||
&self,
|
||||
access: AccessRights,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<Environment<N>>
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let runtime = reth_tasks::Runtime::with_existing_handle(tokio::runtime::Handle::current())?;
|
||||
let data_dir = self.datadir.clone().resolve_datadir(self.chain.chain());
|
||||
let db_path = data_dir.db();
|
||||
let sf_path = data_dir.static_files();
|
||||
@@ -183,11 +146,24 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
})
|
||||
}
|
||||
};
|
||||
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?;
|
||||
let rocksdb_provider = if !access.is_read_write() && !RocksDBProvider::exists(&rocksdb_path)
|
||||
{
|
||||
// RocksDB database doesn't exist yet (e.g. datadir restored from a snapshot
|
||||
// or created before RocksDB storage). Create an empty one so read-only
|
||||
// commands can proceed.
|
||||
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
|
||||
reth_fs_util::create_dir_all(&rocksdb_path)?;
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.build()?
|
||||
} else {
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?
|
||||
};
|
||||
|
||||
let provider_factory =
|
||||
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access, runtime)?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use reth_codecs::Compact;
|
||||
use reth_db_api::{cursor::DbDupCursorRO, database::Database, tables, transaction::DbTx};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::info;
|
||||
|
||||
@@ -22,52 +23,94 @@ impl Command {
|
||||
/// Execute `db account-storage` command
|
||||
pub fn execute<N: NodeTypesWithDB>(self, tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
let address = self.address;
|
||||
let (slot_count, plain_size) = tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
let (slot_count, storage_size) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing hashed storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
let total_size = if count > 0 { 32 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??;
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
} else {
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { plain_size + 12 } else { 0 };
|
||||
let total_estimate = plain_size + hashed_size_estimate;
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
};
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
|
||||
println!("Account: {address}");
|
||||
println!("Hashed address: {hashed_address}");
|
||||
println!("Storage slots: {slot_count}");
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(plain_size as f64));
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(hashed_size_estimate as f64));
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
if use_hashed_state {
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
} else {
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { storage_size + 12 } else { 0 };
|
||||
let total_estimate = storage_size + hashed_size_estimate;
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
println!(
|
||||
"Hashed storage size: {} (estimated)",
|
||||
human_bytes(hashed_size_estimate as f64)
|
||||
);
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ use reth_db_api::{
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_static_file_types::{ChangesetOffset, StaticFileSegment};
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
time::{Duration, Instant},
|
||||
@@ -134,12 +134,12 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
|
||||
|
||||
let start_time = Instant::now();
|
||||
let mut hasher = checksum_hasher();
|
||||
let mut total = 0usize;
|
||||
let limit = limit.unwrap_or(usize::MAX);
|
||||
let mut checksummer = Checksummer::new(checksum_hasher(), limit);
|
||||
|
||||
let start_block = start_block.unwrap_or(0);
|
||||
let end_block = end_block.unwrap_or(u64::MAX);
|
||||
let is_change_based = segment.is_change_based();
|
||||
|
||||
info!(
|
||||
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
|
||||
@@ -149,7 +149,8 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
if limit == usize::MAX { None } else { Some(limit) }
|
||||
);
|
||||
|
||||
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
let mut reached_limit = false;
|
||||
for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
|
||||
if block_range.end() < start_block || block_range.start() > end_block {
|
||||
continue;
|
||||
}
|
||||
@@ -167,28 +168,42 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
|
||||
let mut cursor = jar_provider.cursor()?;
|
||||
|
||||
while let Ok(Some(row)) = cursor.next_row() {
|
||||
for col_data in row.iter() {
|
||||
hasher.write(col_data);
|
||||
}
|
||||
if is_change_based {
|
||||
let offsets = jar_provider.read_changeset_offsets()?.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Missing changeset offsets sidecar for segment {} at range {}",
|
||||
segment,
|
||||
block_range
|
||||
)
|
||||
})?;
|
||||
let input = ChangeBasedChecksumInput {
|
||||
segment,
|
||||
block_range_start: block_range.start(),
|
||||
start_block,
|
||||
end_block,
|
||||
offsets: &offsets,
|
||||
};
|
||||
|
||||
total += 1;
|
||||
|
||||
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {total} entries.");
|
||||
}
|
||||
|
||||
if total >= limit {
|
||||
break 'outer;
|
||||
reached_limit = checksum_change_based_segment(&mut checksummer, input, &mut cursor)?;
|
||||
} else {
|
||||
while let Some(row) = cursor.next_row()? {
|
||||
if checksummer.write_row(&row) {
|
||||
reached_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Explicitly drop provider before removing from cache to avoid deadlock
|
||||
drop(jar_provider);
|
||||
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
|
||||
|
||||
if reached_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let checksum = hasher.finish();
|
||||
let (checksum, total) = checksummer.finish();
|
||||
let elapsed = start_time.elapsed();
|
||||
|
||||
info!(
|
||||
@@ -267,7 +282,7 @@ impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N
|
||||
|
||||
total = index + 1;
|
||||
if total >= limit {
|
||||
break
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,3 +300,139 @@ impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N
|
||||
Ok((checksum, elapsed))
|
||||
}
|
||||
}
|
||||
|
||||
/// Accumulates a checksum over key-value entries, tracking count and limit.
|
||||
struct Checksummer<H> {
|
||||
hasher: H,
|
||||
total: usize,
|
||||
limit: usize,
|
||||
}
|
||||
|
||||
impl<H: Hasher> Checksummer<H> {
|
||||
fn new(hasher: H, limit: usize) -> Self {
|
||||
Self { hasher, total: 0, limit }
|
||||
}
|
||||
|
||||
/// Hash a row's columns (non-changeset segments). Returns `true` if the limit is reached.
|
||||
fn write_row(&mut self, row: &[&[u8]]) -> bool {
|
||||
for col in row {
|
||||
self.hasher.write(col);
|
||||
}
|
||||
self.advance()
|
||||
}
|
||||
|
||||
/// Hash a key + value as two separate writes, matching MDBX raw entry semantics.
|
||||
/// Write boundaries matter: foldhash rotates its accumulator by `len` on each `write`.
|
||||
fn write_entry(&mut self, key: &[u8], value: &[u8]) -> bool {
|
||||
self.hasher.write(key);
|
||||
self.hasher.write(value);
|
||||
self.advance()
|
||||
}
|
||||
|
||||
fn advance(&mut self) -> bool {
|
||||
self.total += 1;
|
||||
if self.total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
|
||||
info!("Hashed {} entries.", self.total);
|
||||
}
|
||||
self.total >= self.limit
|
||||
}
|
||||
|
||||
fn finish(self) -> (u64, usize) {
|
||||
(self.hasher.finish(), self.total)
|
||||
}
|
||||
}
|
||||
|
||||
/// Reconstruct MDBX `StorageChangeSets` key/value boundaries from a static-file row.
|
||||
///
|
||||
/// MDBX layout:
|
||||
/// - key: `BlockNumberAddress` => `[8B block_number][20B address]`
|
||||
/// - value: `StorageEntry` => `[32B storage_key][compact U256 value]`
|
||||
///
|
||||
/// Static-file row layout for `StorageBeforeTx`:
|
||||
/// - `[20B address][32B storage_key][compact U256 value]`
|
||||
fn split_storage_changeset_row(block_number: u64, row: &[u8]) -> eyre::Result<([u8; 28], &[u8])> {
|
||||
if row.len() < 20 {
|
||||
return Err(eyre::eyre!(
|
||||
"Storage changeset row too short: expected at least 20 bytes, got {}",
|
||||
row.len()
|
||||
));
|
||||
}
|
||||
|
||||
let mut key_buf = [0u8; 28];
|
||||
key_buf[..8].copy_from_slice(&block_number.to_be_bytes());
|
||||
key_buf[8..].copy_from_slice(&row[..20]);
|
||||
Ok((key_buf, &row[20..]))
|
||||
}
|
||||
|
||||
struct ChangeBasedChecksumInput<'a> {
|
||||
segment: StaticFileSegment,
|
||||
block_range_start: u64,
|
||||
start_block: u64,
|
||||
end_block: u64,
|
||||
offsets: &'a [ChangesetOffset],
|
||||
}
|
||||
|
||||
fn checksum_change_based_segment<H: Hasher>(
|
||||
checksummer: &mut Checksummer<H>,
|
||||
input: ChangeBasedChecksumInput<'_>,
|
||||
cursor: &mut reth_db::static_file::StaticFileCursor<'_>,
|
||||
) -> eyre::Result<bool> {
|
||||
let ChangeBasedChecksumInput { segment, block_range_start, start_block, end_block, offsets } =
|
||||
input;
|
||||
let is_storage = segment.is_storage_change_sets();
|
||||
let mut reached_limit = false;
|
||||
|
||||
for (offset_index, offset) in offsets.iter().enumerate() {
|
||||
let block_number = block_range_start + offset_index as u64;
|
||||
let include = block_number >= start_block && block_number <= end_block;
|
||||
|
||||
for _ in 0..offset.num_changes() {
|
||||
let row = cursor.next_row()?.ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Unexpected EOF while checksumming {} static file at range starting {}",
|
||||
segment,
|
||||
block_range_start
|
||||
)
|
||||
})?;
|
||||
|
||||
if !include {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Reconstruct MDBX key/value write boundaries. foldhash rotates
|
||||
// its accumulator by `len` on each write(), so boundaries must
|
||||
// match exactly.
|
||||
let done = if is_storage {
|
||||
// StorageChangeSets: MDBX key = BlockNumberAddress (28B),
|
||||
// value = compact StorageEntry. Column 0 is compact
|
||||
// StorageBeforeTx = [20B address][32B key][compact U256].
|
||||
let col = row[0];
|
||||
let (key, value) = split_storage_changeset_row(block_number, col)?;
|
||||
checksummer.write_entry(&key, value)
|
||||
} else {
|
||||
// AccountChangeSets: MDBX key = BlockNumber (8B),
|
||||
// value = compact AccountBeforeTx (= column 0).
|
||||
checksummer.write_entry(&block_number.to_be_bytes(), row[0])
|
||||
};
|
||||
|
||||
if done {
|
||||
reached_limit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if reached_limit {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if !reached_limit && cursor.next_row()?.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"Changeset offsets do not cover all rows for {} at range starting {}",
|
||||
segment,
|
||||
block_range_start
|
||||
));
|
||||
}
|
||||
|
||||
Ok(reached_limit)
|
||||
}
|
||||
|
||||
61
crates/cli/commands/src/db/copy.rs
Normal file
61
crates/cli/commands/src/db/copy.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
use clap::Parser;
|
||||
use reth_db::mdbx::{self, ffi};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Copies the MDBX database to a new location.
|
||||
///
|
||||
/// Equivalent to the standalone `mdbx_copy` tool but bundled into reth.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Destination path for the database copy.
|
||||
dest: PathBuf,
|
||||
|
||||
/// Compact the database while copying (reclaims free space).
|
||||
#[arg(short, long)]
|
||||
compact: bool,
|
||||
|
||||
/// Force dynamic size for the destination database.
|
||||
#[arg(short = 'd', long)]
|
||||
force_dynamic_size: bool,
|
||||
|
||||
/// Throttle to avoid MVCC pressure on writers.
|
||||
#[arg(short = 'p', long)]
|
||||
throttle_mvcc: bool,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db copy` command
|
||||
pub fn execute(self, db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let mut flags: ffi::MDBX_copy_flags_t = ffi::MDBX_CP_DEFAULTS;
|
||||
if self.compact {
|
||||
flags |= ffi::MDBX_CP_COMPACT;
|
||||
}
|
||||
if self.force_dynamic_size {
|
||||
flags |= ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
}
|
||||
if self.throttle_mvcc {
|
||||
flags |= ffi::MDBX_CP_THROTTLE_MVCC;
|
||||
}
|
||||
|
||||
let dest = self
|
||||
.dest
|
||||
.to_str()
|
||||
.ok_or_else(|| eyre::eyre!("destination path must be valid UTF-8"))?;
|
||||
let dest_cstr = std::ffi::CString::new(dest)?;
|
||||
|
||||
println!("Copying database to {} ...", self.dest.display());
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
println!("Done.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -98,7 +98,8 @@ impl Command {
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
let se: reth_primitives_traits::StorageEntry = entry;
|
||||
println!("{}", serde_json::to_string_pretty(&se)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
@@ -106,7 +107,14 @@ impl Command {
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
let serializable: Vec<_> = changesets
|
||||
.into_iter()
|
||||
.map(|(addr, entry)| {
|
||||
let se: reth_primitives_traits::StorageEntry = entry;
|
||||
(addr, se)
|
||||
})
|
||||
.collect();
|
||||
println!("{}", serde_json::to_string_pretty(&serializable)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
@@ -12,11 +12,14 @@ use std::{
|
||||
mod account_storage;
|
||||
mod checksum;
|
||||
mod clear;
|
||||
mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
mod prune_checkpoints;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
mod stage_checkpoints;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
@@ -42,6 +45,8 @@ pub enum Subcommands {
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Copies the MDBX database to a new location (bundled mdbx_copy)
|
||||
Copy(copy::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
/// Gets the content of a table for the given key
|
||||
@@ -64,6 +69,10 @@ pub enum Subcommands {
|
||||
Path,
|
||||
/// Manage storage settings
|
||||
Settings(settings::Command),
|
||||
/// View or set prune checkpoints
|
||||
PruneCheckpoints(prune_checkpoints::Command),
|
||||
// View or set stage checkpoints
|
||||
StageCheckpoints(stage_checkpoints::Command),
|
||||
/// Gets storage size information for an account
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
@@ -80,7 +89,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
/// provided command.
|
||||
macro_rules! db_exec {
|
||||
($env:expr, $tool:ident, $N:ident, $access_rights:expr, $command:block) => {
|
||||
let Environment { provider_factory, .. } = $env.init::<$N>($access_rights)?;
|
||||
let Environment { provider_factory, .. } =
|
||||
$env.init::<$N>($access_rights, ctx.task_executor.clone())?;
|
||||
|
||||
let $tool = DbTool::new(provider_factory)?;
|
||||
$command;
|
||||
@@ -124,6 +134,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::Copy(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(tool.provider_factory.db_ref())?;
|
||||
});
|
||||
}
|
||||
Subcommands::Diff(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
@@ -196,6 +211,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::PruneCheckpoints(command) => {
|
||||
db_exec!(self.env, tool, N, command.access_rights(), {
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::StageCheckpoints(command) => {
|
||||
db_exec!(self.env, tool, N, command.access_rights(), {
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::AccountStorage(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
|
||||
221
crates/cli/commands/src/db/prune_checkpoints.rs
Normal file
221
crates/cli/commands/src/db/prune_checkpoints.rs
Normal file
@@ -0,0 +1,221 @@
|
||||
//! `reth db prune-checkpoints` command for viewing and setting prune checkpoint values.
|
||||
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_provider::{providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment};
|
||||
use reth_storage_api::{PruneCheckpointReader, PruneCheckpointWriter};
|
||||
|
||||
use crate::common::AccessRights;
|
||||
|
||||
/// `reth db prune-checkpoints` subcommand
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
command: Subcommands,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Returns database access rights required for the command.
|
||||
pub fn access_rights(&self) -> AccessRights {
|
||||
match &self.command {
|
||||
Subcommands::Get { .. } => AccessRights::RO,
|
||||
Subcommands::Set(_) => AccessRights::RW,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Subcommands {
|
||||
/// Get prune checkpoint(s) from database.
|
||||
///
|
||||
/// Shows the current prune progress for each segment, including the highest
|
||||
/// pruned block/tx number and the active prune mode.
|
||||
Get {
|
||||
/// Specific segment to query. If omitted, shows all segments.
|
||||
#[arg(long, value_enum)]
|
||||
segment: Option<SegmentArg>,
|
||||
},
|
||||
/// Set a prune checkpoint for a segment.
|
||||
///
|
||||
/// WARNING: Manually setting checkpoints can cause data inconsistencies.
|
||||
/// Only use this if you know what you're doing (e.g., recovering from a
|
||||
/// corrupted checkpoint or forcing a re-prune from a specific block).
|
||||
Set(SetArgs),
|
||||
}
|
||||
|
||||
/// Arguments for the `set` subcommand
|
||||
#[derive(Debug, Args)]
|
||||
pub struct SetArgs {
|
||||
/// The prune segment to update
|
||||
#[arg(long, value_enum)]
|
||||
segment: SegmentArg,
|
||||
|
||||
/// Highest pruned block number
|
||||
#[arg(long)]
|
||||
block_number: Option<u64>,
|
||||
|
||||
/// Highest pruned transaction number
|
||||
#[arg(long)]
|
||||
tx_number: Option<u64>,
|
||||
|
||||
/// Prune mode to write: full, distance, or before
|
||||
#[arg(long, value_enum)]
|
||||
mode: PruneModeArg,
|
||||
|
||||
/// Value for distance or before mode (required unless mode is full)
|
||||
#[arg(long, required_if_eq_any([("mode", "distance"), ("mode", "before")]))]
|
||||
mode_value: Option<u64>,
|
||||
}
|
||||
|
||||
/// CLI-friendly prune segment names (excludes deprecated variants)
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "kebab-case")]
|
||||
pub enum SegmentArg {
|
||||
SenderRecovery,
|
||||
TransactionLookup,
|
||||
Receipts,
|
||||
ContractLogs,
|
||||
AccountHistory,
|
||||
StorageHistory,
|
||||
Bodies,
|
||||
}
|
||||
|
||||
impl From<SegmentArg> for PruneSegment {
|
||||
fn from(arg: SegmentArg) -> Self {
|
||||
match arg {
|
||||
SegmentArg::SenderRecovery => Self::SenderRecovery,
|
||||
SegmentArg::TransactionLookup => Self::TransactionLookup,
|
||||
SegmentArg::Receipts => Self::Receipts,
|
||||
SegmentArg::ContractLogs => Self::ContractLogs,
|
||||
SegmentArg::AccountHistory => Self::AccountHistory,
|
||||
SegmentArg::StorageHistory => Self::StorageHistory,
|
||||
SegmentArg::Bodies => Self::Bodies,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// CLI-friendly prune mode
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "kebab-case")]
|
||||
pub enum PruneModeArg {
|
||||
/// Prune all blocks
|
||||
Full,
|
||||
/// Keep the last N blocks (requires --mode-value)
|
||||
Distance,
|
||||
/// Prune blocks before a specific block number (requires --mode-value)
|
||||
Before,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the command
|
||||
pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
match self.command {
|
||||
Subcommands::Get { segment } => Self::get(tool, segment),
|
||||
Subcommands::Set(args) => Self::set(tool, args),
|
||||
}
|
||||
}
|
||||
|
||||
fn get<N: ProviderNodeTypes>(
|
||||
tool: &DbTool<N>,
|
||||
segment: Option<SegmentArg>,
|
||||
) -> eyre::Result<()> {
|
||||
let provider = tool.provider_factory.provider()?;
|
||||
|
||||
match segment {
|
||||
Some(seg) => {
|
||||
let segment: PruneSegment = seg.into();
|
||||
match provider.get_prune_checkpoint(segment)? {
|
||||
Some(checkpoint) => print_checkpoint(segment, &checkpoint),
|
||||
None => println!("No checkpoint found for {segment}"),
|
||||
}
|
||||
}
|
||||
None => {
|
||||
let mut checkpoints = provider.get_prune_checkpoints()?;
|
||||
checkpoints.sort_by_key(|(seg, _)| *seg);
|
||||
if checkpoints.is_empty() {
|
||||
println!("No prune checkpoints found.");
|
||||
} else {
|
||||
println!(
|
||||
"{:<25} {:>15} {:>15} {:>20}",
|
||||
"Segment", "Block Number", "Tx Number", "Prune Mode"
|
||||
);
|
||||
println!("{}", "-".repeat(80));
|
||||
for (segment, checkpoint) in &checkpoints {
|
||||
println!(
|
||||
"{:<25} {:>15} {:>15} {:>20}",
|
||||
segment.to_string(),
|
||||
fmt_opt(checkpoint.block_number),
|
||||
fmt_opt(checkpoint.tx_number),
|
||||
fmt_mode(&checkpoint.prune_mode),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set<N: ProviderNodeTypes>(tool: &DbTool<N>, args: SetArgs) -> eyre::Result<()> {
|
||||
eyre::ensure!(
|
||||
args.block_number.is_some() || args.tx_number.is_some(),
|
||||
"at least one of --block-number or --tx-number must be provided"
|
||||
);
|
||||
|
||||
let prune_mode = match args.mode {
|
||||
PruneModeArg::Full => PruneMode::Full,
|
||||
PruneModeArg::Distance => PruneMode::Distance(
|
||||
args.mode_value
|
||||
.ok_or_else(|| eyre::eyre!("--mode-value is required for distance mode"))?,
|
||||
),
|
||||
PruneModeArg::Before => PruneMode::Before(
|
||||
args.mode_value
|
||||
.ok_or_else(|| eyre::eyre!("--mode-value is required for before mode"))?,
|
||||
),
|
||||
};
|
||||
|
||||
let segment: PruneSegment = args.segment.into();
|
||||
let checkpoint = PruneCheckpoint {
|
||||
block_number: args.block_number,
|
||||
tx_number: args.tx_number,
|
||||
prune_mode,
|
||||
};
|
||||
|
||||
let provider_rw = tool.provider_factory.database_provider_rw()?;
|
||||
|
||||
// Show previous value if any
|
||||
if let Some(prev) = provider_rw.get_prune_checkpoint(segment)? {
|
||||
println!("Previous checkpoint for {segment}:");
|
||||
print_checkpoint(segment, &prev);
|
||||
println!();
|
||||
}
|
||||
|
||||
provider_rw.save_prune_checkpoint(segment, checkpoint)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
println!("Updated checkpoint for {segment}:");
|
||||
print_checkpoint(segment, &checkpoint);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn print_checkpoint(segment: PruneSegment, checkpoint: &PruneCheckpoint) {
|
||||
println!(" Segment: {segment}");
|
||||
println!(" Block Number: {}", fmt_opt(checkpoint.block_number));
|
||||
println!(" Tx Number: {}", fmt_opt(checkpoint.tx_number));
|
||||
println!(" Prune Mode: {}", fmt_mode(&checkpoint.prune_mode));
|
||||
}
|
||||
|
||||
fn fmt_opt(val: Option<u64>) -> String {
|
||||
val.map_or("-".to_string(), |n| n.to_string())
|
||||
}
|
||||
|
||||
fn fmt_mode(mode: &PruneMode) -> String {
|
||||
match mode {
|
||||
PruneMode::Full => "Full".to_string(),
|
||||
PruneMode::Distance(d) => format!("Distance({d})"),
|
||||
PruneMode::Before(b) => format!("Before({b})"),
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,6 @@ use reth_cli_util::parse_socket_address;
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
|
||||
database::Database,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
@@ -21,13 +20,15 @@ use reth_node_metrics::{
|
||||
};
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChainSpecProvider, StageCheckpointReader};
|
||||
use reth_stages::StageId;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use reth_tasks::TaskExecutor;
|
||||
use reth_trie::{
|
||||
verify::{Output, Verifier},
|
||||
Nibbles,
|
||||
};
|
||||
use reth_trie_common::{StorageTrieEntry, StoredNibbles, StoredNibblesSubKey};
|
||||
use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory};
|
||||
use reth_trie_db::{
|
||||
DatabaseHashedCursorFactory, DatabaseTrieCursorFactory, StorageTrieEntryLike, TrieTableAdapter,
|
||||
};
|
||||
use std::{
|
||||
net::SocketAddr,
|
||||
time::{Duration, Instant},
|
||||
@@ -116,9 +117,13 @@ fn verify_only<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
let mut tx = db.tx()?;
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
reth_trie_db::with_adapter!(tool.provider_factory, |A| do_verify_only::<_, A>(&tx))
|
||||
}
|
||||
|
||||
fn do_verify_only<TX: DbTx, A: TrieTableAdapter>(tx: &TX) -> eyre::Result<()> {
|
||||
// Create the verifier
|
||||
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(&tx);
|
||||
let trie_cursor_factory = DatabaseTrieCursorFactory::new(&tx);
|
||||
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx);
|
||||
let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx);
|
||||
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
|
||||
|
||||
let metrics = RepairTrieMetrics::new();
|
||||
@@ -209,17 +214,37 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
// Check that a pipeline sync isn't in progress.
|
||||
verify_checkpoints(provider_rw.as_ref())?;
|
||||
|
||||
let inconsistent_nodes = reth_trie_db::with_adapter!(tool.provider_factory, |A| {
|
||||
do_verify_and_repair::<_, A>(&mut provider_rw)?
|
||||
});
|
||||
|
||||
if inconsistent_nodes == 0 {
|
||||
info!("No inconsistencies found");
|
||||
} else {
|
||||
provider_rw.commit()?;
|
||||
info!("Repaired {} inconsistencies and committed changes", inconsistent_nodes);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn do_verify_and_repair<N: ProviderNodeTypes, A: TrieTableAdapter>(
|
||||
provider_rw: &mut reth_provider::DatabaseProviderRW<N::DB, N>,
|
||||
) -> eyre::Result<usize>
|
||||
where
|
||||
<N::DB as reth_db_api::database::Database>::TXMut: DbTxMut + DbTx,
|
||||
{
|
||||
// Create cursors for making modifications with
|
||||
let tx = provider_rw.tx_mut();
|
||||
tx.disable_long_read_transaction_safety();
|
||||
let mut account_trie_cursor = tx.cursor_write::<tables::AccountsTrie>()?;
|
||||
let mut storage_trie_cursor = tx.cursor_dup_write::<tables::StoragesTrie>()?;
|
||||
let mut account_trie_cursor = tx.cursor_write::<A::AccountTrieTable>()?;
|
||||
let mut storage_trie_cursor = tx.cursor_dup_write::<A::StorageTrieTable>()?;
|
||||
|
||||
// Create the cursor factories. These cannot accept the `&mut` tx above because they require it
|
||||
// to be AsRef.
|
||||
// Create the cursor factories. These cannot accept the `&mut` tx above because they
|
||||
// require it to be AsRef.
|
||||
let tx = provider_rw.tx_ref();
|
||||
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx);
|
||||
let trie_cursor_factory = DatabaseTrieCursorFactory::new(tx);
|
||||
let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx);
|
||||
|
||||
// Create the verifier
|
||||
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
|
||||
@@ -257,17 +282,17 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
match output {
|
||||
Output::AccountExtra(path, _node) => {
|
||||
// Extra account node in trie, remove it
|
||||
let nibbles = StoredNibbles(path);
|
||||
if account_trie_cursor.seek_exact(nibbles)?.is_some() {
|
||||
let key: A::AccountKey = path.into();
|
||||
if account_trie_cursor.seek_exact(key)?.is_some() {
|
||||
account_trie_cursor.delete_current()?;
|
||||
}
|
||||
}
|
||||
Output::StorageExtra(account, path, _node) => {
|
||||
// Extra storage node in trie, remove it
|
||||
let nibbles = StoredNibblesSubKey(path);
|
||||
let subkey: A::StorageSubKey = path.into();
|
||||
if storage_trie_cursor
|
||||
.seek_by_key_subkey(account, nibbles.clone())?
|
||||
.filter(|e| e.nibbles == nibbles)
|
||||
.seek_by_key_subkey(account, subkey.clone())?
|
||||
.filter(|e| *e.nibbles() == subkey)
|
||||
.is_some()
|
||||
{
|
||||
storage_trie_cursor.delete_current()?;
|
||||
@@ -276,19 +301,19 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
Output::AccountWrong { path, expected: node, .. } |
|
||||
Output::AccountMissing(path, node) => {
|
||||
// Wrong/missing account node value, upsert it
|
||||
let nibbles = StoredNibbles(path);
|
||||
account_trie_cursor.upsert(nibbles, &node)?;
|
||||
let key: A::AccountKey = path.into();
|
||||
account_trie_cursor.upsert(key, &node)?;
|
||||
}
|
||||
Output::StorageWrong { account, path, expected: node, .. } |
|
||||
Output::StorageMissing(account, path, node) => {
|
||||
// Wrong/missing storage node value, upsert it
|
||||
// (We can't just use `upsert` method with a dup cursor, it's not properly
|
||||
// supported)
|
||||
let nibbles = StoredNibblesSubKey(path);
|
||||
let entry = StorageTrieEntry { nibbles: nibbles.clone(), node };
|
||||
let subkey: A::StorageSubKey = path.into();
|
||||
let entry = A::StorageValue::new(subkey.clone(), node);
|
||||
if storage_trie_cursor
|
||||
.seek_by_key_subkey(account, nibbles.clone())?
|
||||
.filter(|v| v.nibbles == nibbles)
|
||||
.seek_by_key_subkey(account, subkey.clone())?
|
||||
.filter(|v| *v.nibbles() == subkey)
|
||||
.is_some()
|
||||
{
|
||||
storage_trie_cursor.delete_current()?;
|
||||
@@ -304,14 +329,7 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
}
|
||||
}
|
||||
|
||||
if inconsistent_nodes == 0 {
|
||||
info!("No inconsistencies found");
|
||||
} else {
|
||||
provider_rw.commit()?;
|
||||
info!("Repaired {} inconsistencies and committed changes", inconsistent_nodes);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
Ok(inconsistent_nodes as usize)
|
||||
}
|
||||
|
||||
/// Output progress information based on the last seen account path.
|
||||
|
||||
@@ -39,50 +39,12 @@ enum Subcommands {
|
||||
#[derive(Debug, Clone, Copy, Subcommand)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
pub enum SetCommand {
|
||||
/// Store receipts in static files instead of the database
|
||||
Receipts {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction senders in static files instead of the database
|
||||
TransactionSenders {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account changesets in static files instead of the database
|
||||
AccountChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage history in rocksdb instead of MDBX
|
||||
StoragesHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
TransactionHashNumbers {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account history in rocksdb instead of MDBX
|
||||
AccountHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Use hashed state tables (HashedAccounts/HashedStorages) as canonical state
|
||||
/// Enable or disable v2 storage layout
|
||||
///
|
||||
/// When enabled, execution writes directly to hashed tables, eliminating need for
|
||||
/// separate hashing stages. State reads come from hashed tables.
|
||||
///
|
||||
/// WARNING: Changing this setting in either direction requires re-syncing the database.
|
||||
/// Enabling on an existing plain-state database leaves hashed tables empty.
|
||||
/// Disabling on an existing hashed-state database leaves plain tables empty.
|
||||
UseHashedState {
|
||||
/// When enabled, uses static files for receipts/senders/changesets and RocksDB for
|
||||
/// history indices and transaction hashes. When disabled, uses v1/legacy layout (everything in
|
||||
/// MDBX).
|
||||
V2 {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
@@ -125,87 +87,18 @@ impl Command {
|
||||
println!("No storage settings found, creating new settings.");
|
||||
}
|
||||
|
||||
let mut settings @ StorageSettings {
|
||||
receipts_in_static_files: _,
|
||||
transaction_senders_in_static_files: _,
|
||||
storages_history_in_rocksdb: _,
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
use_hashed_state: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::v1);
|
||||
let mut settings @ StorageSettings { storage_v2: _ } =
|
||||
settings.unwrap_or_else(StorageSettings::v1);
|
||||
|
||||
// Update the setting based on the key
|
||||
match cmd {
|
||||
SetCommand::Receipts { value } => {
|
||||
if settings.receipts_in_static_files == value {
|
||||
println!("receipts_in_static_files is already set to {}", value);
|
||||
SetCommand::V2 { value } => {
|
||||
if settings.storage_v2 == value {
|
||||
println!("storage_v2 is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.receipts_in_static_files = value;
|
||||
println!("Set receipts_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::TransactionSenders { value } => {
|
||||
if settings.transaction_senders_in_static_files == value {
|
||||
println!("transaction_senders_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_senders_in_static_files = value;
|
||||
println!("Set transaction_senders_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::AccountChangesets { value } => {
|
||||
if settings.account_changesets_in_static_files == value {
|
||||
println!("account_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_changesets_in_static_files = value;
|
||||
println!("Set account_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::StoragesHistory { value } => {
|
||||
if settings.storages_history_in_rocksdb == value {
|
||||
println!("storages_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storages_history_in_rocksdb = value;
|
||||
println!("Set storages_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::TransactionHashNumbers { value } => {
|
||||
if settings.transaction_hash_numbers_in_rocksdb == value {
|
||||
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_hash_numbers_in_rocksdb = value;
|
||||
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::AccountHistory { value } => {
|
||||
if settings.account_history_in_rocksdb == value {
|
||||
println!("account_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::UseHashedState { value } => {
|
||||
if settings.use_hashed_state == value {
|
||||
println!("use_hashed_state is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
if settings.use_hashed_state && !value {
|
||||
println!("WARNING: Disabling use_hashed_state on an existing hashed-state database requires a full resync.");
|
||||
} else {
|
||||
println!("WARNING: Enabling use_hashed_state on an existing plain-state database requires a full resync.");
|
||||
}
|
||||
settings.use_hashed_state = value;
|
||||
println!("Set use_hashed_state = {}", value);
|
||||
settings.storage_v2 = value;
|
||||
println!("Set storage_v2 = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
297
crates/cli/commands/src/db/stage_checkpoints.rs
Normal file
297
crates/cli/commands/src/db/stage_checkpoints.rs
Normal file
@@ -0,0 +1,297 @@
|
||||
//! `reth db stage-checkpoints` command for viewing and setting stage checkpoint values.
|
||||
|
||||
use clap::{Args, Parser, Subcommand, ValueEnum};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
|
||||
StageCheckpointWriter,
|
||||
};
|
||||
use reth_stages::StageId;
|
||||
|
||||
use crate::common::AccessRights;
|
||||
|
||||
/// `reth db stage-checkpoints` subcommand
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
#[command(subcommand)]
|
||||
command: Subcommands,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Returns database access rights required for the command.
|
||||
pub fn access_rights(&self) -> AccessRights {
|
||||
match &self.command {
|
||||
Subcommands::Get { .. } => AccessRights::RO,
|
||||
Subcommands::Set(_) => AccessRights::RW,
|
||||
}
|
||||
}
|
||||
|
||||
/// Execute the command
|
||||
pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
match self.command {
|
||||
Subcommands::Get { stage } => Self::get(tool, stage),
|
||||
Subcommands::Set(args) => Self::set(tool, args),
|
||||
}
|
||||
}
|
||||
|
||||
fn get<N: ProviderNodeTypes>(tool: &DbTool<N>, stage: Option<StageArg>) -> eyre::Result<()> {
|
||||
let provider = tool.provider_factory.provider()?;
|
||||
|
||||
match stage {
|
||||
Some(stage) => {
|
||||
let stage_id = stage.into();
|
||||
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
|
||||
println!("{stage_id}: {checkpoint:?}");
|
||||
}
|
||||
None => {
|
||||
let mut checkpoints = provider.get_all_checkpoints()?;
|
||||
checkpoints.sort_by(|a, b| a.0.cmp(&b.0));
|
||||
for (stage, checkpoint) in checkpoints {
|
||||
println!("{stage}: {checkpoint:?}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set<N: ProviderNodeTypes>(tool: &DbTool<N>, args: SetArgs) -> eyre::Result<()> {
|
||||
let stage_id: StageId = args.stage.into();
|
||||
let provider_rw = tool.provider_factory.database_provider_rw()?;
|
||||
|
||||
let previous = provider_rw.get_stage_checkpoint(stage_id)?;
|
||||
let mut checkpoint = previous.unwrap_or_default();
|
||||
checkpoint.block_number = args.block_number;
|
||||
|
||||
if args.clear_stage_unit {
|
||||
checkpoint.stage_checkpoint = None;
|
||||
}
|
||||
|
||||
provider_rw.save_stage_checkpoint(stage_id, checkpoint)?;
|
||||
|
||||
provider_rw.commit()?;
|
||||
|
||||
println!("Updated checkpoint for {stage_id}: {checkpoint:?}");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Subcommands {
|
||||
/// Get stage checkpoint(s) from database.
|
||||
Get {
|
||||
/// Specific stage to query. If omitted, shows all stages.
|
||||
#[arg(long, value_enum)]
|
||||
stage: Option<StageArg>,
|
||||
},
|
||||
/// Set a stage checkpoint.
|
||||
Set(SetArgs),
|
||||
}
|
||||
|
||||
/// Arguments for the `set` subcommand.
|
||||
#[derive(Debug, Args)]
|
||||
pub struct SetArgs {
|
||||
/// Stage to update.
|
||||
#[arg(long, value_enum)]
|
||||
stage: StageArg,
|
||||
|
||||
/// Block number to set as stage checkpoint.
|
||||
#[arg(long)]
|
||||
block_number: u64,
|
||||
|
||||
/// Clear stage-specific unit checkpoint payload.
|
||||
#[arg(long)]
|
||||
clear_stage_unit: bool,
|
||||
}
|
||||
|
||||
/// CLI-friendly stage names.
|
||||
#[derive(Debug, Clone, Copy, ValueEnum)]
|
||||
#[clap(rename_all = "kebab-case")]
|
||||
pub enum StageArg {
|
||||
Era,
|
||||
Headers,
|
||||
Bodies,
|
||||
SenderRecovery,
|
||||
Execution,
|
||||
PruneSenderRecovery,
|
||||
MerkleUnwind,
|
||||
AccountHashing,
|
||||
StorageHashing,
|
||||
MerkleExecute,
|
||||
TransactionLookup,
|
||||
IndexStorageHistory,
|
||||
IndexAccountHistory,
|
||||
Prune,
|
||||
Finish,
|
||||
}
|
||||
|
||||
impl From<StageArg> for StageId {
|
||||
fn from(arg: StageArg) -> Self {
|
||||
match arg {
|
||||
StageArg::Era => Self::Era,
|
||||
StageArg::Headers => Self::Headers,
|
||||
StageArg::Bodies => Self::Bodies,
|
||||
StageArg::SenderRecovery => Self::SenderRecovery,
|
||||
StageArg::Execution => Self::Execution,
|
||||
StageArg::PruneSenderRecovery => Self::PruneSenderRecovery,
|
||||
StageArg::MerkleUnwind => Self::MerkleUnwind,
|
||||
StageArg::AccountHashing => Self::AccountHashing,
|
||||
StageArg::StorageHashing => Self::StorageHashing,
|
||||
StageArg::MerkleExecute => Self::MerkleExecute,
|
||||
StageArg::TransactionLookup => Self::TransactionLookup,
|
||||
StageArg::IndexStorageHistory => Self::IndexStorageHistory,
|
||||
StageArg::IndexAccountHistory => Self::IndexAccountHistory,
|
||||
StageArg::Prune => Self::Prune,
|
||||
StageArg::Finish => Self::Finish,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
use reth_provider::{
|
||||
test_utils::create_test_provider_factory, DBProvider, DatabaseProviderFactory,
|
||||
StageCheckpointReader, StageCheckpointWriter,
|
||||
};
|
||||
use reth_stages::StageCheckpoint;
|
||||
|
||||
#[test]
|
||||
fn parse_set_args() {
|
||||
let command = Command::parse_from([
|
||||
"stage-checkpoints",
|
||||
"set",
|
||||
"--stage",
|
||||
"headers",
|
||||
"--block-number",
|
||||
"123",
|
||||
]);
|
||||
|
||||
assert!(matches!(
|
||||
command.command,
|
||||
Subcommands::Set(SetArgs {
|
||||
stage: StageArg::Headers,
|
||||
block_number: 123,
|
||||
clear_stage_unit: false,
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_overwrites_block_number() {
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
|
||||
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
|
||||
provider_rw
|
||||
.save_stage_checkpoint(StageId::Headers, StageCheckpoint::new(10))
|
||||
.expect("save checkpoint");
|
||||
provider_rw.commit().expect("commit initial checkpoint");
|
||||
}
|
||||
|
||||
let command = Command {
|
||||
command: Subcommands::Set(SetArgs {
|
||||
stage: StageArg::Headers,
|
||||
block_number: 42,
|
||||
clear_stage_unit: false,
|
||||
}),
|
||||
};
|
||||
|
||||
command.execute(&tool).expect("execute command");
|
||||
|
||||
let provider = provider_factory.provider().expect("provider");
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::Headers)
|
||||
.expect("get stage checkpoint")
|
||||
.expect("missing stage checkpoint");
|
||||
|
||||
assert_eq!(checkpoint.block_number, 42);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_preserves_stage_unit_checkpoint_unless_cleared() {
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
|
||||
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
|
||||
let checkpoint = StageCheckpoint::new(10).with_block_range(&StageId::Execution, 5, 10);
|
||||
provider_rw
|
||||
.save_stage_checkpoint(StageId::Execution, checkpoint)
|
||||
.expect("save checkpoint");
|
||||
provider_rw.commit().expect("commit initial checkpoint");
|
||||
}
|
||||
|
||||
Command {
|
||||
command: Subcommands::Set(SetArgs {
|
||||
stage: StageArg::Execution,
|
||||
block_number: 11,
|
||||
clear_stage_unit: false,
|
||||
}),
|
||||
}
|
||||
.execute(&tool)
|
||||
.expect("execute command");
|
||||
|
||||
let provider = provider_factory.provider().expect("provider");
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::Execution)
|
||||
.expect("get stage checkpoint")
|
||||
.expect("missing stage checkpoint");
|
||||
assert!(checkpoint.stage_checkpoint.is_some());
|
||||
|
||||
Command {
|
||||
command: Subcommands::Set(SetArgs {
|
||||
stage: StageArg::Execution,
|
||||
block_number: 12,
|
||||
clear_stage_unit: true,
|
||||
}),
|
||||
}
|
||||
.execute(&tool)
|
||||
.expect("execute command");
|
||||
|
||||
let checkpoint = provider_factory
|
||||
.provider()
|
||||
.expect("provider")
|
||||
.get_stage_checkpoint(StageId::Execution)
|
||||
.expect("get stage checkpoint")
|
||||
.expect("missing stage checkpoint");
|
||||
assert!(checkpoint.stage_checkpoint.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_preserves_checkpoint_progress() {
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
|
||||
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
|
||||
provider_rw
|
||||
.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::new(10))
|
||||
.expect("save checkpoint");
|
||||
provider_rw
|
||||
.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![1, 2, 3])
|
||||
.expect("save progress");
|
||||
provider_rw.commit().expect("commit initial checkpoint");
|
||||
}
|
||||
|
||||
Command {
|
||||
command: Subcommands::Set(SetArgs {
|
||||
stage: StageArg::MerkleExecute,
|
||||
block_number: 20,
|
||||
clear_stage_unit: false,
|
||||
}),
|
||||
}
|
||||
.execute(&tool)
|
||||
.expect("execute command");
|
||||
|
||||
let provider = provider_factory.provider().expect("provider");
|
||||
let progress = provider
|
||||
.get_stage_checkpoint_progress(StageId::MerkleExecute)
|
||||
.expect("get stage checkpoint progress");
|
||||
|
||||
assert_eq!(progress, Some(vec![1, 2, 3]));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use alloy_primitives::{keccak256, Address, BlockNumber, B256, U256};
|
||||
use clap::Parser;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::{
|
||||
@@ -63,39 +63,65 @@ impl Command {
|
||||
address: Address,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let entries = tool.provider_factory.db_ref().view(|tx| {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
let (account, walker_entries) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
let account = tx.get::<tables::HashedAccounts>(hashed_address)?;
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
(account, entries)
|
||||
} else {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
(account, entries)
|
||||
};
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>((account, entries))
|
||||
Ok::<_, eyre::Report>((account, walker_entries))
|
||||
})??;
|
||||
|
||||
let (account, storage_entries) = entries;
|
||||
@@ -119,7 +145,7 @@ impl Command {
|
||||
|
||||
// Check storage settings to determine where history is stored
|
||||
let storage_settings = tool.provider_factory.cached_storage_settings();
|
||||
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
|
||||
let history_in_rocksdb = storage_settings.storage_v2;
|
||||
|
||||
// For historical queries, enumerate keys from history indices only
|
||||
// (not PlainStorageState, which reflects current state)
|
||||
|
||||
@@ -297,21 +297,18 @@ where
|
||||
}
|
||||
|
||||
match event {
|
||||
Event::Key(key) => {
|
||||
if key.kind == event::KeyEventKind::Press {
|
||||
match key.code {
|
||||
KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(true),
|
||||
KeyCode::Down => app.next(),
|
||||
KeyCode::Up => app.previous(),
|
||||
KeyCode::Right => app.next_page(),
|
||||
KeyCode::Left => app.previous_page(),
|
||||
KeyCode::Char('G') => {
|
||||
app.mode = ViewMode::GoToPage;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
Event::Key(key) if key.kind == event::KeyEventKind::Press => match key.code {
|
||||
KeyCode::Char('q') | KeyCode::Char('Q') => return Ok(true),
|
||||
KeyCode::Down => app.next(),
|
||||
KeyCode::Up => app.previous(),
|
||||
KeyCode::Right => app.next_page(),
|
||||
KeyCode::Left => app.previous_page(),
|
||||
KeyCode::Char('G') => {
|
||||
app.mode = ViewMode::GoToPage;
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
},
|
||||
Event::Key(_) => {}
|
||||
Event::Mouse(e) => match e.kind {
|
||||
MouseEventKind::ScrollDown => app.next(),
|
||||
MouseEventKind::ScrollUp => app.previous(),
|
||||
|
||||
@@ -37,6 +37,14 @@ pub struct DownloadDefaults {
|
||||
pub available_snapshots: Vec<Cow<'static, str>>,
|
||||
/// Default base URL for snapshots
|
||||
pub default_base_url: Cow<'static, str>,
|
||||
/// Default base URL for chain-aware snapshots.
|
||||
///
|
||||
/// When set, the chain ID is appended to form the full URL: `{base_url}/{chain_id}`.
|
||||
/// For example, given a base URL of `https://snapshots.example.com` and chain ID `1`,
|
||||
/// the resulting URL would be `https://snapshots.example.com/1`.
|
||||
///
|
||||
/// Falls back to [`default_base_url`](Self::default_base_url) when `None`.
|
||||
pub default_chain_aware_base_url: Option<Cow<'static, str>>,
|
||||
/// Optional custom long help text that overrides the generated help
|
||||
pub long_help: Option<String>,
|
||||
}
|
||||
@@ -60,6 +68,7 @@ impl DownloadDefaults {
|
||||
Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
|
||||
],
|
||||
default_base_url: Cow::Borrowed(MERKLE_BASE_URL),
|
||||
default_chain_aware_base_url: None,
|
||||
long_help: None,
|
||||
}
|
||||
}
|
||||
@@ -84,9 +93,11 @@ impl DownloadDefaults {
|
||||
}
|
||||
|
||||
help.push_str(
|
||||
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
|
||||
"\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(
|
||||
self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
|
||||
);
|
||||
help.push_str(self.default_base_url.as_ref());
|
||||
help.push_str(
|
||||
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
|
||||
);
|
||||
@@ -111,6 +122,12 @@ impl DownloadDefaults {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default chain-aware base URL.
|
||||
pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
|
||||
self.default_chain_aware_base_url = Some(url.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder: Set custom long help text, overriding the generated help
|
||||
pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
|
||||
self.long_help = Some(help.into());
|
||||
@@ -142,7 +159,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
|
||||
let url = match self.url {
|
||||
Some(url) => url,
|
||||
None => {
|
||||
let url = get_latest_snapshot_url().await?;
|
||||
let url = get_latest_snapshot_url(self.env.chain.chain().id()).await?;
|
||||
info!(target: "reth::cli", "Using default snapshot URL: {}", url);
|
||||
url
|
||||
}
|
||||
@@ -367,15 +384,19 @@ fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
let mut total_size: Option<u64> = None;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
let finalize_download = |size: u64| -> Result<(PathBuf, u64)> {
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
Ok((final_path.clone(), size))
|
||||
};
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
if let Some(total) = total_size &&
|
||||
existing_size >= total
|
||||
{
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, total));
|
||||
return finalize_download(total);
|
||||
}
|
||||
|
||||
if attempt > 1 {
|
||||
@@ -459,9 +480,7 @@ fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
continue;
|
||||
}
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, current_total));
|
||||
return finalize_download(current_total);
|
||||
}
|
||||
|
||||
Err(last_error
|
||||
@@ -509,8 +528,12 @@ async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
}
|
||||
|
||||
// Builds default URL for latest mainnet archive snapshot using configured defaults
|
||||
async fn get_latest_snapshot_url() -> Result<String> {
|
||||
let base_url = &DownloadDefaults::get_global().default_base_url;
|
||||
async fn get_latest_snapshot_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let base_url = match &defaults.default_chain_aware_base_url {
|
||||
Some(url) => format!("{url}/{chain_id}"),
|
||||
None => defaults.default_base_url.to_string(),
|
||||
};
|
||||
let latest_url = format!("{base_url}/latest.txt");
|
||||
let filename = Client::new()
|
||||
.get(latest_url)
|
||||
|
||||
@@ -44,11 +44,11 @@ pub struct ExportArgs {
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ExportEraCommand<C> {
|
||||
/// Execute `export-era` command
|
||||
pub async fn execute<N>(self) -> eyre::Result<()>
|
||||
pub async fn execute<N>(self, runtime: reth_tasks::Runtime) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO)?;
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
|
||||
|
||||
// Either specified path or default to `<data-dir>/<chain>/era1-export/`
|
||||
let data_dir = match &self.export.path {
|
||||
|
||||
@@ -47,6 +47,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
pub async fn execute<N, Comp>(
|
||||
self,
|
||||
components: impl FnOnce(Arc<N::ChainSpec>) -> Comp,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
@@ -54,7 +55,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
{
|
||||
info!(target: "reth::cli", "reth {} starting", version_metadata().short_version);
|
||||
|
||||
let Environment { provider_factory, config, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
let Environment { provider_factory, config, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, runtime.clone())?;
|
||||
|
||||
let components = components(provider_factory.chain_spec());
|
||||
|
||||
@@ -85,6 +87,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
|
||||
&config,
|
||||
executor.clone(),
|
||||
consensus.clone(),
|
||||
runtime.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
|
||||
@@ -87,6 +87,7 @@ pub async fn import_blocks_from_file<N>(
|
||||
config: &Config,
|
||||
executor: impl ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
consensus: Arc<impl FullConsensus<N::Primitives> + 'static>,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<ImportResult>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
@@ -147,6 +148,7 @@ where
|
||||
static_file_producer.clone(),
|
||||
import_config.no_state,
|
||||
executor.clone(),
|
||||
runtime.clone(),
|
||||
)?;
|
||||
|
||||
// override the tip
|
||||
@@ -257,6 +259,7 @@ where
|
||||
///
|
||||
/// If configured to execute, all stages will run. Otherwise, only stages that don't require state
|
||||
/// will run.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub fn build_import_pipeline_impl<N, C, E>(
|
||||
config: &Config,
|
||||
provider_factory: ProviderFactory<N>,
|
||||
@@ -265,6 +268,7 @@ pub fn build_import_pipeline_impl<N, C, E>(
|
||||
static_file_producer: StaticFileProducer<ProviderFactory<N>>,
|
||||
disable_exec: bool,
|
||||
evm_config: E,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<(Pipeline<N>, impl futures::Stream<Item = NodeEvent<N::Primitives>> + use<N, C, E>)>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
@@ -283,7 +287,7 @@ where
|
||||
|
||||
let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers)
|
||||
.build(file_client.clone(), consensus.clone())
|
||||
.into_task();
|
||||
.into_task_with(&runtime);
|
||||
// TODO: The pipeline should correctly configure the downloader on its own.
|
||||
// Find the possibility to remove unnecessary pre-configuration.
|
||||
header_downloader.update_local_head(local_head);
|
||||
@@ -291,7 +295,7 @@ where
|
||||
|
||||
let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies)
|
||||
.build(file_client.clone(), consensus.clone(), provider_factory.clone())
|
||||
.into_task();
|
||||
.into_task_with(&runtime);
|
||||
// TODO: The pipeline should correctly configure the downloader on its own.
|
||||
// Find the possibility to remove unnecessary pre-configuration.
|
||||
body_downloader
|
||||
|
||||
@@ -64,13 +64,14 @@ impl TryFromChain for ChainKind {
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportEraCommand<C> {
|
||||
/// Execute `import-era` command
|
||||
pub async fn execute<N>(self) -> eyre::Result<()>
|
||||
pub async fn execute<N>(self, runtime: reth_tasks::Runtime) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
info!(target: "reth::cli", "reth {} starting", version_metadata().short_version);
|
||||
|
||||
let Environment { provider_factory, config, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
let Environment { provider_factory, config, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, runtime)?;
|
||||
|
||||
let mut hash_collector = Collector::new(config.stages.etl.file_size, config.stages.etl.dir);
|
||||
|
||||
|
||||
@@ -18,10 +18,13 @@ pub struct InitCommand<C: ChainSpecParser> {
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitCommand<C> {
|
||||
/// Execute the `init` command
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
|
||||
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
|
||||
self,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "reth init starting");
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RW, runtime)?;
|
||||
|
||||
let genesis_block_number = provider_factory.chain_spec().genesis_header().number();
|
||||
let hash = provider_factory
|
||||
|
||||
@@ -65,7 +65,7 @@ pub struct InitStateCommand<C: ChainSpecParser> {
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateCommand<C> {
|
||||
/// Execute the `init` command
|
||||
pub async fn execute<N>(self) -> eyre::Result<()>
|
||||
pub async fn execute<N>(self, runtime: reth_tasks::Runtime) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<
|
||||
ChainSpec = C::ChainSpec,
|
||||
@@ -74,7 +74,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
|
||||
{
|
||||
info!(target: "reth::cli", "Reth init-state starting");
|
||||
|
||||
let Environment { config, provider_factory, .. } = self.env.init::<N>(AccessRights::RW)?;
|
||||
let Environment { config, provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, runtime)?;
|
||||
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
|
||||
@@ -10,8 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
StorageArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -103,10 +103,6 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -119,8 +115,8 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten, next_help_heading = "Static Files")]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
/// All storage related arguments with --storage prefix
|
||||
#[command(flatten, next_help_heading = "Storage")]
|
||||
pub storage: StorageArgs,
|
||||
|
||||
/// Additional cli arguments
|
||||
@@ -175,7 +171,6 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
@@ -183,9 +178,6 @@ where
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -201,7 +193,6 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user