mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
784b4e9dd2 | ||
|
|
2badbe3f3f | ||
|
|
f2cada2660 | ||
|
|
a2006c5f86 | ||
|
|
a95cf8fe8e | ||
|
|
852551b790 | ||
|
|
564ffa5868 | ||
|
|
483eaae67c | ||
|
|
12891dd171 | ||
|
|
c1015022f5 | ||
|
|
f76a748d92 | ||
|
|
e3fe6326bc | ||
|
|
e3d520b24f | ||
|
|
9f29939ea1 | ||
|
|
10881d1c73 | ||
|
|
408593467b | ||
|
|
fb1039cf7a | ||
|
|
8caf8cdf11 | ||
|
|
1e8030ef28 | ||
|
|
ffa681795f | ||
|
|
f72c503d6f | ||
|
|
42890e6e7f | ||
|
|
cd7f43bdf0 | ||
|
|
e30e441ada | ||
|
|
121160d248 | ||
|
|
7ff78ca082 | ||
|
|
d7f56d509c | ||
|
|
3300e404cf | ||
|
|
77cb99fc78 | ||
|
|
66169c7e7c | ||
|
|
4f5fafc8f3 | ||
|
|
0b8e6c6ed3 | ||
|
|
3de7589755 |
4
.changelog/fast-waves-smile.md
Normal file
4
.changelog/fast-waves-smile.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved nightly Docker build failure Slack notification with more detailed formatting and context.
|
||||
4
.changelog/lively-clouds-bake.md
Normal file
4
.changelog/lively-clouds-bake.md
Normal file
@@ -0,0 +1,4 @@
|
||||
---
|
||||
---
|
||||
|
||||
Improved documentation overview page with better structure and clarity.
|
||||
5
.changelog/lively-foxes-play.md
Normal file
5
.changelog/lively-foxes-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-node-events: patch
|
||||
---
|
||||
|
||||
Updated consensus engine log message to be more accurate about received updates.
|
||||
9
.changelog/nice-trees-drink.md
Normal file
9
.changelog/nice-trees-drink.md
Normal file
@@ -0,0 +1,9 @@
|
||||
---
|
||||
reth-network-api: minor
|
||||
reth-network-types: minor
|
||||
reth-network: minor
|
||||
reth-node-core: minor
|
||||
reth: minor
|
||||
---
|
||||
|
||||
Added optional ENR fork ID enforcement to filter out peers from incompatible networks during peer discovery, controlled by the `--enforce-enr-fork-id` CLI flag.
|
||||
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@@ -38,7 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin
|
||||
crates/storage/nippy-jar/ @joshieDo @shekhirin
|
||||
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
|
||||
crates/storage/storage-api/ @joshieDo
|
||||
crates/tasks/ @mattsse
|
||||
crates/tasks/ @mattsse @DaniPopes
|
||||
crates/tokio-util/ @mattsse
|
||||
crates/tracing/ @mattsse @shekhirin
|
||||
crates/tracing-otlp/ @mattsse @Rjected
|
||||
|
||||
27
.github/workflows/docker.yml
vendored
27
.github/workflows/docker.yml
vendored
@@ -70,18 +70,27 @@ jobs:
|
||||
# Add 'latest' tag for non-RC releases
|
||||
if [[ ! "$VERSION" =~ -rc ]]; then
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
|
||||
{
|
||||
echo "ethereum_set<<EOF"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:${VERSION}"
|
||||
echo "ethereum.tags=${REGISTRY}/reth:latest"
|
||||
echo "EOF"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
|
||||
else
|
||||
# git-sha build
|
||||
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
@@ -97,7 +106,7 @@ jobs:
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
${{ steps.params.outputs.ethereum_set }}
|
||||
|
||||
- name: Verify image architectures
|
||||
env:
|
||||
@@ -117,6 +126,18 @@ jobs:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
SLACK_COLOR: danger
|
||||
SLACK_ICON_EMOJI: ":rotating_light:"
|
||||
SLACK_USERNAME: "GitHub Actions"
|
||||
SLACK_TITLE: ":rotating_light: Nightly Docker Build Failed"
|
||||
SLACK_MESSAGE: |
|
||||
The scheduled nightly Docker build failed.
|
||||
|
||||
*Commit:* `${{ github.sha }}`
|
||||
*Branch:* `${{ github.ref_name }}`
|
||||
*Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
|
||||
|
||||
*Action required:* Re-run the workflow or investigate the build failure.
|
||||
SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
|
||||
MSG_MINIMAL: true
|
||||
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
|
||||
|
||||
370
Cargo.lock
generated
370
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.10.2"
|
||||
version = "1.11.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -110,6 +110,7 @@ members = [
|
||||
"crates/storage/errors/",
|
||||
"crates/storage/libmdbx-rs/",
|
||||
"crates/storage/libmdbx-rs/mdbx-sys/",
|
||||
"crates/storage/mdbx-viz/",
|
||||
"crates/storage/nippy-jar/",
|
||||
"crates/storage/provider/",
|
||||
"crates/storage/storage-api/",
|
||||
@@ -377,6 +378,7 @@ reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" }
|
||||
reth-ipc = { path = "crates/rpc/ipc" }
|
||||
reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
|
||||
reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
|
||||
reth-mdbx-viz = { path = "crates/storage/mdbx-viz" }
|
||||
reth-metrics = { path = "crates/metrics" }
|
||||
reth-net-banlist = { path = "crates/net/banlist" }
|
||||
reth-net-nat = { path = "crates/net/nat" }
|
||||
|
||||
@@ -192,6 +192,15 @@ impl Command {
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
blocks_processed += 1;
|
||||
|
||||
let progress = match mode {
|
||||
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
|
||||
format!("{pct:.1}%")
|
||||
}
|
||||
};
|
||||
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
|
||||
@@ -153,6 +153,7 @@ impl Command {
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -206,6 +207,7 @@ impl Command {
|
||||
});
|
||||
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
@@ -249,8 +251,13 @@ impl Command {
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
|
||||
@@ -52,6 +52,7 @@ impl Command {
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -82,8 +83,8 @@ impl Command {
|
||||
}
|
||||
});
|
||||
|
||||
// put results in a summary vec so they can be printed at the end
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
@@ -105,7 +106,12 @@ impl Command {
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
info!(target: "reth-bench", %new_payload_result);
|
||||
blocks_processed += 1;
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
None => format!("{blocks_processed}"),
|
||||
};
|
||||
info!(target: "reth-bench", progress, %new_payload_result);
|
||||
|
||||
// current duration since the start of the benchmark minus the time
|
||||
// waiting for blocks
|
||||
|
||||
@@ -341,7 +341,8 @@ impl Command {
|
||||
};
|
||||
|
||||
let current_duration = total_benchmark_duration.elapsed();
|
||||
info!(target: "reth-bench", %combined_result);
|
||||
let progress = format!("{}/{}", i + 1, payloads.len());
|
||||
info!(target: "reth-bench", progress, %combined_result);
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
|
||||
@@ -20,6 +20,19 @@ impl BenchMode {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total number of blocks in the benchmark, if known.
|
||||
///
|
||||
/// For [`BenchMode::Range`] this is the length of the range.
|
||||
/// For [`BenchMode::Continuous`] the total is unbounded, so `None` is returned.
|
||||
pub const fn total_blocks(&self) -> Option<u64> {
|
||||
match self {
|
||||
Self::Continuous(_) => None,
|
||||
Self::Range(range) => {
|
||||
Some(range.end().saturating_sub(*range.start()).saturating_add(1))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a [`BenchMode`] from optional `from` and `to` fields.
|
||||
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
|
||||
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,
|
||||
|
||||
@@ -190,6 +190,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
pageviz = ["reth-db/pageviz", "reth-node-builder/pageviz"]
|
||||
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
|
||||
edge = ["rocksdb"]
|
||||
|
||||
|
||||
@@ -1061,6 +1061,14 @@ mod tests {
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
_address: Address,
|
||||
_hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl BytecodeReader for MockStateProvider {
|
||||
|
||||
@@ -223,6 +223,26 @@ impl<N: NodePrimitives> StateProvider for MemoryOverlayStateProviderRef<'_, N> {
|
||||
|
||||
self.historical.storage(address, storage_key)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
let hashed_address = keccak256(address);
|
||||
let state = &self.trie_input().state;
|
||||
|
||||
if let Some(hs) = state.storages.get(&hashed_address) {
|
||||
if let Some(value) = hs.storage.get(&hashed_storage_key) {
|
||||
return Ok(Some(*value));
|
||||
}
|
||||
if hs.wiped {
|
||||
return Ok(Some(StorageValue::ZERO));
|
||||
}
|
||||
}
|
||||
|
||||
self.historical.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> BytecodeReader for MemoryOverlayStateProviderRef<'_, N> {
|
||||
|
||||
@@ -87,6 +87,8 @@ tokio-stream.workspace = true
|
||||
reqwest.workspace = true
|
||||
url.workspace = true
|
||||
metrics.workspace = true
|
||||
memmap2.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
# io
|
||||
fdlimit.workspace = true
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_node_builder::{
|
||||
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
|
||||
};
|
||||
use reth_node_core::{
|
||||
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs, StorageArgs},
|
||||
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs, StorageArgs},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
};
|
||||
use reth_provider::{
|
||||
@@ -67,62 +67,23 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` related arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
pub storage: StorageArgs,
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
|
||||
/// `RocksDB` CLI args.
|
||||
/// Returns the effective storage settings derived from `--storage.v2`.
|
||||
///
|
||||
/// The base storage mode is determined by `--storage.v2`:
|
||||
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
|
||||
///
|
||||
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
|
||||
/// - Otherwise: uses [`StorageSettings::base()`] defaults
|
||||
pub fn storage_settings(&self) -> StorageSettings {
|
||||
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
|
||||
|
||||
// Apply static files overrides (only when explicitly set)
|
||||
if let Some(v) = self.static_files.receipts {
|
||||
s = s.with_receipts_in_static_files(v);
|
||||
if self.storage.v2 {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::base()
|
||||
}
|
||||
if let Some(v) = self.static_files.transaction_senders {
|
||||
s = s.with_transaction_senders_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.account_changesets {
|
||||
s = s.with_account_changesets_in_static_files(v);
|
||||
}
|
||||
if let Some(v) = self.static_files.storage_changesets {
|
||||
s = s.with_storage_changesets_in_static_files(v);
|
||||
}
|
||||
|
||||
// Apply rocksdb overrides
|
||||
// --rocksdb.all sets all rocksdb flags to true
|
||||
if self.rocksdb.all {
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true)
|
||||
.with_account_history_in_rocksdb(true);
|
||||
}
|
||||
|
||||
// Individual rocksdb flags override --rocksdb.all when explicitly set
|
||||
if let Some(v) = self.rocksdb.tx_hash {
|
||||
s = s.with_transaction_hash_numbers_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.storages_history {
|
||||
s = s.with_storages_history_in_rocksdb(v);
|
||||
}
|
||||
if let Some(v) = self.rocksdb.account_history {
|
||||
s = s.with_account_history_in_rocksdb(v);
|
||||
}
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
/// Initializes environment according to [`AccessRights`] and returns an instance of
|
||||
|
||||
@@ -5,6 +5,7 @@ use reth_codecs::Compact;
|
||||
use reth_db_api::{cursor::DbDupCursorRO, database::Database, tables, transaction::DbTx};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::info;
|
||||
|
||||
@@ -22,52 +23,94 @@ impl Command {
|
||||
/// Execute `db account-storage` command
|
||||
pub fn execute<N: NodeTypesWithDB>(self, tool: &DbTool<N>) -> eyre::Result<()> {
|
||||
let address = self.address;
|
||||
let (slot_count, plain_size) = tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
let (slot_count, storage_size) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing hashed storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
let total_size = if count > 0 { 32 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??;
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
} else {
|
||||
tool.provider_factory.db_ref().view(|tx| {
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut count = 0usize;
|
||||
let mut total_value_bytes = 0usize;
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { plain_size + 12 } else { 0 };
|
||||
let total_estimate = plain_size + hashed_size_estimate;
|
||||
// Walk all storage entries for this address
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for entry in walker {
|
||||
let (_, storage_entry) = entry?;
|
||||
count += 1;
|
||||
let mut buf = Vec::new();
|
||||
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
|
||||
let entry_len = storage_entry.to_compact(&mut buf);
|
||||
total_value_bytes += entry_len;
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots = count,
|
||||
key = %storage_entry.key,
|
||||
"Processing storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Add 20 bytes for the Address key (stored once per account in dupsort)
|
||||
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
|
||||
|
||||
Ok::<_, eyre::Report>((count, total_size))
|
||||
})??
|
||||
};
|
||||
|
||||
let hashed_address = keccak256(address);
|
||||
|
||||
println!("Account: {address}");
|
||||
println!("Hashed address: {hashed_address}");
|
||||
println!("Storage slots: {slot_count}");
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(plain_size as f64));
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(hashed_size_estimate as f64));
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
if use_hashed_state {
|
||||
println!("Hashed storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
} else {
|
||||
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
|
||||
let hashed_size_estimate = if slot_count > 0 { storage_size + 12 } else { 0 };
|
||||
let total_estimate = storage_size + hashed_size_estimate;
|
||||
println!("Plain storage size: {} (estimated)", human_bytes(storage_size as f64));
|
||||
println!(
|
||||
"Hashed storage size: {} (estimated)",
|
||||
human_bytes(hashed_size_estimate as f64)
|
||||
);
|
||||
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
61
crates/cli/commands/src/db/copy.rs
Normal file
61
crates/cli/commands/src/db/copy.rs
Normal file
@@ -0,0 +1,61 @@
|
||||
use clap::Parser;
|
||||
use reth_db::mdbx::{self, ffi};
|
||||
use std::path::PathBuf;
|
||||
|
||||
/// Copies the MDBX database to a new location.
|
||||
///
|
||||
/// Equivalent to the standalone `mdbx_copy` tool but bundled into reth.
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
/// Destination path for the database copy.
|
||||
dest: PathBuf,
|
||||
|
||||
/// Compact the database while copying (reclaims free space).
|
||||
#[arg(short, long)]
|
||||
compact: bool,
|
||||
|
||||
/// Force dynamic size for the destination database.
|
||||
#[arg(short = 'd', long)]
|
||||
force_dynamic_size: bool,
|
||||
|
||||
/// Throttle to avoid MVCC pressure on writers.
|
||||
#[arg(short = 'p', long)]
|
||||
throttle_mvcc: bool,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `db copy` command
|
||||
pub fn execute(self, db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let mut flags: ffi::MDBX_copy_flags_t = ffi::MDBX_CP_DEFAULTS;
|
||||
if self.compact {
|
||||
flags |= ffi::MDBX_CP_COMPACT;
|
||||
}
|
||||
if self.force_dynamic_size {
|
||||
flags |= ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
}
|
||||
if self.throttle_mvcc {
|
||||
flags |= ffi::MDBX_CP_THROTTLE_MVCC;
|
||||
}
|
||||
|
||||
let dest = self
|
||||
.dest
|
||||
.to_str()
|
||||
.ok_or_else(|| eyre::eyre!("destination path must be valid UTF-8"))?;
|
||||
let dest_cstr = std::ffi::CString::new(dest)?;
|
||||
|
||||
println!("Copying database to {} ...", self.dest.display());
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
println!("Done.");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
564
crates/cli/commands/src/db/gen_ownermap.rs
Normal file
564
crates/cli/commands/src/db/gen_ownermap.rs
Normal file
@@ -0,0 +1,564 @@
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use reth_db_api::tables::Tables;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
io::Write,
|
||||
path::PathBuf,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const PAGEHDRSZ: usize = 20;
|
||||
const NODESIZE: usize = 8;
|
||||
const P_BRANCH: u16 = 0x01;
|
||||
const P_LEAF: u16 = 0x02;
|
||||
const P_LARGE: u16 = 0x04;
|
||||
const P_META: u16 = 0x08;
|
||||
const P_DUPFIX: u16 = 0x20;
|
||||
const N_BIG: u8 = 0x01;
|
||||
const N_TREE: u8 = 0x02;
|
||||
const N_DUP: u8 = 0x04;
|
||||
const P_INVALID: u32 = 0xFFFFFFFF;
|
||||
const UNREFERENCED: u8 = 0xFF;
|
||||
const DBI_META: u8 = 0xFE;
|
||||
|
||||
const META_GEO: usize = 0x14;
|
||||
const META_FREE_DB: usize = 0x28;
|
||||
const META_MAIN_DB: usize = 0x58;
|
||||
const META_TXNID_A: usize = 0x08;
|
||||
const META_TXNID_B: usize = 0xB0;
|
||||
const GEO_NOW: usize = 0x0C;
|
||||
const TREE_ROOT: usize = 0x08;
|
||||
const TREE_BRANCH_PAGES: usize = 0x0C;
|
||||
const TREE_LEAF_PAGES: usize = 0x10;
|
||||
const TREE_LARGE_PAGES: usize = 0x14;
|
||||
const TREE_ITEMS: usize = 0x20;
|
||||
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
|
||||
|
||||
fn u16_le(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
fn u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
fn u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
u64::from_le_bytes(buf[off..off + 8].try_into().unwrap())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TreeDescriptor {
|
||||
root: u32,
|
||||
branch_pages: u32,
|
||||
leaf_pages: u32,
|
||||
large_pages: u32,
|
||||
items: u64,
|
||||
}
|
||||
|
||||
impl TreeDescriptor {
|
||||
fn parse(buf: &[u8], off: usize) -> Self {
|
||||
Self {
|
||||
root: u32_le(buf, off + TREE_ROOT),
|
||||
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
|
||||
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
|
||||
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
|
||||
items: u64_le(buf, off + TREE_ITEMS),
|
||||
}
|
||||
}
|
||||
|
||||
fn total_pages(&self) -> u64 {
|
||||
self.branch_pages as u64 + self.leaf_pages as u64 + self.large_pages as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DBIInfo {
|
||||
name: String,
|
||||
dbi_index: u8,
|
||||
tree: TreeDescriptor,
|
||||
}
|
||||
|
||||
fn page_flags(buf: &[u8], page_off: usize) -> u16 {
|
||||
u16_le(buf, page_off + 0x0A)
|
||||
}
|
||||
|
||||
fn page_nkeys(buf: &[u8], page_off: usize) -> usize {
|
||||
u16_le(buf, page_off + 0x0C) as usize / 2
|
||||
}
|
||||
|
||||
fn page_overflow_count(buf: &[u8], page_off: usize) -> u32 {
|
||||
u32_le(buf, page_off + 0x0C)
|
||||
}
|
||||
|
||||
fn mark(owner_map: &mut [u8], pgno: usize, dbi_index: u8, conflicts: &mut u64) -> bool {
|
||||
if pgno >= owner_map.len() {
|
||||
return false;
|
||||
}
|
||||
let prev = owner_map[pgno];
|
||||
if prev == UNREFERENCED {
|
||||
owner_map[pgno] = dbi_index;
|
||||
return true;
|
||||
}
|
||||
if prev != dbi_index {
|
||||
*conflicts += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
|
||||
fn walk_tree_collect(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
root_pgno: u32,
|
||||
page_count: usize,
|
||||
dbi_index: u8,
|
||||
) -> Vec<(usize, u8)> {
|
||||
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut result: Vec<(usize, u8)> = Vec::new();
|
||||
let mut visited = vec![false; page_count];
|
||||
let mut stack: Vec<u32> = vec![root_pgno];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= page_count || visited[pgno_usize] {
|
||||
continue;
|
||||
}
|
||||
visited[pgno_usize] = true;
|
||||
result.push((pgno_usize, dbi_index));
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() {
|
||||
continue;
|
||||
}
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
stack.push(u32_le(buf, node_abs));
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
|
||||
if data_off + 48 > buf.len() { continue; }
|
||||
let sub = TreeDescriptor::parse(buf, data_off);
|
||||
if sub.root != P_INVALID {
|
||||
stack.push(sub.root);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if flags & P_LARGE != 0 {
|
||||
let ov_count = page_overflow_count(buf, page_off) as usize;
|
||||
for op in (pgno_usize + 1)..(pgno_usize + ov_count) {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn discover_named_dbis(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
main_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_main: u8,
|
||||
dbi_start: u8,
|
||||
) -> Vec<DBIInfo> {
|
||||
if main_root == P_INVALID || main_root as usize >= owner_map.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut named: Vec<DBIInfo> = Vec::new();
|
||||
let mut next_index = dbi_start;
|
||||
let mut stack: Vec<u32> = vec![main_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_main, conflicts) { continue; }
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
let child_pgno = u32_le(buf, node_abs);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_TREE != 0 {
|
||||
let key_off = node_abs + NODESIZE;
|
||||
if key_off + ksize > buf.len() || data_off + 48 > buf.len() { continue; }
|
||||
let name_bytes = &buf[key_off..key_off + ksize];
|
||||
let name = String::from_utf8_lossy(
|
||||
name_bytes.split(|&b| b == 0).next().unwrap_or(name_bytes)
|
||||
).to_string();
|
||||
let tree = TreeDescriptor::parse(buf, data_off);
|
||||
named.push(DBIInfo { name, dbi_index: next_index, tree });
|
||||
next_index = next_index.saturating_add(1);
|
||||
} else if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
mark(owner_map, op, dbi_main, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
named
|
||||
}
|
||||
|
||||
fn mark_free_pages(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
free_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_free: u8,
|
||||
) -> u64 {
|
||||
if free_root == P_INVALID || free_root as usize >= owner_map.len() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut marked: u64 = 0;
|
||||
let mut stack: Vec<u32> = vec![free_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_free, conflicts) { continue; }
|
||||
marked += 1;
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let child_pgno = u32_le(buf, page_off + PAGEHDRSZ + node_rel);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let dsize = u32_le(buf, node_abs) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if mark(owner_map, op, dbi_free, conflicts) { marked += 1; }
|
||||
}
|
||||
let ov_data_off = ov_pgno * ps + PAGEHDRSZ;
|
||||
if dsize >= 4 && ov_data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, ov_data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = ov_data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dsize >= 4 && data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
marked
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
#[arg(short, long, default_value = "owner_map.bin")]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
pub fn execute<N: NodeTypesWithDB>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
_tool: &DbTool<N>,
|
||||
) -> eyre::Result<()> {
|
||||
let db_path = data_dir.db().join("mdbx.dat");
|
||||
eyre::ensure!(db_path.exists(), "mdbx.dat not found at {:?}", db_path);
|
||||
|
||||
let t0 = Instant::now();
|
||||
|
||||
let file = fs::File::open(&db_path)
|
||||
.wrap_err_with(|| format!("Failed to open {}", db_path.display()))?;
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
let buf: &[u8] = &mmap;
|
||||
|
||||
if buf.len() < PAGEHDRSZ + 0xC0 {
|
||||
eyre::bail!("File too small for MDBX meta page");
|
||||
}
|
||||
let flags0 = u16_le(buf, 0x0A);
|
||||
if flags0 & P_META == 0 {
|
||||
eyre::bail!("Page 0 missing P_META flag");
|
||||
}
|
||||
let magic = u64_le(buf, PAGEHDRSZ);
|
||||
if (magic >> 8) != MDBX_MAGIC {
|
||||
eyre::bail!("MDBX magic mismatch");
|
||||
}
|
||||
|
||||
let candidates = [4096usize, 8192, 16384, 32768, 65536];
|
||||
let mut ps = 4096usize;
|
||||
let geo_now_raw = u32_le(buf, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
|
||||
for &candidate in &candidates {
|
||||
let mapped = geo_now_raw * candidate;
|
||||
if mapped >= buf.len() / 2 && mapped <= buf.len() * 4 {
|
||||
ps = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut best_txnid: u64 = 0;
|
||||
let mut best_meta_base: usize = 0;
|
||||
for pgno in 0..3usize {
|
||||
let page_base = pgno * ps;
|
||||
let meta_base = page_base + PAGEHDRSZ;
|
||||
if meta_base + 0xC0 > buf.len() { continue; }
|
||||
let pflags = u16_le(buf, page_base + 0x0A);
|
||||
if pflags & P_META == 0 { continue; }
|
||||
let m = u64_le(buf, meta_base);
|
||||
if (m >> 8) != MDBX_MAGIC { continue; }
|
||||
let txnid_a = u64_le(buf, meta_base + META_TXNID_A);
|
||||
let txnid_b = u64_le(buf, meta_base + META_TXNID_B);
|
||||
if txnid_a == txnid_b && txnid_a > best_txnid {
|
||||
best_txnid = txnid_a;
|
||||
best_meta_base = meta_base;
|
||||
}
|
||||
}
|
||||
eyre::ensure!(best_txnid > 0, "No valid meta page found");
|
||||
|
||||
let page_count = u32_le(buf, best_meta_base + META_GEO + GEO_NOW) as usize;
|
||||
let free_tree = TreeDescriptor::parse(buf, best_meta_base + META_FREE_DB);
|
||||
let main_tree = TreeDescriptor::parse(buf, best_meta_base + META_MAIN_DB);
|
||||
|
||||
println!("MDBX owner map generator (parallel)");
|
||||
println!(" page_size: {ps}");
|
||||
println!(" page_count: {page_count}");
|
||||
println!(" txnid: {best_txnid}");
|
||||
println!(" FreeDB root: {} ({} items)", free_tree.root, free_tree.items);
|
||||
println!(" MainDB root: {} ({} items)", main_tree.root, main_tree.items);
|
||||
println!();
|
||||
|
||||
let mut owner_map = vec![UNREFERENCED; page_count];
|
||||
let mut conflicts: u64 = 0;
|
||||
|
||||
for pgno in 0..std::cmp::min(3, page_count) {
|
||||
owner_map[pgno] = DBI_META;
|
||||
}
|
||||
|
||||
println!("Discovering named DBIs via MainDB walk...");
|
||||
let discovered = discover_named_dbis(
|
||||
buf, ps, main_tree.root, &mut owner_map, &mut conflicts, 1, 2,
|
||||
);
|
||||
|
||||
let mut name_to_dbi: HashMap<&str, u8> = HashMap::new();
|
||||
for (idx, table) in Tables::ALL.iter().enumerate() {
|
||||
name_to_dbi.insert(table.name(), (idx + 2) as u8);
|
||||
}
|
||||
|
||||
let mut named_dbis: Vec<DBIInfo> = Vec::new();
|
||||
let mut remap: HashMap<u8, u8> = HashMap::new();
|
||||
for dbi in &discovered {
|
||||
if let Some(&real_dbi) = name_to_dbi.get(dbi.name.as_str()) {
|
||||
remap.insert(dbi.dbi_index, real_dbi);
|
||||
named_dbis.push(DBIInfo {
|
||||
name: dbi.name.clone(),
|
||||
dbi_index: real_dbi,
|
||||
tree: dbi.tree.clone(),
|
||||
});
|
||||
} else {
|
||||
named_dbis.push(dbi.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for byte in owner_map.iter_mut() {
|
||||
if let Some(&new_dbi) = remap.get(byte) {
|
||||
*byte = new_dbi;
|
||||
}
|
||||
}
|
||||
|
||||
println!("Found {} named DBI(s)", named_dbis.len());
|
||||
|
||||
println!("Walking FreeDB / GC...");
|
||||
let free_marked = mark_free_pages(buf, ps, free_tree.root, &mut owner_map, &mut conflicts, 0);
|
||||
println!("FreeDB: {} pages marked", free_marked);
|
||||
|
||||
println!("Walking {} named DBIs in parallel...", named_dbis.len());
|
||||
let t_par = Instant::now();
|
||||
|
||||
let results: Vec<(String, u8, Vec<(usize, u8)>)> = named_dbis
|
||||
.par_iter()
|
||||
.filter(|dbi| dbi.tree.root != P_INVALID)
|
||||
.map(|dbi| {
|
||||
let pages = walk_tree_collect(buf, ps, dbi.tree.root, page_count, dbi.dbi_index);
|
||||
(dbi.name.clone(), dbi.dbi_index, pages)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (name, dbi_index, pages) in &results {
|
||||
let count = pages.len();
|
||||
for &(pgno, idx) in pages {
|
||||
mark(&mut owner_map, pgno, idx, &mut conflicts);
|
||||
}
|
||||
println!(" [{:2}] {:30} {:>10} pages", dbi_index, name, count);
|
||||
}
|
||||
|
||||
println!("Parallel walk done in {:.2}s", t_par.elapsed().as_secs_f64());
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
|
||||
let mut counts: HashMap<u8, u64> = HashMap::new();
|
||||
let mut unreferenced: u64 = 0;
|
||||
for &b in &owner_map {
|
||||
if b == UNREFERENCED {
|
||||
unreferenced += 1;
|
||||
} else {
|
||||
*counts.entry(b).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("Walk complete in {:.2}s", elapsed.as_secs_f64());
|
||||
println!(" Total pages: {page_count}");
|
||||
println!(" Unreferenced: {unreferenced}");
|
||||
println!(" Conflicts: {conflicts}");
|
||||
println!();
|
||||
|
||||
if let Some(&c) = counts.get(&DBI_META) {
|
||||
println!(" [0xFE] {:30} {:>10} pages", "<meta>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&0) {
|
||||
println!(" [ 0] {:30} {:>10} pages", "<free/GC>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&1) {
|
||||
println!(" [ 1] {:30} {:>10} pages", "<main>", c);
|
||||
}
|
||||
let mut sorted_dbis = named_dbis.clone();
|
||||
sorted_dbis.sort_by_key(|d| d.dbi_index);
|
||||
for dbi in &sorted_dbis {
|
||||
let walked = counts.get(&dbi.dbi_index).copied().unwrap_or(0);
|
||||
let expected = dbi.tree.total_pages();
|
||||
let mismatch = if expected > 0 && walked != expected {
|
||||
format!(" !! MISMATCH (tree_t says {})", expected)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(" [{:3}] {:30} {:>10} pages{}", dbi.dbi_index, dbi.name, walked, mismatch);
|
||||
}
|
||||
println!(" [0xFF] {:30} {:>10} pages", "<unreferenced>", unreferenced);
|
||||
|
||||
let mut out = fs::File::create(&self.output)?;
|
||||
out.write_all(&owner_map)?;
|
||||
println!();
|
||||
println!("Written {} bytes to {}", owner_map.len(), self.output.display());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -98,7 +98,8 @@ impl Command {
|
||||
)?;
|
||||
|
||||
if let Some(entry) = entry {
|
||||
println!("{}", serde_json::to_string_pretty(&entry)?);
|
||||
let se: reth_primitives_traits::StorageEntry = entry.into();
|
||||
println!("{}", serde_json::to_string_pretty(&se)?);
|
||||
} else {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
@@ -106,7 +107,14 @@ impl Command {
|
||||
}
|
||||
|
||||
let changesets = provider.storage_changeset(key.block_number())?;
|
||||
println!("{}", serde_json::to_string_pretty(&changesets)?);
|
||||
let serializable: Vec<_> = changesets
|
||||
.into_iter()
|
||||
.map(|(addr, entry)| {
|
||||
let se: reth_primitives_traits::StorageEntry = entry.into();
|
||||
(addr, se)
|
||||
})
|
||||
.collect();
|
||||
println!("{}", serde_json::to_string_pretty(&serializable)?);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@ use std::{
|
||||
mod account_storage;
|
||||
mod checksum;
|
||||
mod clear;
|
||||
mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
@@ -20,6 +21,7 @@ mod settings;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
pub mod gen_ownermap;
|
||||
/// DB List TUI
|
||||
mod tui;
|
||||
|
||||
@@ -42,6 +44,8 @@ pub enum Subcommands {
|
||||
List(list::Command),
|
||||
/// Calculates the content checksum of a table or static file segment
|
||||
Checksum(checksum::Command),
|
||||
/// Copies the MDBX database to a new location (bundled mdbx_copy)
|
||||
Copy(copy::Command),
|
||||
/// Create a diff between two database tables or two entire databases.
|
||||
Diff(diff::Command),
|
||||
/// Gets the content of a table for the given key
|
||||
@@ -68,6 +72,8 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Generate a page owner map for MDBX page visualization
|
||||
GenOwnermap(gen_ownermap::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -124,6 +130,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::Copy(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(tool.provider_factory.db_ref())?;
|
||||
});
|
||||
}
|
||||
Subcommands::Diff(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(&tool)?;
|
||||
@@ -206,6 +217,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::GenOwnermap(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(data_dir, &tool)?;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -39,50 +39,12 @@ enum Subcommands {
|
||||
#[derive(Debug, Clone, Copy, Subcommand)]
|
||||
#[clap(rename_all = "snake_case")]
|
||||
pub enum SetCommand {
|
||||
/// Store receipts in static files instead of the database
|
||||
Receipts {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction senders in static files instead of the database
|
||||
TransactionSenders {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account changesets in static files instead of the database
|
||||
AccountChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage history in rocksdb instead of MDBX
|
||||
StoragesHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store transaction hash to number mapping in rocksdb instead of MDBX
|
||||
TransactionHashNumbers {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store account history in rocksdb instead of MDBX
|
||||
AccountHistory {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Store storage changesets in static files instead of the database
|
||||
StorageChangesets {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
/// Use hashed state tables (HashedAccounts/HashedStorages) as canonical state
|
||||
/// Enable or disable v2 storage layout
|
||||
///
|
||||
/// When enabled, execution writes directly to hashed tables, eliminating need for
|
||||
/// separate hashing stages. State reads come from hashed tables.
|
||||
///
|
||||
/// WARNING: Changing this setting in either direction requires re-syncing the database.
|
||||
/// Enabling on an existing plain-state database leaves hashed tables empty.
|
||||
/// Disabling on an existing hashed-state database leaves plain tables empty.
|
||||
UseHashedState {
|
||||
/// When enabled, uses static files for receipts/senders/changesets and RocksDB for
|
||||
/// history indices and transaction hashes. When disabled, uses v1/legacy layout (everything in
|
||||
/// MDBX).
|
||||
V2 {
|
||||
#[clap(action(ArgAction::Set))]
|
||||
value: bool,
|
||||
},
|
||||
@@ -125,87 +87,18 @@ impl Command {
|
||||
println!("No storage settings found, creating new settings.");
|
||||
}
|
||||
|
||||
let mut settings @ StorageSettings {
|
||||
receipts_in_static_files: _,
|
||||
transaction_senders_in_static_files: _,
|
||||
storages_history_in_rocksdb: _,
|
||||
transaction_hash_numbers_in_rocksdb: _,
|
||||
account_history_in_rocksdb: _,
|
||||
account_changesets_in_static_files: _,
|
||||
storage_changesets_in_static_files: _,
|
||||
use_hashed_state: _,
|
||||
} = settings.unwrap_or_else(StorageSettings::v1);
|
||||
let mut settings @ StorageSettings { storage_v2: _ } =
|
||||
settings.unwrap_or_else(StorageSettings::v1);
|
||||
|
||||
// Update the setting based on the key
|
||||
match cmd {
|
||||
SetCommand::Receipts { value } => {
|
||||
if settings.receipts_in_static_files == value {
|
||||
println!("receipts_in_static_files is already set to {}", value);
|
||||
SetCommand::V2 { value } => {
|
||||
if settings.storage_v2 == value {
|
||||
println!("storage_v2 is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.receipts_in_static_files = value;
|
||||
println!("Set receipts_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::TransactionSenders { value } => {
|
||||
if settings.transaction_senders_in_static_files == value {
|
||||
println!("transaction_senders_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_senders_in_static_files = value;
|
||||
println!("Set transaction_senders_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::AccountChangesets { value } => {
|
||||
if settings.account_changesets_in_static_files == value {
|
||||
println!("account_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_changesets_in_static_files = value;
|
||||
println!("Set account_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::StoragesHistory { value } => {
|
||||
if settings.storages_history_in_rocksdb == value {
|
||||
println!("storages_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storages_history_in_rocksdb = value;
|
||||
println!("Set storages_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::TransactionHashNumbers { value } => {
|
||||
if settings.transaction_hash_numbers_in_rocksdb == value {
|
||||
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.transaction_hash_numbers_in_rocksdb = value;
|
||||
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::AccountHistory { value } => {
|
||||
if settings.account_history_in_rocksdb == value {
|
||||
println!("account_history_in_rocksdb is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.account_history_in_rocksdb = value;
|
||||
println!("Set account_history_in_rocksdb = {}", value);
|
||||
}
|
||||
SetCommand::StorageChangesets { value } => {
|
||||
if settings.storage_changesets_in_static_files == value {
|
||||
println!("storage_changesets_in_static_files is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
settings.storage_changesets_in_static_files = value;
|
||||
println!("Set storage_changesets_in_static_files = {}", value);
|
||||
}
|
||||
SetCommand::UseHashedState { value } => {
|
||||
if settings.use_hashed_state == value {
|
||||
println!("use_hashed_state is already set to {}", value);
|
||||
return Ok(());
|
||||
}
|
||||
if settings.use_hashed_state && !value {
|
||||
println!("WARNING: Disabling use_hashed_state on an existing hashed-state database requires a full resync.");
|
||||
} else {
|
||||
println!("WARNING: Enabling use_hashed_state on an existing plain-state database requires a full resync.");
|
||||
}
|
||||
settings.use_hashed_state = value;
|
||||
println!("Set use_hashed_state = {}", value);
|
||||
settings.storage_v2 = value;
|
||||
println!("Set storage_v2 = {}", value);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{Address, BlockNumber, B256, U256};
|
||||
use alloy_primitives::{keccak256, Address, BlockNumber, B256, U256};
|
||||
use clap::Parser;
|
||||
use parking_lot::Mutex;
|
||||
use reth_db_api::{
|
||||
@@ -63,39 +63,65 @@ impl Command {
|
||||
address: Address,
|
||||
limit: usize,
|
||||
) -> eyre::Result<()> {
|
||||
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let entries = tool.provider_factory.db_ref().view(|tx| {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
let (account, walker_entries) = if use_hashed_state {
|
||||
let hashed_address = keccak256(address);
|
||||
let account = tx.get::<tables::HashedAccounts>(hashed_address)?;
|
||||
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
|
||||
let walker = cursor.walk_dup(Some(hashed_address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
(account, entries)
|
||||
} else {
|
||||
// Get account info
|
||||
let account = tx.get::<tables::PlainAccountState>(address)?;
|
||||
// Get storage entries
|
||||
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
|
||||
let walker = cursor.walk_dup(Some(address), None)?;
|
||||
let mut entries = Vec::new();
|
||||
let mut last_log = Instant::now();
|
||||
for (idx, entry) in walker.enumerate() {
|
||||
let (_, storage_entry) = entry?;
|
||||
if storage_entry.value != U256::ZERO {
|
||||
entries.push((storage_entry.key, storage_entry.value));
|
||||
}
|
||||
if entries.len() >= limit {
|
||||
break;
|
||||
}
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
(account, entries)
|
||||
};
|
||||
|
||||
if last_log.elapsed() >= LOG_INTERVAL {
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
address = %address,
|
||||
slots_scanned = idx,
|
||||
"Scanning storage slots"
|
||||
);
|
||||
last_log = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
Ok::<_, eyre::Report>((account, entries))
|
||||
Ok::<_, eyre::Report>((account, walker_entries))
|
||||
})??;
|
||||
|
||||
let (account, storage_entries) = entries;
|
||||
@@ -119,7 +145,7 @@ impl Command {
|
||||
|
||||
// Check storage settings to determine where history is stored
|
||||
let storage_settings = tool.provider_factory.cached_storage_settings();
|
||||
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
|
||||
let history_in_rocksdb = storage_settings.storage_v2;
|
||||
|
||||
// For historical queries, enumerate keys from history indices only
|
||||
// (not PlainStorageState, which reflects current state)
|
||||
|
||||
@@ -37,6 +37,14 @@ pub struct DownloadDefaults {
|
||||
pub available_snapshots: Vec<Cow<'static, str>>,
|
||||
/// Default base URL for snapshots
|
||||
pub default_base_url: Cow<'static, str>,
|
||||
/// Default base URL for chain-aware snapshots.
|
||||
///
|
||||
/// When set, the chain ID is appended to form the full URL: `{base_url}/{chain_id}`.
|
||||
/// For example, given a base URL of `https://snapshots.example.com` and chain ID `1`,
|
||||
/// the resulting URL would be `https://snapshots.example.com/1`.
|
||||
///
|
||||
/// Falls back to [`default_base_url`](Self::default_base_url) when `None`.
|
||||
pub default_chain_aware_base_url: Option<Cow<'static, str>>,
|
||||
/// Optional custom long help text that overrides the generated help
|
||||
pub long_help: Option<String>,
|
||||
}
|
||||
@@ -60,6 +68,7 @@ impl DownloadDefaults {
|
||||
Cow::Borrowed("https://publicnode.com/snapshots (full nodes & testnets)"),
|
||||
],
|
||||
default_base_url: Cow::Borrowed(MERKLE_BASE_URL),
|
||||
default_chain_aware_base_url: None,
|
||||
long_help: None,
|
||||
}
|
||||
}
|
||||
@@ -84,9 +93,11 @@ impl DownloadDefaults {
|
||||
}
|
||||
|
||||
help.push_str(
|
||||
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
|
||||
"\nIf no URL is provided, the latest archive snapshot for the selected chain\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(
|
||||
self.default_chain_aware_base_url.as_deref().unwrap_or(&self.default_base_url),
|
||||
);
|
||||
help.push_str(self.default_base_url.as_ref());
|
||||
help.push_str(
|
||||
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
|
||||
);
|
||||
@@ -111,6 +122,12 @@ impl DownloadDefaults {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default chain-aware base URL.
|
||||
pub fn with_chain_aware_base_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
|
||||
self.default_chain_aware_base_url = Some(url.into());
|
||||
self
|
||||
}
|
||||
|
||||
/// Builder: Set custom long help text, overriding the generated help
|
||||
pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
|
||||
self.long_help = Some(help.into());
|
||||
@@ -142,7 +159,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> DownloadCo
|
||||
let url = match self.url {
|
||||
Some(url) => url,
|
||||
None => {
|
||||
let url = get_latest_snapshot_url().await?;
|
||||
let url = get_latest_snapshot_url(self.env.chain.chain().id()).await?;
|
||||
info!(target: "reth::cli", "Using default snapshot URL: {}", url);
|
||||
url
|
||||
}
|
||||
@@ -509,8 +526,12 @@ async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
}
|
||||
|
||||
// Builds default URL for latest mainnet archive snapshot using configured defaults
|
||||
async fn get_latest_snapshot_url() -> Result<String> {
|
||||
let base_url = &DownloadDefaults::get_global().default_base_url;
|
||||
async fn get_latest_snapshot_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let base_url = match &defaults.default_chain_aware_base_url {
|
||||
Some(url) => format!("{url}/{chain_id}"),
|
||||
None => defaults.default_base_url.to_string(),
|
||||
};
|
||||
let latest_url = format!("{base_url}/latest.txt");
|
||||
let filename = Client::new()
|
||||
.get(latest_url)
|
||||
|
||||
@@ -10,8 +10,8 @@ use reth_node_builder::NodeBuilder;
|
||||
use reth_node_core::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
|
||||
StorageArgs, TxPoolArgs,
|
||||
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs,
|
||||
TxPoolArgs,
|
||||
},
|
||||
node_config::NodeConfig,
|
||||
version,
|
||||
@@ -103,10 +103,6 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten)]
|
||||
pub pruning: PruningArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
#[command(flatten)]
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Engine cli arguments
|
||||
#[command(flatten, next_help_heading = "Engine")]
|
||||
pub engine: EngineArgs,
|
||||
@@ -119,8 +115,8 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
|
||||
#[command(flatten, next_help_heading = "Static Files")]
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
#[command(flatten)]
|
||||
/// All storage related arguments with --storage prefix
|
||||
#[command(flatten, next_help_heading = "Storage")]
|
||||
pub storage: StorageArgs,
|
||||
|
||||
/// Additional cli arguments
|
||||
@@ -175,7 +171,6 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
@@ -183,9 +178,6 @@ where
|
||||
ext,
|
||||
} = self;
|
||||
|
||||
// Validate RocksDB arguments
|
||||
rocksdb.validate()?;
|
||||
|
||||
// set up node config
|
||||
let mut node_config = NodeConfig {
|
||||
datadir,
|
||||
@@ -201,7 +193,6 @@ where
|
||||
db,
|
||||
dev,
|
||||
pruning,
|
||||
rocksdb,
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
|
||||
@@ -45,12 +45,16 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
|
||||
let tool = DbTool::new(provider_factory)?;
|
||||
|
||||
let static_file_segment = match self.stage {
|
||||
StageEnum::Headers => Some(StaticFileSegment::Headers),
|
||||
StageEnum::Bodies => Some(StaticFileSegment::Transactions),
|
||||
StageEnum::Execution => Some(StaticFileSegment::Receipts),
|
||||
StageEnum::Senders => Some(StaticFileSegment::TransactionSenders),
|
||||
_ => None,
|
||||
let static_file_segments = match self.stage {
|
||||
StageEnum::Headers => vec![StaticFileSegment::Headers],
|
||||
StageEnum::Bodies => vec![StaticFileSegment::Transactions],
|
||||
StageEnum::Execution => vec![
|
||||
StaticFileSegment::Receipts,
|
||||
StaticFileSegment::AccountChangeSets,
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
],
|
||||
StageEnum::Senders => vec![StaticFileSegment::TransactionSenders],
|
||||
_ => vec![],
|
||||
};
|
||||
|
||||
// Calling `StaticFileProviderRW::prune_*` will instruct the writer to prune rows only
|
||||
@@ -58,35 +62,33 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
// deleting the jar files, otherwise if the task were to be interrupted after we
|
||||
// have deleted them, BUT before we have committed the checkpoints to the database, we'd
|
||||
// lose essential data.
|
||||
if let Some(static_file_segment) = static_file_segment {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
if let Some(highest_block) =
|
||||
static_file_provider.get_highest_static_file_block(static_file_segment)
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
for segment in static_file_segments {
|
||||
if let Some(highest_block) = static_file_provider.get_highest_static_file_block(segment)
|
||||
{
|
||||
let mut writer = static_file_provider.latest_writer(static_file_segment)?;
|
||||
let mut writer = static_file_provider.latest_writer(segment)?;
|
||||
|
||||
match static_file_segment {
|
||||
match segment {
|
||||
StaticFileSegment::Headers => {
|
||||
// Prune all headers leaving genesis intact.
|
||||
writer.prune_headers(highest_block)?;
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.get_highest_static_file_tx(segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_transactions(to_delete, 0)?;
|
||||
}
|
||||
StaticFileSegment::Receipts => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.get_highest_static_file_tx(segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_receipts(to_delete, 0)?;
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
let to_delete = static_file_provider
|
||||
.get_highest_static_file_tx(static_file_segment)
|
||||
.get_highest_static_file_tx(segment)
|
||||
.map(|tx_num| tx_num + 1)
|
||||
.unwrap_or_default();
|
||||
writer.prune_transaction_senders(to_delete, 0)?;
|
||||
@@ -131,8 +133,15 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
reset_stage_checkpoint(tx, StageId::SenderRecovery)?;
|
||||
}
|
||||
StageEnum::Execution => {
|
||||
tx.clear::<tables::PlainAccountState>()?;
|
||||
tx.clear::<tables::PlainStorageState>()?;
|
||||
if provider_rw.cached_storage_settings().use_hashed_state() {
|
||||
tx.clear::<tables::HashedAccounts>()?;
|
||||
tx.clear::<tables::HashedStorages>()?;
|
||||
reset_stage_checkpoint(tx, StageId::AccountHashing)?;
|
||||
reset_stage_checkpoint(tx, StageId::StorageHashing)?;
|
||||
} else {
|
||||
tx.clear::<tables::PlainAccountState>()?;
|
||||
tx.clear::<tables::PlainStorageState>()?;
|
||||
}
|
||||
tx.clear::<tables::AccountChangeSets>()?;
|
||||
tx.clear::<tables::StorageChangeSets>()?;
|
||||
tx.clear::<tables::Bytecodes>()?;
|
||||
@@ -178,7 +187,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.account_history_in_rocksdb {
|
||||
if settings.storage_v2 {
|
||||
rocksdb.clear::<tables::AccountsHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::AccountsHistory>()?;
|
||||
@@ -195,7 +204,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
let settings = provider_rw.cached_storage_settings();
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
if settings.storages_history_in_rocksdb {
|
||||
if settings.storage_v2 {
|
||||
rocksdb.clear::<tables::StoragesHistory>()?;
|
||||
} else {
|
||||
tx.clear::<tables::StoragesHistory>()?;
|
||||
@@ -209,7 +218,7 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
)?;
|
||||
}
|
||||
StageEnum::TxLookup => {
|
||||
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
if provider_rw.cached_storage_settings().storage_v2 {
|
||||
tool.provider_factory
|
||||
.rocksdb_provider()
|
||||
.clear::<tables::TransactionHashNumbers>()?;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Test setup utilities for configuring the initial state.
|
||||
|
||||
use crate::{setup_engine_with_connection, testsuite::Environment, NodeBuilderHelper};
|
||||
use crate::{testsuite::Environment, E2ETestSetupBuilder, NodeBuilderHelper};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{ForkchoiceState, PayloadAttributes};
|
||||
@@ -38,6 +38,8 @@ pub struct Setup<I> {
|
||||
shutdown_tx: Option<mpsc::Sender<()>>,
|
||||
/// Is this setup in dev mode
|
||||
pub is_dev: bool,
|
||||
/// Whether to use v2 storage mode (hashed keys, static file changesets, rocksdb history)
|
||||
pub storage_v2: bool,
|
||||
/// Tracks instance generic.
|
||||
_phantom: PhantomData<I>,
|
||||
/// Holds the import result to keep nodes alive when using imported chain
|
||||
@@ -58,6 +60,7 @@ impl<I> Default for Setup<I> {
|
||||
tree_config: TreeConfig::default(),
|
||||
shutdown_tx: None,
|
||||
is_dev: true,
|
||||
storage_v2: false,
|
||||
_phantom: Default::default(),
|
||||
import_result_holder: None,
|
||||
import_rlp_path: None,
|
||||
@@ -126,6 +129,12 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable v2 storage mode (hashed keys, static file changesets, rocksdb history)
|
||||
pub const fn with_storage_v2(mut self) -> Self {
|
||||
self.storage_v2 = true;
|
||||
self
|
||||
}
|
||||
|
||||
/// Apply setup using pre-imported chain data from RLP file
|
||||
pub async fn apply_with_import<N>(
|
||||
&mut self,
|
||||
@@ -194,19 +203,28 @@ where
|
||||
self.shutdown_tx = Some(shutdown_tx);
|
||||
|
||||
let is_dev = self.is_dev;
|
||||
let storage_v2 = self.storage_v2;
|
||||
let node_count = self.network.node_count;
|
||||
let tree_config = self.tree_config.clone();
|
||||
|
||||
let attributes_generator = Self::create_static_attributes_generator::<N>();
|
||||
|
||||
let result = setup_engine_with_connection::<N>(
|
||||
let mut builder = E2ETestSetupBuilder::<N, _>::new(
|
||||
node_count,
|
||||
Arc::<N::ChainSpec>::new((*chain_spec).clone().into()),
|
||||
is_dev,
|
||||
self.tree_config.clone(),
|
||||
attributes_generator,
|
||||
self.network.connect_nodes,
|
||||
)
|
||||
.await;
|
||||
.with_tree_config_modifier(move |base| {
|
||||
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
|
||||
})
|
||||
.with_node_config_modifier(move |config| config.set_dev(is_dev))
|
||||
.with_connect_nodes(self.network.connect_nodes);
|
||||
|
||||
if storage_v2 {
|
||||
builder = builder.with_storage_v2();
|
||||
}
|
||||
|
||||
let result = builder.build().await;
|
||||
|
||||
let mut node_clients = Vec::new();
|
||||
match result {
|
||||
|
||||
@@ -10,7 +10,6 @@ use jsonrpsee::core::client::ClientT;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
use reth_db::tables;
|
||||
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
|
||||
use reth_node_core::args::RocksDbArgs;
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_payload_builder::EthPayloadBuilderAttributes;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
@@ -96,22 +95,6 @@ fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
|
||||
/// Verifies that `RocksDB` CLI defaults are `None` (deferred to storage mode).
|
||||
#[test]
|
||||
fn test_rocksdb_defaults_are_none() {
|
||||
let args = RocksDbArgs::default();
|
||||
|
||||
assert!(args.tx_hash.is_none(), "tx_hash default should be None (deferred to --storage.v2)");
|
||||
assert!(
|
||||
args.storages_history.is_none(),
|
||||
"storages_history default should be None (deferred to --storage.v2)"
|
||||
);
|
||||
assert!(
|
||||
args.account_history.is_none(),
|
||||
"account_history default should be None (deferred to --storage.v2)"
|
||||
);
|
||||
}
|
||||
|
||||
/// Smoke test: node boots with `RocksDB` routing enabled.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_node_startup() -> Result<()> {
|
||||
@@ -477,7 +460,7 @@ async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
|
||||
///
|
||||
/// This test exercises `unwind_trie_state_from` which previously failed with
|
||||
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
|
||||
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
|
||||
/// instead of using storage-aware methods that check `is_v2()`.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
@@ -179,6 +179,8 @@ pub struct TreeConfig {
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum number of storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
/// Whether to fully disable sparse trie cache pruning between blocks.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Timeout for the state root task before spawning a sequential fallback computation.
|
||||
/// If `Some`, after waiting this duration for the state root task, a sequential state root
|
||||
/// computation is spawned in parallel and whichever finishes first is used.
|
||||
@@ -216,6 +218,7 @@ impl Default for TreeConfig {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
|
||||
}
|
||||
}
|
||||
@@ -281,6 +284,7 @@ impl TreeConfig {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout,
|
||||
}
|
||||
}
|
||||
@@ -631,6 +635,17 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether sparse trie cache pruning is disabled.
|
||||
pub const fn disable_sparse_trie_cache_pruning(&self) -> bool {
|
||||
self.disable_sparse_trie_cache_pruning
|
||||
}
|
||||
|
||||
/// Setter for whether to disable sparse trie cache pruning.
|
||||
pub const fn with_disable_sparse_trie_cache_pruning(mut self, value: bool) -> Self {
|
||||
self.disable_sparse_trie_cache_pruning = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the state root task timeout.
|
||||
pub const fn state_root_task_timeout(&self) -> Option<Duration> {
|
||||
self.state_root_task_timeout
|
||||
|
||||
@@ -20,7 +20,7 @@ use reth_node_types::{BlockTy, NodeTypes};
|
||||
use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, ProviderNodeTypes},
|
||||
ProviderFactory,
|
||||
ProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune::PrunerWithFactory;
|
||||
use reth_stages_api::{MetricEventsSender, Pipeline};
|
||||
@@ -94,6 +94,7 @@ where
|
||||
if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum };
|
||||
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
let use_hashed_state = provider.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
@@ -111,6 +112,7 @@ where
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
|
||||
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
|
||||
|
||||
@@ -73,6 +73,7 @@ reth-prune-types = { workspace = true, optional = true }
|
||||
reth-stages = { workspace = true, optional = true }
|
||||
reth-static-file = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
@@ -143,6 +144,14 @@ test-utils = [
|
||||
"reth-evm-ethereum/test-utils",
|
||||
"reth-tasks/test-utils",
|
||||
]
|
||||
rocksdb = [
|
||||
"reth-provider/rocksdb",
|
||||
"reth-prune/rocksdb",
|
||||
"reth-stages?/rocksdb",
|
||||
"reth-e2e-test-utils/rocksdb",
|
||||
]
|
||||
edge = ["rocksdb"]
|
||||
pageviz = ["dep:reth-mdbx-viz"]
|
||||
|
||||
[[test]]
|
||||
name = "e2e_testsuite"
|
||||
|
||||
@@ -351,6 +351,14 @@ impl<S: StateProvider, const PREWARM: bool> StateProvider for CachedStateProvide
|
||||
self.state_provider.storage(account, storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
self.state_provider.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BytecodeReader, const PREWARM: bool> BytecodeReader for CachedStateProvider<S, PREWARM> {
|
||||
|
||||
@@ -199,6 +199,17 @@ impl<S: StateProvider> StateProvider for InstrumentedStateProvider<S> {
|
||||
self.record_storage_fetch(start.elapsed());
|
||||
res
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<StorageValue>> {
|
||||
let start = Instant::now();
|
||||
let res = self.state_provider.storage_by_hashed_key(address, hashed_storage_key);
|
||||
self.record_storage_fetch(start.elapsed());
|
||||
res
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BytecodeReader> BytecodeReader for InstrumentedStateProvider<S> {
|
||||
|
||||
@@ -12,12 +12,13 @@ use reth_primitives_traits::constants::gas_units::MEGAGAS;
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
/// Width of each gas bucket in gas units (10 Mgas).
|
||||
const GAS_BUCKET_SIZE: u64 = 10 * MEGAGAS;
|
||||
/// Upper bounds for each gas bucket. The last bucket is a catch-all for
|
||||
/// everything above the final threshold: <5M, 5-10M, 10-20M, 20-30M, 30-40M, >40M.
|
||||
const GAS_BUCKET_THRESHOLDS: [u64; 5] =
|
||||
[5 * MEGAGAS, 10 * MEGAGAS, 20 * MEGAGAS, 30 * MEGAGAS, 40 * MEGAGAS];
|
||||
|
||||
/// Number of gas buckets. The last bucket is a catch-all for everything above
|
||||
/// `(NUM_GAS_BUCKETS - 1) * GAS_BUCKET_SIZE`.
|
||||
const NUM_GAS_BUCKETS: usize = 5;
|
||||
/// Total number of gas buckets (thresholds + 1 catch-all).
|
||||
const NUM_GAS_BUCKETS: usize = GAS_BUCKET_THRESHOLDS.len() + 1;
|
||||
|
||||
/// Metrics for the `EngineApi`.
|
||||
#[derive(Debug, Default)]
|
||||
@@ -280,21 +281,23 @@ impl GasBucketMetrics {
|
||||
}
|
||||
|
||||
fn bucket_index(gas_used: u64) -> usize {
|
||||
let idx = gas_used / GAS_BUCKET_SIZE;
|
||||
(idx as usize).min(NUM_GAS_BUCKETS - 1)
|
||||
GAS_BUCKET_THRESHOLDS
|
||||
.iter()
|
||||
.position(|&threshold| gas_used < threshold)
|
||||
.unwrap_or(GAS_BUCKET_THRESHOLDS.len())
|
||||
}
|
||||
|
||||
/// Returns a human-readable label like `<10M`, `10-20M`, … `>40M`.
|
||||
/// Returns a human-readable label like `<5M`, `5-10M`, … `>40M`.
|
||||
fn bucket_label(index: usize) -> String {
|
||||
let m = GAS_BUCKET_SIZE / 1_000_000;
|
||||
if index == 0 {
|
||||
format!("<{m}M")
|
||||
} else if index < NUM_GAS_BUCKETS - 1 {
|
||||
let lo = m * index as u64;
|
||||
let hi = lo + m;
|
||||
let hi = GAS_BUCKET_THRESHOLDS[0] / MEGAGAS;
|
||||
format!("<{hi}M")
|
||||
} else if index < GAS_BUCKET_THRESHOLDS.len() {
|
||||
let lo = GAS_BUCKET_THRESHOLDS[index - 1] / MEGAGAS;
|
||||
let hi = GAS_BUCKET_THRESHOLDS[index] / MEGAGAS;
|
||||
format!("{lo}-{hi}M")
|
||||
} else {
|
||||
let lo = m * index as u64;
|
||||
let lo = GAS_BUCKET_THRESHOLDS[GAS_BUCKET_THRESHOLDS.len() - 1] / MEGAGAS;
|
||||
format!(">{lo}M")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,7 +32,7 @@ use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
TransactionVariant,
|
||||
StorageSettingsCache, TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
@@ -271,6 +271,9 @@ where
|
||||
evm_config: C,
|
||||
/// Changeset cache for in-memory trie changesets
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Whether the node uses hashed state as canonical storage (v2 mode).
|
||||
/// Cached at construction to avoid threading `StorageSettingsCache` bounds everywhere.
|
||||
use_hashed_state: bool,
|
||||
}
|
||||
|
||||
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
|
||||
@@ -296,6 +299,7 @@ where
|
||||
.field("engine_kind", &self.engine_kind)
|
||||
.field("evm_config", &self.evm_config)
|
||||
.field("changeset_cache", &self.changeset_cache)
|
||||
.field("use_hashed_state", &self.use_hashed_state)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -313,7 +317,8 @@ where
|
||||
P::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
V: EngineValidator<T>,
|
||||
@@ -334,6 +339,7 @@ where
|
||||
engine_kind: EngineApiKind,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
use_hashed_state: bool,
|
||||
) -> Self {
|
||||
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
|
||||
|
||||
@@ -355,6 +361,7 @@ where
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -375,6 +382,7 @@ where
|
||||
kind: EngineApiKind,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
use_hashed_state: bool,
|
||||
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
|
||||
{
|
||||
let best_block_number = provider.best_block_number().unwrap_or(0);
|
||||
@@ -407,6 +415,7 @@ where
|
||||
kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
let incoming = task.incoming_tx.clone();
|
||||
spawn_os_thread("engine", || task.run());
|
||||
@@ -2379,7 +2388,12 @@ where
|
||||
|
||||
self.update_reorg_metrics(old.len(), old_first);
|
||||
self.reinsert_reorged_blocks(new.clone());
|
||||
self.reinsert_reorged_blocks(old.clone());
|
||||
|
||||
// When use_hashed_state is enabled, skip reinserting the old chain — the
|
||||
// bundle state references plain state reverts which don't exist.
|
||||
if !self.use_hashed_state {
|
||||
self.reinsert_reorged_blocks(old.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// update the tracked in-memory state with the new chain
|
||||
@@ -2663,8 +2677,23 @@ where
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(block_num_hash.number, true, 0, 0, 0);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
let pageviz_start = std::time::Instant::now();
|
||||
|
||||
let executed = execute(&mut self.payload_validator, input, ctx)?;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(
|
||||
block_num_hash.number,
|
||||
false,
|
||||
executed.recovered_block().senders().len() as u16,
|
||||
pageviz_start.elapsed().as_millis() as u32,
|
||||
executed.recovered_block().gas_used(),
|
||||
);
|
||||
|
||||
// if the parent is the canonical head, we can insert the block as the pending block
|
||||
if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash()
|
||||
{
|
||||
|
||||
@@ -94,6 +94,9 @@ pub const SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
/// 144MB.
|
||||
pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
|
||||
/// Blocks with fewer transactions than this skip prewarming, since the fixed overhead of spawning
|
||||
/// prewarm workers exceeds the execution time saved.
|
||||
pub const SMALL_BLOCK_TX_THRESHOLD: usize = 5;
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
|
||||
@@ -135,6 +138,8 @@ where
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum storage tries to retain after pruning.
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
/// Whether sparse trie cache pruning is fully disabled.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
@@ -170,6 +175,7 @@ where
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
|
||||
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
@@ -466,7 +472,8 @@ where
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let skip_prewarm = self.disable_transaction_prewarming;
|
||||
let skip_prewarm =
|
||||
self.disable_transaction_prewarming || env.transaction_count < SMALL_BLOCK_TX_THRESHOLD;
|
||||
|
||||
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
|
||||
|
||||
@@ -546,6 +553,7 @@ where
|
||||
let disable_trie_cache = config.disable_trie_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
let disable_cache_pruning = self.disable_sparse_trie_cache_pruning;
|
||||
let chunk_size =
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
|
||||
let executor = self.executor.clone();
|
||||
@@ -642,6 +650,7 @@ where
|
||||
max_storage_tries,
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
disable_cache_pruning,
|
||||
);
|
||||
trie_metrics
|
||||
.into_trie_for_reuse_duration_histogram
|
||||
|
||||
@@ -1541,6 +1541,7 @@ mod tests {
|
||||
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
|
||||
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
|
||||
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
|
||||
StorageSettingsCache,
|
||||
};
|
||||
use reth_trie::MultiProof;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
@@ -1562,6 +1563,7 @@ mod tests {
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ BlockNumReader,
|
||||
> + Clone
|
||||
+ Send
|
||||
@@ -1581,7 +1583,10 @@ mod tests {
|
||||
fn create_cached_provider<F>(factory: F) -> CachedStateProvider<StateProviderBox>
|
||||
where
|
||||
F: DatabaseProviderFactory<
|
||||
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader,
|
||||
Provider: BlockReader
|
||||
+ StageCheckpointReader
|
||||
+ PruneCheckpointReader
|
||||
+ reth_provider::StorageSettingsCache,
|
||||
> + Clone
|
||||
+ Send
|
||||
+ 'static,
|
||||
|
||||
@@ -72,6 +72,7 @@ where
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
@@ -80,6 +81,7 @@ where
|
||||
max_storage_tries,
|
||||
max_nodes_capacity,
|
||||
max_values_capacity,
|
||||
disable_pruning,
|
||||
),
|
||||
}
|
||||
}
|
||||
@@ -356,16 +358,23 @@ where
|
||||
/// Prunes and shrinks the trie for reuse in the next payload built on top of this one.
|
||||
///
|
||||
/// Should be called after the state root result has been sent.
|
||||
///
|
||||
/// When `disable_pruning` is true, the trie is preserved without any node pruning,
|
||||
/// storage trie eviction, or capacity shrinking, keeping the full cache intact for
|
||||
/// benchmarking purposes.
|
||||
pub(super) fn into_trie_for_reuse(
|
||||
self,
|
||||
prune_depth: usize,
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
let Self { mut trie, .. } = self;
|
||||
trie.prune(prune_depth, max_storage_tries);
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
if !disable_pruning {
|
||||
trie.prune(prune_depth, max_storage_tries);
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
}
|
||||
let deferred = trie.take_deferred_drops();
|
||||
(trie, deferred)
|
||||
}
|
||||
@@ -407,7 +416,9 @@ where
|
||||
let update = match message {
|
||||
Ok(m) => m,
|
||||
Err(_) => {
|
||||
break
|
||||
return Err(ParallelStateRootError::Other(
|
||||
"updates channel disconnected before state root calculation".to_string(),
|
||||
))
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -38,7 +38,7 @@ use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, State};
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
@@ -146,7 +146,8 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ ChangeSetReader
|
||||
+ BlockNumReader
|
||||
@@ -1526,7 +1527,8 @@ where
|
||||
+ PruneCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ BlockNumReader,
|
||||
+ BlockNumReader
|
||||
+ StorageSettingsCache,
|
||||
> + BlockReader<Header = N::BlockHeader>
|
||||
+ StateProviderFactory
|
||||
+ StateReader
|
||||
|
||||
@@ -221,6 +221,7 @@ impl TestHarness {
|
||||
EngineApiKind::Ethereum,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
provider.cached_storage_settings().use_hashed_state(),
|
||||
);
|
||||
|
||||
let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone());
|
||||
|
||||
@@ -2,13 +2,15 @@
|
||||
|
||||
mod fcu_finalized_blocks;
|
||||
|
||||
use alloy_rpc_types_engine::PayloadStatusEnum;
|
||||
use eyre::Result;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_e2e_test_utils::testsuite::{
|
||||
actions::{
|
||||
CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus, MakeCanonical,
|
||||
ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo, SelectActiveNode,
|
||||
SendNewPayloads, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync,
|
||||
BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus,
|
||||
MakeCanonical, ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo,
|
||||
SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, SetForkBase, UpdateBlockInfo,
|
||||
ValidateCanonicalTag, WaitForSync,
|
||||
},
|
||||
setup::{NetworkSetup, Setup},
|
||||
TestBuilder,
|
||||
@@ -39,6 +41,14 @@ fn default_engine_tree_setup() -> Setup<EthEngineTypes> {
|
||||
)
|
||||
}
|
||||
|
||||
/// Creates a v2 storage mode setup for engine tree e2e tests.
|
||||
///
|
||||
/// v2 mode uses keccak256-hashed slot keys in static file changesets and rocksdb history
|
||||
/// instead of plain keys in MDBX.
|
||||
fn v2_engine_tree_setup() -> Setup<EthEngineTypes> {
|
||||
default_engine_tree_setup().with_storage_v2()
|
||||
}
|
||||
|
||||
/// Test that verifies forkchoice update and canonical chain insertion functionality.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_canon_chain_insertion_e2e() -> Result<()> {
|
||||
@@ -334,3 +344,152 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ==================== v2 storage mode variants ====================
|
||||
|
||||
/// v2 variant: Verifies forkchoice update and canonical chain insertion in v2 storage mode.
|
||||
///
|
||||
/// Exercises the full `save_blocks` → `write_state` → static file changeset path with hashed keys.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_canon_chain_insertion_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
|
||||
.with_action(MakeCanonical::new());
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies forkchoice update with a reorg where all blocks are already available.
|
||||
///
|
||||
/// Exercises `write_state_reverts` path with hashed changeset keys during CL-driven reorgs.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(5))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(CreateFork::<EthEngineTypes>::new(2, 3))
|
||||
.with_action(CaptureBlock::new("fork_tip"))
|
||||
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("fork_tip"));
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies progressive canonical chain extension in v2 storage mode.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_fcu_extends_canon_chain_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let test = TestBuilder::new()
|
||||
.with_setup(v2_engine_tree_setup())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(1))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(10))
|
||||
.with_action(CaptureBlock::new("target_block"))
|
||||
.with_action(ReorgTo::<EthEngineTypes>::new_from_tag("target_block"))
|
||||
.with_action(MakeCanonical::new());
|
||||
|
||||
test.run::<EthereumNode>().await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a 2-node setup for disk-level reorg testing.
|
||||
///
|
||||
/// Uses unconnected nodes so fork blocks can be produced independently on Node 1 and then
|
||||
/// sent to Node 0 via newPayload only (no FCU), keeping Node 0's persisted chain intact
|
||||
/// until the final `ReorgTo` triggers `find_disk_reorg`.
|
||||
fn disk_reorg_setup(storage_v2: bool) -> Setup<EthEngineTypes> {
|
||||
let mut setup = Setup::default()
|
||||
.with_chain_spec(Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(
|
||||
serde_json::from_str(include_str!(
|
||||
"../../../../e2e-test-utils/src/testsuite/assets/genesis.json"
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.cancun_activated()
|
||||
.build(),
|
||||
))
|
||||
.with_network(NetworkSetup::multi_node_unconnected(2))
|
||||
.with_tree_config(
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
|
||||
);
|
||||
if storage_v2 {
|
||||
setup = setup.with_storage_v2();
|
||||
}
|
||||
setup
|
||||
}
|
||||
|
||||
/// Builds a disk-level reorg test scenario.
|
||||
///
|
||||
/// 1. Both nodes receive 3 shared blocks
|
||||
/// 2. Node 0 extends to 10 blocks locally (persisted to disk)
|
||||
/// 3. Node 1 builds an 8-block fork from block 3 (its canonical head)
|
||||
/// 4. Fork blocks are sent to Node 0 via newPayload (no FCU, old chain stays on disk)
|
||||
/// 5. FCU to fork tip on Node 0 triggers `find_disk_reorg` → `RemoveBlocksAbove(3)`
|
||||
fn disk_reorg_test(storage_v2: bool) -> TestBuilder<EthEngineTypes> {
|
||||
TestBuilder::new()
|
||||
.with_setup(disk_reorg_setup(storage_v2))
|
||||
.with_action(SelectActiveNode::new(0))
|
||||
.with_action(ProduceBlocks::<EthEngineTypes>::new(3))
|
||||
.with_action(MakeCanonical::new())
|
||||
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(7))
|
||||
.with_action(MakeCanonical::with_active_node())
|
||||
.with_action(SelectActiveNode::new(1))
|
||||
.with_action(SetForkBase::new(3))
|
||||
.with_action(ProduceBlocksLocally::<EthEngineTypes>::new(8))
|
||||
.with_action(MakeCanonical::with_active_node())
|
||||
.with_action(CaptureBlock::new("fork_tip"))
|
||||
.with_action(
|
||||
SendNewPayloads::<EthEngineTypes>::new()
|
||||
.with_source_node(1)
|
||||
.with_target_node(0)
|
||||
.with_start_block(4)
|
||||
.with_total_blocks(8),
|
||||
)
|
||||
.with_action(
|
||||
SendForkchoiceUpdate::<EthEngineTypes>::new(
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
BlockReference::Tag("fork_tip".into()),
|
||||
)
|
||||
.with_expected_status(PayloadStatusEnum::Valid)
|
||||
.with_node_idx(0),
|
||||
)
|
||||
}
|
||||
|
||||
/// Verifies disk-level reorg in v1 (plain key) storage mode.
|
||||
///
|
||||
/// Confirms `find_disk_reorg()` detects persisted blocks on the wrong fork and calls
|
||||
/// `RemoveBlocksAbove` to truncate, then re-persists the correct fork chain.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_disk_reorg_v1_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
disk_reorg_test(false).run::<EthereumNode>().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// v2 variant: Verifies disk-level reorg in v2 storage mode.
|
||||
///
|
||||
/// Same scenario as v1 but with hashed changeset keys in static files and rocksdb history.
|
||||
/// Exercises `find_disk_reorg()` → `RemoveBlocksAbove` with v2 hashed key format.
|
||||
#[tokio::test]
|
||||
async fn test_engine_tree_disk_reorg_v2_e2e() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
disk_reorg_test(true).run::<EthereumNode>().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ use reth_provider::{
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_testing_utils::generators::sign_tx_with_key_pair;
|
||||
use reth_trie_common::KeccakKeyHasher;
|
||||
use secp256k1::Keypair;
|
||||
|
||||
pub(crate) fn to_execution_outcome(
|
||||
@@ -77,12 +78,9 @@ where
|
||||
let execution_outcome = to_execution_outcome(block.number(), &block_execution_output);
|
||||
|
||||
// Commit the block's execution outcome to the database
|
||||
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
|
||||
let provider_rw = provider_factory.provider_rw()?;
|
||||
provider_rw.append_blocks_with_state(
|
||||
vec![block.clone()],
|
||||
&execution_outcome,
|
||||
Default::default(),
|
||||
)?;
|
||||
provider_rw.append_blocks_with_state(vec![block.clone()], &execution_outcome, hashed_state)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
Ok(block_execution_output)
|
||||
@@ -210,11 +208,12 @@ where
|
||||
execution_outcome.state_mut().reverts.sort();
|
||||
|
||||
// Commit the block's execution outcome to the database
|
||||
let hashed_state = execution_outcome.hash_state_slow::<KeccakKeyHasher>().into_sorted();
|
||||
let provider_rw = provider_factory.provider_rw()?;
|
||||
provider_rw.append_blocks_with_state(
|
||||
vec![block1.clone(), block2.clone()],
|
||||
&execution_outcome,
|
||||
Default::default(),
|
||||
hashed_state,
|
||||
)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
|
||||
@@ -589,7 +589,7 @@ impl EthMessageID {
|
||||
|
||||
/// Returns the max value for the given version.
|
||||
pub const fn max(version: EthVersion) -> u8 {
|
||||
if version.is_eth69() {
|
||||
if version as u8 >= EthVersion::Eth69 as u8 {
|
||||
Self::BlockRangeUpdate.to_u8()
|
||||
} else {
|
||||
Self::Receipts.to_u8()
|
||||
@@ -937,6 +937,13 @@ mod tests {
|
||||
assert!(matches!(decoded, StatusMessage::Legacy(s) if s == status));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn eth_message_id_max_includes_block_range_update() {
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth69), EthMessageID::BlockRangeUpdate.to_u8(),);
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth70), EthMessageID::BlockRangeUpdate.to_u8(),);
|
||||
assert_eq!(EthMessageID::max(EthVersion::Eth68), EthMessageID::Receipts.to_u8());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_status_rejects_non_status() {
|
||||
let msg = EthMessage::<EthNetworkPrimitives>::GetBlockBodies(RequestPair {
|
||||
|
||||
@@ -8,7 +8,7 @@ use reth_eth_wire_types::{
|
||||
};
|
||||
use reth_ethereum_forks::ForkId;
|
||||
use reth_network_p2p::error::{RequestError, RequestResult};
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_network_peers::{NodeRecord, PeerId};
|
||||
use reth_network_types::{PeerAddr, PeerKind};
|
||||
use reth_tokio_util::EventStream;
|
||||
use std::{
|
||||
@@ -152,8 +152,13 @@ pub trait NetworkEventListenerProvider: NetworkPeersEvents {
|
||||
pub enum DiscoveryEvent {
|
||||
/// Discovered a node
|
||||
NewNode(DiscoveredEvent),
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
|
||||
EnrForkId(PeerId, ForkId),
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request.
|
||||
///
|
||||
/// Contains the full [`NodeRecord`] (peer ID + address) and the reported [`ForkId`].
|
||||
/// Used to verify fork compatibility before admitting the peer.
|
||||
///
|
||||
/// See also <https://eips.ethereum.org/EIPS/eip-868>
|
||||
EnrForkId(NodeRecord, ForkId),
|
||||
}
|
||||
|
||||
/// Represents events related to peer discovery in the network.
|
||||
|
||||
@@ -172,6 +172,11 @@ pub struct PeersConfig {
|
||||
/// IPs within the specified CIDR ranges will be allowed.
|
||||
#[cfg_attr(feature = "serde", serde(skip))]
|
||||
pub ip_filter: IpFilter,
|
||||
/// If true, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId)
|
||||
/// (EIP-868) will not be added to the peer set until their fork ID is verified.
|
||||
///
|
||||
/// This filters out peers from other networks that pollute the discovery table.
|
||||
pub enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl Default for PeersConfig {
|
||||
@@ -191,6 +196,7 @@ impl Default for PeersConfig {
|
||||
max_backoff_count: 5,
|
||||
incoming_ip_throttle_duration: INBOUND_IP_THROTTLE_DURATION,
|
||||
ip_filter: IpFilter::default(),
|
||||
enforce_enr_fork_id: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -314,6 +320,13 @@ impl PeersConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// If set, discovered peers without a confirmed ENR [`ForkId`](alloy_eip2124::ForkId) will not
|
||||
/// be added to the peer set until their fork ID is verified via EIP-868.
|
||||
pub const fn with_enforce_enr_fork_id(mut self, enforce: bool) -> Self {
|
||||
self.enforce_enr_fork_id = enforce;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns settings for testing
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
pub fn test() -> Self {
|
||||
|
||||
@@ -240,7 +240,7 @@ impl Discovery {
|
||||
self.on_node_record_update(record, None);
|
||||
}
|
||||
DiscoveryUpdate::EnrForkId(node, fork_id) => {
|
||||
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node.id, fork_id))
|
||||
self.queued_events.push_back(DiscoveryEvent::EnrForkId(node, fork_id))
|
||||
}
|
||||
DiscoveryUpdate::Removed(peer_id) => {
|
||||
self.discovered_nodes.remove(&peer_id);
|
||||
|
||||
@@ -25,7 +25,7 @@ use crate::{
|
||||
listener::ConnectionListener,
|
||||
message::{NewBlockMessage, PeerMessage},
|
||||
metrics::{
|
||||
BackedOffPeersMetrics, ClosedSessionsMetrics, DisconnectMetrics, NetworkMetrics,
|
||||
BackedOffPeersMetrics, ClosedSessionsMetrics, DirectionalDisconnectMetrics, NetworkMetrics,
|
||||
PendingSessionFailureMetrics, NETWORK_POOL_TRANSACTIONS_SCOPE,
|
||||
},
|
||||
network::{NetworkHandle, NetworkHandleMessage},
|
||||
@@ -140,8 +140,8 @@ pub struct NetworkManager<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
num_active_peers: Arc<AtomicUsize>,
|
||||
/// Metrics for the Network
|
||||
metrics: NetworkMetrics,
|
||||
/// Disconnect metrics for the Network
|
||||
disconnect_metrics: DisconnectMetrics,
|
||||
/// Disconnect metrics for the Network, split by connection direction.
|
||||
disconnect_metrics: DirectionalDisconnectMetrics,
|
||||
/// Closed sessions metrics, split by direction.
|
||||
closed_sessions_metrics: ClosedSessionsMetrics,
|
||||
/// Pending session failure metrics, split by direction.
|
||||
@@ -864,6 +864,9 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
"Session disconnected"
|
||||
);
|
||||
|
||||
// Capture direction before state is reset to Idle
|
||||
let is_inbound = self.swarm.state().peers().is_inbound_peer(&peer_id);
|
||||
|
||||
let reason = if let Some(ref err) = error {
|
||||
// If the connection was closed due to an error, we report
|
||||
// the peer
|
||||
@@ -887,7 +890,11 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
self.update_active_connection_metrics();
|
||||
|
||||
if let Some(reason) = reason {
|
||||
self.disconnect_metrics.increment(reason);
|
||||
if is_inbound {
|
||||
self.disconnect_metrics.increment_inbound(reason);
|
||||
} else {
|
||||
self.disconnect_metrics.increment_outbound(reason);
|
||||
}
|
||||
}
|
||||
self.metrics
|
||||
.backed_off_peers
|
||||
@@ -910,7 +917,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
.on_incoming_pending_session_dropped(remote_addr, err);
|
||||
self.pending_session_failure_metrics.inbound.increment(1);
|
||||
if let Some(reason) = err.as_disconnected() {
|
||||
self.disconnect_metrics.increment(reason);
|
||||
self.disconnect_metrics.increment_inbound(reason);
|
||||
}
|
||||
} else {
|
||||
self.swarm
|
||||
@@ -943,7 +950,7 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
BackoffReason::from_disconnect(err.as_disconnected()),
|
||||
);
|
||||
if let Some(reason) = err.as_disconnected() {
|
||||
self.disconnect_metrics.increment(reason);
|
||||
self.disconnect_metrics.increment_outbound(reason);
|
||||
}
|
||||
} else {
|
||||
self.swarm
|
||||
|
||||
@@ -301,6 +301,34 @@ macro_rules! duration_metered_exec {
|
||||
}};
|
||||
}
|
||||
|
||||
/// Direction-aware wrapper for disconnect metrics.
|
||||
///
|
||||
/// Tracks disconnect reasons for inbound and outbound connections separately, in addition to
|
||||
/// the combined (legacy) counters.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct DirectionalDisconnectMetrics {
|
||||
/// Combined disconnect metrics (all directions).
|
||||
pub(crate) total: DisconnectMetrics,
|
||||
/// Disconnect metrics for inbound connections only.
|
||||
pub(crate) inbound: InboundDisconnectMetrics,
|
||||
/// Disconnect metrics for outbound connections only.
|
||||
pub(crate) outbound: OutboundDisconnectMetrics,
|
||||
}
|
||||
|
||||
impl DirectionalDisconnectMetrics {
|
||||
/// Increments disconnect counters for an inbound connection.
|
||||
pub(crate) fn increment_inbound(&self, reason: DisconnectReason) {
|
||||
self.total.increment(reason);
|
||||
self.inbound.increment(reason);
|
||||
}
|
||||
|
||||
/// Increments disconnect counters for an outbound connection.
|
||||
pub(crate) fn increment_outbound(&self, reason: DisconnectReason) {
|
||||
self.total.increment(reason);
|
||||
self.outbound.increment(reason);
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for Disconnection types
|
||||
///
|
||||
/// These are just counters, and ideally we would implement these metrics on a peer-by-peer basis,
|
||||
@@ -370,6 +398,144 @@ impl DisconnectMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
/// Disconnect metrics scoped to inbound connections only.
|
||||
///
|
||||
/// These counters track disconnect reasons exclusively for sessions that were initiated by
|
||||
/// remote peers connecting to this node. This helps operators distinguish between being rejected
|
||||
/// by remote peers (outbound) vs rejecting incoming peers (inbound).
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network.inbound")]
|
||||
pub struct InboundDisconnectMetrics {
|
||||
/// Number of inbound peer disconnects due to `DisconnectRequested` (0x00)
|
||||
pub(crate) disconnect_requested: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `TcpSubsystemError` (0x01)
|
||||
pub(crate) tcp_subsystem_error: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ProtocolBreach` (0x02)
|
||||
pub(crate) protocol_breach: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `UselessPeer` (0x03)
|
||||
pub(crate) useless_peer: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `TooManyPeers` (0x04)
|
||||
pub(crate) too_many_peers: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `AlreadyConnected` (0x05)
|
||||
pub(crate) already_connected: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
|
||||
pub(crate) incompatible: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `NullNodeIdentity` (0x07)
|
||||
pub(crate) null_node_identity: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ClientQuitting` (0x08)
|
||||
pub(crate) client_quitting: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
|
||||
pub(crate) unexpected_identity: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `ConnectedToSelf` (0x0a)
|
||||
pub(crate) connected_to_self: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `PingTimeout` (0x0b)
|
||||
pub(crate) ping_timeout: Counter,
|
||||
|
||||
/// Number of inbound peer disconnects due to `SubprotocolSpecific` (0x10)
|
||||
pub(crate) subprotocol_specific: Counter,
|
||||
}
|
||||
|
||||
impl InboundDisconnectMetrics {
|
||||
/// Increments the proper counter for the given disconnect reason
|
||||
pub(crate) fn increment(&self, reason: DisconnectReason) {
|
||||
match reason {
|
||||
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
|
||||
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
|
||||
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
|
||||
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
|
||||
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
|
||||
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
|
||||
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
|
||||
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
|
||||
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
|
||||
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
|
||||
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
|
||||
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
|
||||
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Disconnect metrics scoped to outbound connections only.
|
||||
///
|
||||
/// These counters track disconnect reasons exclusively for sessions that this node initiated
|
||||
/// by dialing out to remote peers. A high `too_many_peers` count here indicates remote peers
|
||||
/// are rejecting our connection attempts because they are full.
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network.outbound")]
|
||||
pub struct OutboundDisconnectMetrics {
|
||||
/// Number of outbound peer disconnects due to `DisconnectRequested` (0x00)
|
||||
pub(crate) disconnect_requested: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `TcpSubsystemError` (0x01)
|
||||
pub(crate) tcp_subsystem_error: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ProtocolBreach` (0x02)
|
||||
pub(crate) protocol_breach: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `UselessPeer` (0x03)
|
||||
pub(crate) useless_peer: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `TooManyPeers` (0x04)
|
||||
pub(crate) too_many_peers: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `AlreadyConnected` (0x05)
|
||||
pub(crate) already_connected: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `IncompatibleP2PProtocolVersion` (0x06)
|
||||
pub(crate) incompatible: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `NullNodeIdentity` (0x07)
|
||||
pub(crate) null_node_identity: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ClientQuitting` (0x08)
|
||||
pub(crate) client_quitting: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `UnexpectedHandshakeIdentity` (0x09)
|
||||
pub(crate) unexpected_identity: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `ConnectedToSelf` (0x0a)
|
||||
pub(crate) connected_to_self: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `PingTimeout` (0x0b)
|
||||
pub(crate) ping_timeout: Counter,
|
||||
|
||||
/// Number of outbound peer disconnects due to `SubprotocolSpecific` (0x10)
|
||||
pub(crate) subprotocol_specific: Counter,
|
||||
}
|
||||
|
||||
impl OutboundDisconnectMetrics {
|
||||
/// Increments the proper counter for the given disconnect reason
|
||||
pub(crate) fn increment(&self, reason: DisconnectReason) {
|
||||
match reason {
|
||||
DisconnectReason::DisconnectRequested => self.disconnect_requested.increment(1),
|
||||
DisconnectReason::TcpSubsystemError => self.tcp_subsystem_error.increment(1),
|
||||
DisconnectReason::ProtocolBreach => self.protocol_breach.increment(1),
|
||||
DisconnectReason::UselessPeer => self.useless_peer.increment(1),
|
||||
DisconnectReason::TooManyPeers => self.too_many_peers.increment(1),
|
||||
DisconnectReason::AlreadyConnected => self.already_connected.increment(1),
|
||||
DisconnectReason::IncompatibleP2PProtocolVersion => self.incompatible.increment(1),
|
||||
DisconnectReason::NullNodeIdentity => self.null_node_identity.increment(1),
|
||||
DisconnectReason::ClientQuitting => self.client_quitting.increment(1),
|
||||
DisconnectReason::UnexpectedHandshakeIdentity => self.unexpected_identity.increment(1),
|
||||
DisconnectReason::ConnectedToSelf => self.connected_to_self.increment(1),
|
||||
DisconnectReason::PingTimeout => self.ping_timeout.increment(1),
|
||||
DisconnectReason::SubprotocolSpecific => self.subprotocol_specific.increment(1),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for the `EthRequestHandler`
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "network")]
|
||||
|
||||
@@ -92,6 +92,9 @@ pub struct PeersManager {
|
||||
incoming_ip_throttle_duration: Duration,
|
||||
/// IP address filter for restricting network connections to specific IP ranges.
|
||||
ip_filter: reth_net_banlist::IpFilter,
|
||||
/// If true, discovered peers without a confirmed ENR fork ID will not be added until their
|
||||
/// fork ID is verified via EIP-868.
|
||||
enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl PeersManager {
|
||||
@@ -111,6 +114,7 @@ impl PeersManager {
|
||||
max_backoff_count,
|
||||
incoming_ip_throttle_duration,
|
||||
ip_filter,
|
||||
enforce_enr_fork_id,
|
||||
} = config;
|
||||
let (manager_tx, handle_rx) = mpsc::unbounded_channel();
|
||||
let now = Instant::now();
|
||||
@@ -167,6 +171,7 @@ impl PeersManager {
|
||||
net_connection_state: NetworkConnectionState::default(),
|
||||
incoming_ip_throttle_duration,
|
||||
ip_filter,
|
||||
enforce_enr_fork_id,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,6 +180,11 @@ impl PeersManager {
|
||||
PeersHandle::new(self.manager_tx.clone())
|
||||
}
|
||||
|
||||
/// Returns `true` if discovered peers must have a confirmed ENR fork ID before being added.
|
||||
pub(crate) const fn enforce_enr_fork_id(&self) -> bool {
|
||||
self.enforce_enr_fork_id
|
||||
}
|
||||
|
||||
/// Returns the number of peers in the peer set
|
||||
#[inline]
|
||||
pub(crate) fn num_known_peers(&self) -> usize {
|
||||
@@ -208,6 +218,13 @@ impl PeersManager {
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if the given peer is connected via an inbound session.
|
||||
pub(crate) fn is_inbound_peer(&self, peer_id: &PeerId) -> bool {
|
||||
self.peers.get(peer_id).is_some_and(|p| {
|
||||
matches!(p.state, PeerConnectionState::In | PeerConnectionState::DisconnectingIn)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns an iterator over all peer ids for peers with the given kind
|
||||
pub(crate) fn peers_by_kind(&self, kind: PeerKind) -> impl Iterator<Item = PeerId> + '_ {
|
||||
self.peers.iter().filter_map(move |(peer_id, peer)| (peer.kind == kind).then_some(*peer_id))
|
||||
@@ -738,17 +755,6 @@ impl PeersManager {
|
||||
}
|
||||
}
|
||||
|
||||
/// Called as follow-up for a discovered peer.
|
||||
///
|
||||
/// The [`ForkId`] is retrieved from an ENR record that the peer announces over the discovery
|
||||
/// protocol
|
||||
pub(crate) fn set_discovered_fork_id(&mut self, peer_id: PeerId, fork_id: ForkId) {
|
||||
if let Some(peer) = self.peers.get_mut(&peer_id) {
|
||||
trace!(target: "net::peers", ?peer_id, ?fork_id, "set discovered fork id");
|
||||
peer.fork_id = Some(Box::new(fork_id));
|
||||
}
|
||||
}
|
||||
|
||||
/// Called for a newly discovered peer.
|
||||
///
|
||||
/// If the peer already exists, then the address, kind and `fork_id` will be updated.
|
||||
|
||||
@@ -332,9 +332,19 @@ impl<N: NetworkPrimitives> NetworkState<N> {
|
||||
fork_id,
|
||||
});
|
||||
}
|
||||
DiscoveryEvent::EnrForkId(peer_id, fork_id) => {
|
||||
self.queued_messages
|
||||
.push_back(StateAction::DiscoveredEnrForkId { peer_id, fork_id });
|
||||
DiscoveryEvent::EnrForkId(record, fork_id) => {
|
||||
let peer_id = record.id;
|
||||
let tcp_addr = record.tcp_addr();
|
||||
if tcp_addr.port() == 0 {
|
||||
return
|
||||
}
|
||||
let udp_addr = record.udp_addr();
|
||||
let addr = PeerAddr::new(tcp_addr, Some(udp_addr));
|
||||
self.queued_messages.push_back(StateAction::DiscoveredEnrForkId {
|
||||
peer_id,
|
||||
addr,
|
||||
fork_id,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -552,6 +562,8 @@ pub(crate) enum StateAction<N: NetworkPrimitives> {
|
||||
/// Retrieved a [`ForkId`] from the peer via ENR request, See <https://eips.ethereum.org/EIPS/eip-868>
|
||||
DiscoveredEnrForkId {
|
||||
peer_id: PeerId,
|
||||
/// The address of the peer.
|
||||
addr: PeerAddr,
|
||||
/// The reported [`ForkId`] by this peer.
|
||||
fork_id: ForkId,
|
||||
},
|
||||
|
||||
@@ -246,18 +246,28 @@ impl<N: NetworkPrimitives> Swarm<N> {
|
||||
StateAction::PeerAdded(peer_id) => return Some(SwarmEvent::PeerAdded(peer_id)),
|
||||
StateAction::PeerRemoved(peer_id) => return Some(SwarmEvent::PeerRemoved(peer_id)),
|
||||
StateAction::DiscoveredNode { peer_id, addr, fork_id } => {
|
||||
// Don't try to connect to peer if node is shutting down
|
||||
if self.is_shutting_down() {
|
||||
return None
|
||||
}
|
||||
// Insert peer only if no fork id or a valid fork id
|
||||
if fork_id.map_or_else(|| true, |f| self.sessions.is_valid_fork_id(f)) {
|
||||
|
||||
// When `enforce_enr_fork_id` is enabled, peers discovered without a confirmed
|
||||
// fork ID (via EIP-868 ENR) are deferred — they'll only be added once a
|
||||
// `DiscoveredEnrForkId` event arrives with a validated fork ID.
|
||||
//
|
||||
// When disabled (default), peers without a fork ID are admitted immediately.
|
||||
// Peers that *do* carry a fork ID are always validated against ours.
|
||||
let enforce = self.state().peers().enforce_enr_fork_id();
|
||||
let allow = match fork_id {
|
||||
Some(f) => self.sessions.is_valid_fork_id(f),
|
||||
None => !enforce,
|
||||
};
|
||||
if allow {
|
||||
self.state_mut().peers_mut().add_peer(peer_id, addr, fork_id);
|
||||
}
|
||||
}
|
||||
StateAction::DiscoveredEnrForkId { peer_id, fork_id } => {
|
||||
StateAction::DiscoveredEnrForkId { peer_id, addr, fork_id } => {
|
||||
if self.sessions.is_valid_fork_id(fork_id) {
|
||||
self.state_mut().peers_mut().set_discovered_fork_id(peer_id, fork_id);
|
||||
self.state_mut().peers_mut().add_peer(peer_id, addr, Some(fork_id));
|
||||
} else {
|
||||
trace!(target: "net", ?peer_id, remote_fork_id=?fork_id, our_fork_id=?self.sessions.fork_id(), "fork id mismatch, removing peer");
|
||||
self.state_mut().peers_mut().remove_peer(peer_id);
|
||||
|
||||
@@ -129,3 +129,4 @@ op = [
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-engine-tree/pageviz"]
|
||||
|
||||
@@ -43,6 +43,7 @@ pub struct DefaultEngineValues {
|
||||
disable_trie_cache: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
state_root_task_timeout: Option<String>,
|
||||
}
|
||||
|
||||
@@ -198,6 +199,12 @@ impl DefaultEngineValues {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set whether to disable sparse trie cache pruning by default
|
||||
pub const fn with_disable_sparse_trie_cache_pruning(mut self, v: bool) -> Self {
|
||||
self.disable_sparse_trie_cache_pruning = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the default state root task timeout
|
||||
pub fn with_state_root_task_timeout(mut self, v: Option<String>) -> Self {
|
||||
self.state_root_task_timeout = v;
|
||||
@@ -231,6 +238,7 @@ impl Default for DefaultEngineValues {
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
state_root_task_timeout: Some("1s".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -372,6 +380,12 @@ pub struct EngineArgs {
|
||||
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries)]
|
||||
pub sparse_trie_max_storage_tries: usize,
|
||||
|
||||
/// Fully disable sparse trie cache pruning. When set, the cached sparse trie is preserved
|
||||
/// without any node pruning or storage trie eviction between blocks. Useful for benchmarking
|
||||
/// the effects of retaining the full trie cache.
|
||||
#[arg(long = "engine.disable-sparse-trie-cache-pruning", default_value_t = DefaultEngineValues::get_global().disable_sparse_trie_cache_pruning)]
|
||||
pub disable_sparse_trie_cache_pruning: bool,
|
||||
|
||||
/// Configure the timeout for the state root task before spawning a sequential fallback.
|
||||
/// If the state root task takes longer than this, a sequential computation starts in
|
||||
/// parallel and whichever finishes first is used.
|
||||
@@ -415,6 +429,7 @@ impl Default for EngineArgs {
|
||||
disable_trie_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning,
|
||||
state_root_task_timeout,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
@@ -445,6 +460,7 @@ impl Default for EngineArgs {
|
||||
disable_trie_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning,
|
||||
state_root_task_timeout: state_root_task_timeout
|
||||
.as_deref()
|
||||
.map(|s| humantime::parse_duration(s).expect("valid default duration")),
|
||||
@@ -480,6 +496,7 @@ impl EngineArgs {
|
||||
.with_disable_trie_cache(self.disable_trie_cache)
|
||||
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
|
||||
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
|
||||
.with_disable_sparse_trie_cache_pruning(self.disable_sparse_trie_cache_pruning)
|
||||
.with_state_root_task_timeout(self.state_root_task_timeout.filter(|d| !d.is_zero()))
|
||||
}
|
||||
}
|
||||
@@ -534,6 +551,7 @@ mod tests {
|
||||
disable_trie_cache: true,
|
||||
sparse_trie_prune_depth: 10,
|
||||
sparse_trie_max_storage_tries: 100,
|
||||
disable_sparse_trie_cache_pruning: true,
|
||||
state_root_task_timeout: Some(Duration::from_secs(2)),
|
||||
};
|
||||
|
||||
@@ -570,6 +588,7 @@ mod tests {
|
||||
"10",
|
||||
"--engine.sparse-trie-max-storage-tries",
|
||||
"100",
|
||||
"--engine.disable-sparse-trie-cache-pruning",
|
||||
"--engine.state-root-task-timeout",
|
||||
"2s",
|
||||
])
|
||||
|
||||
@@ -76,11 +76,7 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
|
||||
mod static_files;
|
||||
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
|
||||
|
||||
/// `RocksDbArgs` for configuring RocksDB table routing.
|
||||
mod rocksdb;
|
||||
pub use rocksdb::{RocksDbArgs, RocksDbArgsError};
|
||||
|
||||
/// `StorageArgs` for configuring storage mode (v2 vs v1/legacy).
|
||||
/// `StorageArgs` for configuring storage settings.
|
||||
mod storage;
|
||||
pub use storage::StorageArgs;
|
||||
|
||||
|
||||
@@ -227,6 +227,14 @@ pub struct NetworkArgs {
|
||||
/// Example: --netrestrict "192.168.0.0/16,10.0.0.0/8"
|
||||
#[arg(long, value_name = "NETRESTRICT")]
|
||||
pub netrestrict: Option<String>,
|
||||
|
||||
/// Enforce EIP-868 ENR fork ID validation for discovered peers.
|
||||
///
|
||||
/// When enabled, peers discovered without a confirmed fork ID are not added to the peer set
|
||||
/// until their fork ID is verified via EIP-868 ENR request. This filters out peers from other
|
||||
/// networks that pollute the discovery table.
|
||||
#[arg(long)]
|
||||
pub enforce_enr_fork_id: bool,
|
||||
}
|
||||
|
||||
impl NetworkArgs {
|
||||
@@ -333,7 +341,8 @@ impl NetworkArgs {
|
||||
)
|
||||
.with_max_inbound_opt(self.resolved_max_inbound_peers())
|
||||
.with_max_outbound_opt(self.resolved_max_outbound_peers())
|
||||
.with_ip_filter(ip_filter);
|
||||
.with_ip_filter(ip_filter)
|
||||
.with_enforce_enr_fork_id(self.enforce_enr_fork_id);
|
||||
|
||||
// Configure basic network stack
|
||||
NetworkConfigBuilder::<N>::new(secret_key)
|
||||
@@ -491,6 +500,7 @@ impl Default for NetworkArgs {
|
||||
required_block_hashes: vec![],
|
||||
network_id: None,
|
||||
netrestrict: None,
|
||||
enforce_enr_fork_id: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,160 +0,0 @@
|
||||
//! clap [Args](clap::Args) for `RocksDB` table routing configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Parameters for `RocksDB` table routing configuration.
|
||||
///
|
||||
/// These flags control which database tables are stored in `RocksDB` instead of MDBX.
|
||||
/// All flags are genesis-initialization-only: changing them after genesis requires a re-sync.
|
||||
///
|
||||
/// When `--storage.v2` is used, the defaults for these flags change to enable `RocksDB` routing.
|
||||
/// Individual flags can still override those defaults when explicitly set.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "RocksDB")]
|
||||
pub struct RocksDbArgs {
|
||||
/// Route all supported tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This enables `RocksDB` for `tx-hash`, `storages-history`, and `account-history` tables.
|
||||
/// Cannot be combined with individual flags set to false.
|
||||
#[arg(long = "rocksdb.all", action = ArgAction::SetTrue)]
|
||||
pub all: bool,
|
||||
|
||||
/// Route tx hash -> number table to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.tx-hash", action = ArgAction::Set)]
|
||||
pub tx_hash: Option<bool>,
|
||||
|
||||
/// Route storages history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.storages-history", action = ArgAction::Set)]
|
||||
pub storages_history: Option<bool>,
|
||||
|
||||
/// Route account history tables to `RocksDB` instead of MDBX.
|
||||
///
|
||||
/// This is a genesis-initialization-only flag: changing it after genesis requires a re-sync.
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "rocksdb.account-history", action = ArgAction::Set)]
|
||||
pub account_history: Option<bool>,
|
||||
}
|
||||
|
||||
impl RocksDbArgs {
|
||||
/// Validates the `RocksDB` arguments.
|
||||
///
|
||||
/// Returns an error if `--rocksdb.all` is used with any individual flag explicitly set to
|
||||
/// `false`.
|
||||
pub const fn validate(&self) -> Result<(), RocksDbArgsError> {
|
||||
if self.all {
|
||||
if matches!(self.tx_hash, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("tx-hash"));
|
||||
}
|
||||
if matches!(self.storages_history, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("storages-history"));
|
||||
}
|
||||
if matches!(self.account_history, Some(false)) {
|
||||
return Err(RocksDbArgsError::ConflictingFlags("account-history"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Error type for `RocksDB` argument validation.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RocksDbArgsError {
|
||||
/// `--rocksdb.all` cannot be combined with an individual flag set to false.
|
||||
#[error("--rocksdb.all cannot be combined with --rocksdb.{0}=false")]
|
||||
ConflictingFlags(&'static str),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
struct CommandParser<T: Args> {
|
||||
#[command(flatten)]
|
||||
args: T,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_rocksdb_args() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, RocksDbArgs::default());
|
||||
assert!(!args.all);
|
||||
assert!(args.tx_hash.is_none());
|
||||
assert!(args.storages_history.is_none());
|
||||
assert!(args.account_history.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_all_flag() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from(["reth", "--rocksdb.all"]).args;
|
||||
assert!(args.all);
|
||||
assert!(args.tx_hash.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_individual_flags() {
|
||||
let args = CommandParser::<RocksDbArgs>::parse_from([
|
||||
"reth",
|
||||
"--rocksdb.tx-hash=true",
|
||||
"--rocksdb.storages-history=false",
|
||||
"--rocksdb.account-history=true",
|
||||
])
|
||||
.args;
|
||||
assert!(!args.all);
|
||||
assert_eq!(args.tx_hash, Some(true));
|
||||
assert_eq!(args.storages_history, Some(false));
|
||||
assert_eq!(args.account_history, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_none_ok() {
|
||||
let args =
|
||||
RocksDbArgs { all: true, tx_hash: None, storages_history: None, account_history: None };
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_true_ok() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: Some(true),
|
||||
storages_history: Some(true),
|
||||
account_history: Some(true),
|
||||
};
|
||||
assert!(args.validate().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_validate_all_with_false_errors() {
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: Some(false),
|
||||
storages_history: None,
|
||||
account_history: None,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("tx-hash")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: None,
|
||||
storages_history: Some(false),
|
||||
account_history: None,
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("storages-history")));
|
||||
|
||||
let args = RocksDbArgs {
|
||||
all: true,
|
||||
tx_hash: None,
|
||||
storages_history: None,
|
||||
account_history: Some(false),
|
||||
};
|
||||
assert_eq!(args.validate(), Err(RocksDbArgsError::ConflictingFlags("account-history")));
|
||||
}
|
||||
}
|
||||
@@ -9,9 +9,6 @@ use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
|
||||
|
||||
/// Parameters for static files configuration
|
||||
///
|
||||
/// When `--storage.v2` is used, the defaults for the storage flags change to enable static file
|
||||
/// storage. Individual flags can still override those defaults when explicitly set.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "Static Files")]
|
||||
pub struct StaticFilesArgs {
|
||||
@@ -38,53 +35,6 @@ pub struct StaticFilesArgs {
|
||||
/// Number of blocks per file for the storage changesets segment.
|
||||
#[arg(long = "static-files.blocks-per-file.storage-change-sets")]
|
||||
pub blocks_per_file_storage_change_sets: Option<u64>,
|
||||
|
||||
/// Store receipts in static files instead of the database.
|
||||
///
|
||||
/// When enabled, receipts will be written to static files on disk instead of the database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.receipts", action = clap::ArgAction::Set)]
|
||||
pub receipts: Option<bool>,
|
||||
|
||||
/// Store transaction senders in static files instead of the database.
|
||||
///
|
||||
/// When enabled, transaction senders will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.transaction-senders", action = clap::ArgAction::Set)]
|
||||
pub transaction_senders: Option<bool>,
|
||||
|
||||
/// Store account changesets in static files.
|
||||
///
|
||||
/// When enabled, account changesets will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.account-change-sets", action = clap::ArgAction::Set)]
|
||||
pub account_changesets: Option<bool>,
|
||||
|
||||
/// Store storage changesets in static files.
|
||||
///
|
||||
/// When enabled, storage changesets will be written to static files on disk instead of the
|
||||
/// database.
|
||||
///
|
||||
/// Note: This setting can only be configured at genesis initialization. Once
|
||||
/// the node has been initialized, changing this flag requires re-syncing from scratch.
|
||||
///
|
||||
/// Defaults to the base storage mode (v1: false, v2: true).
|
||||
#[arg(long = "static-files.storage-change-sets", action = clap::ArgAction::Set)]
|
||||
pub storage_changesets: Option<bool>,
|
||||
}
|
||||
|
||||
impl StaticFilesArgs {
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
//! clap [Args](clap::Args) for storage mode configuration
|
||||
//! clap [Args](clap::Args) for storage configuration
|
||||
|
||||
use clap::{ArgAction, Args};
|
||||
|
||||
/// Parameters for storage mode configuration.
|
||||
/// Parameters for storage configuration.
|
||||
///
|
||||
/// This controls whether the node uses v2 storage defaults (with `RocksDB` and static file
|
||||
/// optimizations) or v1/legacy storage defaults.
|
||||
///
|
||||
/// Individual storage settings can be overridden with `--static-files.*` and `--rocksdb.*` flags.
|
||||
#[derive(Debug, Args, PartialEq, Eq, Clone, Copy, Default)]
|
||||
#[command(next_help_heading = "Storage")]
|
||||
pub struct StorageArgs {
|
||||
@@ -23,16 +25,6 @@ pub struct StorageArgs {
|
||||
/// flags.
|
||||
#[arg(long = "storage.v2", action = ArgAction::SetTrue)]
|
||||
pub v2: bool,
|
||||
|
||||
/// Use hashed state tables (`HashedAccounts`/`HashedStorages`) as canonical state
|
||||
/// representation instead of plain state tables.
|
||||
///
|
||||
/// When enabled, execution writes directly to hashed tables, eliminating the need for
|
||||
/// separate hashing stages. This should only be enabled for new databases.
|
||||
///
|
||||
/// WARNING: Changing this setting on an existing database requires a full resync.
|
||||
#[arg(long = "storage.use-hashed-state", default_value_t = false)]
|
||||
pub use_hashed_state: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -40,21 +32,24 @@ mod tests {
|
||||
use super::*;
|
||||
use clap::Parser;
|
||||
|
||||
/// A helper type to parse Args more easily
|
||||
#[derive(Parser)]
|
||||
struct CommandParser {
|
||||
struct CommandParser<T: Args> {
|
||||
#[command(flatten)]
|
||||
args: StorageArgs,
|
||||
args: T,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_storage_args() {
|
||||
let args = CommandParser::parse_from(["reth"]).args;
|
||||
let default_args = StorageArgs::default();
|
||||
let args = CommandParser::<StorageArgs>::parse_from(["reth"]).args;
|
||||
assert_eq!(args, default_args);
|
||||
assert!(!args.v2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_parse_v2_flag() {
|
||||
let args = CommandParser::parse_from(["reth", "--storage.v2"]).args;
|
||||
let args = CommandParser::<StorageArgs>::parse_from(["reth", "--storage.v2"]).args;
|
||||
assert!(args.v2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
args::{
|
||||
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, NetworkArgs, PayloadBuilderArgs,
|
||||
PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
|
||||
PruningArgs, RpcServerArgs, StaticFilesArgs, StorageArgs, TxPoolArgs,
|
||||
},
|
||||
dirs::{ChainPath, DataDirPath},
|
||||
utils::get_single_header,
|
||||
@@ -152,10 +152,7 @@ pub struct NodeConfig<ChainSpec> {
|
||||
/// All static files related arguments
|
||||
pub static_files: StaticFilesArgs,
|
||||
|
||||
/// All `RocksDB` table routing arguments
|
||||
pub rocksdb: RocksDbArgs,
|
||||
|
||||
/// Storage mode configuration (v2 vs v1/legacy)
|
||||
/// All storage related arguments with --storage prefix
|
||||
pub storage: StorageArgs,
|
||||
}
|
||||
|
||||
@@ -188,7 +185,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: EngineArgs::default(),
|
||||
era: EraArgs::default(),
|
||||
static_files: StaticFilesArgs::default(),
|
||||
rocksdb: RocksDbArgs::default(),
|
||||
storage: StorageArgs::default(),
|
||||
}
|
||||
}
|
||||
@@ -264,7 +260,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
storage,
|
||||
..
|
||||
} = self;
|
||||
@@ -285,7 +280,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine,
|
||||
era,
|
||||
static_files,
|
||||
rocksdb,
|
||||
storage,
|
||||
}
|
||||
}
|
||||
@@ -355,6 +349,12 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the storage args for the node
|
||||
pub const fn with_storage(mut self, storage: StorageArgs) -> Self {
|
||||
self.storage = storage;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns pruning configuration.
|
||||
pub fn prune_config(&self) -> Option<PruneConfig>
|
||||
where
|
||||
@@ -363,42 +363,17 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
self.pruning.prune_config(&self.chain)
|
||||
}
|
||||
|
||||
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
|
||||
/// `RocksDB` CLI args.
|
||||
/// Returns the effective storage settings derived from `--storage.v2`.
|
||||
///
|
||||
/// The base storage mode is determined by `--storage.v2`:
|
||||
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
|
||||
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
|
||||
///
|
||||
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
|
||||
/// - Otherwise: uses [`StorageSettings::base()`] defaults
|
||||
pub const fn storage_settings(&self) -> StorageSettings {
|
||||
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
|
||||
|
||||
// Apply static files overrides (only when explicitly set)
|
||||
s = s
|
||||
.with_receipts_in_static_files_opt(self.static_files.receipts)
|
||||
.with_transaction_senders_in_static_files_opt(self.static_files.transaction_senders)
|
||||
.with_account_changesets_in_static_files_opt(self.static_files.account_changesets)
|
||||
.with_storage_changesets_in_static_files_opt(self.static_files.storage_changesets);
|
||||
|
||||
// Apply rocksdb overrides
|
||||
// --rocksdb.all sets all rocksdb flags to true
|
||||
if self.rocksdb.all {
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb(true)
|
||||
.with_storages_history_in_rocksdb(true)
|
||||
.with_account_history_in_rocksdb(true);
|
||||
if self.storage.v2 {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::base()
|
||||
}
|
||||
|
||||
// Individual rocksdb flags override --rocksdb.all when explicitly set
|
||||
s = s
|
||||
.with_transaction_hash_numbers_in_rocksdb_opt(self.rocksdb.tx_hash)
|
||||
.with_storages_history_in_rocksdb_opt(self.rocksdb.storages_history)
|
||||
.with_account_history_in_rocksdb_opt(self.rocksdb.account_history);
|
||||
|
||||
s = s.with_use_hashed_state(self.storage.use_hashed_state);
|
||||
|
||||
s
|
||||
}
|
||||
|
||||
/// Returns the max block that the node should run to, looking it up from the network if
|
||||
@@ -595,7 +570,6 @@ impl<ChainSpec> NodeConfig<ChainSpec> {
|
||||
engine: self.engine,
|
||||
era: self.era,
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
storage: self.storage,
|
||||
}
|
||||
}
|
||||
@@ -638,7 +612,6 @@ impl<ChainSpec> Clone for NodeConfig<ChainSpec> {
|
||||
engine: self.engine.clone(),
|
||||
era: self.era.clone(),
|
||||
static_files: self.static_files,
|
||||
rocksdb: self.rocksdb,
|
||||
storage: self.storage,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,7 +265,7 @@ impl NodeState {
|
||||
warn!(number=block.number(), hash=?block.hash(), "Encountered invalid block");
|
||||
}
|
||||
ConsensusEngineEvent::BlockReceived(num_hash) => {
|
||||
info!(number=num_hash.number, hash=?num_hash.hash, "Received block from consensus engine");
|
||||
info!(number=num_hash.number, hash=?num_hash.hash, "Received new payload from consensus engine");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -164,7 +164,7 @@ pub use alloy_primitives::{logs_bloom, Log, LogData};
|
||||
pub mod proofs;
|
||||
|
||||
mod storage;
|
||||
pub use storage::{StorageEntry, ValueWithSubKey};
|
||||
pub use storage::{StorageEntry, StorageSlotKey, ValueWithSubKey};
|
||||
|
||||
pub mod sync;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{B256, U256};
|
||||
use alloy_primitives::{keccak256, B256, U256};
|
||||
|
||||
/// Trait for `DupSort` table values that contain a subkey.
|
||||
///
|
||||
@@ -12,6 +12,117 @@ pub trait ValueWithSubKey {
|
||||
fn get_subkey(&self) -> Self::SubKey;
|
||||
}
|
||||
|
||||
/// A storage slot key that tracks whether it holds a plain (unhashed) EVM slot
|
||||
/// or a keccak256-hashed slot.
|
||||
///
|
||||
/// This enum replaces the `use_hashed_state: bool` parameter pattern by carrying
|
||||
/// provenance with the key itself. Once tagged at a read/write boundary, downstream
|
||||
/// code can call [`Self::to_hashed`] without risk of double-hashing — hashing a
|
||||
/// [`StorageSlotKey::Hashed`] is a no-op.
|
||||
///
|
||||
/// The on-disk encoding is unchanged (raw 32-byte [`B256`]). The variant is set
|
||||
/// by the code that knows the context (which table, which storage mode).
|
||||
#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
|
||||
pub enum StorageSlotKey {
|
||||
/// An unhashed EVM storage slot, as produced by REVM execution.
|
||||
Plain(B256),
|
||||
/// A keccak256-hashed storage slot, as stored in `HashedStorages` and
|
||||
/// in v2-mode `StorageChangeSets`.
|
||||
Hashed(B256),
|
||||
}
|
||||
|
||||
impl Default for StorageSlotKey {
|
||||
fn default() -> Self {
|
||||
Self::Plain(B256::ZERO)
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageSlotKey {
|
||||
/// Create a plain slot key from a REVM [`U256`] storage index.
|
||||
pub const fn from_u256(slot: U256) -> Self {
|
||||
Self::Plain(B256::new(slot.to_be_bytes()))
|
||||
}
|
||||
|
||||
/// Create a plain slot key from a raw [`B256`].
|
||||
pub const fn plain(key: B256) -> Self {
|
||||
Self::Plain(key)
|
||||
}
|
||||
|
||||
/// Create a hashed slot key from a raw [`B256`].
|
||||
pub const fn hashed(key: B256) -> Self {
|
||||
Self::Hashed(key)
|
||||
}
|
||||
|
||||
/// Tag a raw [`B256`] based on the storage mode.
|
||||
///
|
||||
/// When `use_hashed_state` is true the key is assumed already hashed.
|
||||
/// When false it is assumed to be a plain slot.
|
||||
pub const fn from_raw(key: B256, use_hashed_state: bool) -> Self {
|
||||
if use_hashed_state {
|
||||
Self::Hashed(key)
|
||||
} else {
|
||||
Self::Plain(key)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the raw [`B256`] regardless of variant.
|
||||
pub const fn as_b256(&self) -> B256 {
|
||||
match *self {
|
||||
Self::Plain(b) | Self::Hashed(b) => b,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns `true` if this key is already hashed.
|
||||
pub const fn is_hashed(&self) -> bool {
|
||||
matches!(self, Self::Hashed(_))
|
||||
}
|
||||
|
||||
/// Returns `true` if this key is plain (unhashed).
|
||||
pub const fn is_plain(&self) -> bool {
|
||||
matches!(self, Self::Plain(_))
|
||||
}
|
||||
|
||||
/// Produce the keccak256-hashed form of this slot key.
|
||||
///
|
||||
/// - If already [`Hashed`](Self::Hashed), returns the inner value as-is (no double-hash).
|
||||
/// - If [`Plain`](Self::Plain), applies keccak256 and returns the result.
|
||||
pub fn to_hashed(&self) -> B256 {
|
||||
match *self {
|
||||
Self::Hashed(b) => b,
|
||||
Self::Plain(b) => keccak256(b),
|
||||
}
|
||||
}
|
||||
|
||||
/// Convert a plain slot to its changeset representation.
|
||||
///
|
||||
/// In v2 mode (`use_hashed_state = true`), the changeset stores hashed keys,
|
||||
/// so the plain key is hashed. In v1 mode, the plain key is stored as-is.
|
||||
///
|
||||
/// Panics (debug) if called on an already-hashed key.
|
||||
pub fn to_changeset_key(self, use_hashed_state: bool) -> B256 {
|
||||
debug_assert!(self.is_plain(), "to_changeset_key called on already-hashed key");
|
||||
if use_hashed_state {
|
||||
self.to_hashed()
|
||||
} else {
|
||||
self.as_b256()
|
||||
}
|
||||
}
|
||||
|
||||
/// Like [`to_changeset_key`](Self::to_changeset_key) but returns a tagged
|
||||
/// [`StorageSlotKey`] instead of a raw [`B256`].
|
||||
///
|
||||
/// Panics (debug) if called on an already-hashed key.
|
||||
pub fn to_changeset(self, use_hashed_state: bool) -> Self {
|
||||
Self::from_raw(self.to_changeset_key(use_hashed_state), use_hashed_state)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StorageSlotKey> for B256 {
|
||||
fn from(key: StorageSlotKey) -> Self {
|
||||
key.as_b256()
|
||||
}
|
||||
}
|
||||
|
||||
/// Account storage entry.
|
||||
///
|
||||
/// `key` is the subkey when used as a value in the `StorageChangeSets` table.
|
||||
@@ -31,6 +142,14 @@ impl StorageEntry {
|
||||
pub const fn new(key: B256, value: U256) -> Self {
|
||||
Self { key, value }
|
||||
}
|
||||
|
||||
/// Tag this entry's key as a [`StorageSlotKey`] based on the storage mode.
|
||||
///
|
||||
/// When `use_hashed_state` is true, the key is tagged as already-hashed.
|
||||
/// When false, it is tagged as plain.
|
||||
pub const fn slot_key(&self, use_hashed_state: bool) -> StorageSlotKey {
|
||||
StorageSlotKey::from_raw(self.key, use_hashed_state)
|
||||
}
|
||||
}
|
||||
|
||||
impl ValueWithSubKey for StorageEntry {
|
||||
|
||||
@@ -75,7 +75,7 @@ where
|
||||
|
||||
// Check where account history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
@@ -405,9 +405,7 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -508,7 +506,11 @@ mod tests {
|
||||
test_prune(1400, 3, (PruneProgress::Finished, 804));
|
||||
}
|
||||
|
||||
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
|
||||
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
|
||||
/// `prune_rocksdb_path` test covers that configuration).
|
||||
#[test]
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
fn prune_static_file() {
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
@@ -564,9 +566,7 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(true),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -714,11 +714,7 @@ mod tests {
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_account_changesets_in_static_files(true)
|
||||
.with_account_history_in_rocksdb(true),
|
||||
);
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
@@ -832,9 +828,7 @@ mod tests {
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
@@ -892,9 +886,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
provider2.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
@@ -76,7 +76,7 @@ where
|
||||
|
||||
// Check where storage history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ impl StorageHistory {
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
pruned_changesets += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
@@ -273,7 +273,7 @@ impl StorageHistory {
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
highest_deleted_storages.insert((address, entry.key.as_b256()), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
@@ -413,9 +413,7 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -520,7 +518,11 @@ mod tests {
|
||||
test_prune(1200, 3, (PruneProgress::Finished, 202));
|
||||
}
|
||||
|
||||
/// Tests the `prune_static_files` code path. On unix with rocksdb feature, v2 storage
|
||||
/// routes to `prune_rocksdb` instead, so this test only runs without rocksdb (the
|
||||
/// `prune_rocksdb_path` test covers that configuration).
|
||||
#[test]
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
fn prune_static_file() {
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
@@ -577,9 +579,7 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(true),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
limiter.increment_deleted_entries_count_by(result.pruned);
|
||||
|
||||
@@ -739,9 +739,7 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
@@ -793,9 +791,7 @@ mod tests {
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
provider2.set_storage_settings_cache(StorageSettings::v1());
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
@@ -895,11 +891,7 @@ mod tests {
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_storage_changesets_in_static_files(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
provider.set_storage_settings_cache(StorageSettings::v2());
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
|
||||
@@ -96,7 +96,7 @@ where
|
||||
|
||||
// Check where transaction hash numbers are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return self.prune_rocksdb(provider, input, start, end);
|
||||
}
|
||||
|
||||
@@ -491,9 +491,7 @@ mod tests {
|
||||
let segment = TransactionLookup::new(prune_mode);
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
@@ -578,9 +576,7 @@ mod tests {
|
||||
}
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let to_block: BlockNumber = 6;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
|
||||
@@ -160,6 +160,14 @@ impl StateProvider for StateProviderTest {
|
||||
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
Ok(self.accounts.get(&account).and_then(|(storage, _)| storage.get(&storage_key).copied()))
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
_address: Address,
|
||||
_hashed_storage_key: StorageKey,
|
||||
) -> ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl BytecodeReader for StateProviderTest {
|
||||
|
||||
8
crates/rpc/rpc-eth-types/src/cache/db.rs
vendored
8
crates/rpc/rpc-eth-types/src/cache/db.rs
vendored
@@ -154,6 +154,14 @@ impl StateProvider for StateProviderTraitObjWrapper {
|
||||
self.0.storage(account, storage_key)
|
||||
}
|
||||
|
||||
fn storage_by_hashed_key(
|
||||
&self,
|
||||
address: Address,
|
||||
hashed_storage_key: alloy_primitives::StorageKey,
|
||||
) -> reth_errors::ProviderResult<Option<alloy_primitives::StorageValue>> {
|
||||
self.0.storage_by_hashed_key(address, hashed_storage_key)
|
||||
}
|
||||
|
||||
fn account_code(
|
||||
&self,
|
||||
addr: &Address,
|
||||
|
||||
@@ -22,6 +22,7 @@ use reth_stages_api::{
|
||||
UnwindInput, UnwindOutput,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_trie::KeccakKeyHasher;
|
||||
use std::{
|
||||
cmp::{max, Ordering},
|
||||
collections::BTreeMap,
|
||||
@@ -461,9 +462,16 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// write output
|
||||
// Write output. When `use_hashed_state` is enabled, `write_state` skips writing to
|
||||
// plain account/storage tables and only writes bytecodes and changesets. The hashed
|
||||
// state is then written separately below.
|
||||
provider.write_state(&state, OriginalValuesKnown::Yes, StateWriteConfig::default())?;
|
||||
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
let hashed_state = state.hash_state_slow::<KeccakKeyHasher>();
|
||||
provider.write_hashed_state(&hashed_state.into_sorted())?;
|
||||
}
|
||||
|
||||
let db_write_duration = time.elapsed();
|
||||
debug!(
|
||||
target: "sync::stages::execution",
|
||||
@@ -1261,8 +1269,7 @@ mod tests {
|
||||
// but no receipt data is written.
|
||||
|
||||
let factory = create_test_provider_factory();
|
||||
factory
|
||||
.set_storage_settings_cache(StorageSettings::v1().with_receipts_in_static_files(true));
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
// Setup with block 1
|
||||
let provider_rw = factory.database_provider_rw().unwrap();
|
||||
|
||||
@@ -9,7 +9,9 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_etl::Collector;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_provider::{AccountExtReader, DBProvider, HashingWriter, StatsReader};
|
||||
use reth_provider::{
|
||||
AccountExtReader, DBProvider, HashingWriter, StatsReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_stages_api::{
|
||||
AccountHashingCheckpoint, EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint,
|
||||
StageError, StageId, UnwindInput, UnwindOutput,
|
||||
@@ -134,7 +136,11 @@ impl Default for AccountHashingStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for AccountHashingStage
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut> + HashingWriter + AccountExtReader + StatsReader,
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ HashingWriter
|
||||
+ AccountExtReader
|
||||
+ StatsReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -142,11 +148,21 @@ where
|
||||
}
|
||||
|
||||
/// Execute the stage.
|
||||
///
|
||||
/// When `use_hashed_state` is enabled, this stage is a no-op because the execution stage
|
||||
/// writes directly to `HashedAccounts`. Otherwise, it hashes plain state to populate hashed
|
||||
/// tables.
|
||||
fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result<ExecOutput, StageError> {
|
||||
if input.target_reached() {
|
||||
return Ok(ExecOutput::done(input.checkpoint()))
|
||||
}
|
||||
|
||||
// If using hashed state as canonical, execution already writes to `HashedAccounts`,
|
||||
// so this stage becomes a no-op.
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
|
||||
}
|
||||
|
||||
let (from_block, to_block) = input.next_block_range().into_inner();
|
||||
|
||||
// if there are more blocks then threshold it is faster to go over Plain state and hash all
|
||||
@@ -234,10 +250,14 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
|
||||
// directly to `HashedAccounts`, but the unwind must still revert those
|
||||
// entries here because `MerkleUnwind` runs after this stage (in unwind
|
||||
// order) and needs `HashedAccounts` to reflect the target block state
|
||||
// before it can verify the state root.
|
||||
let (range, unwind_progress, _) =
|
||||
input.unwind_block_range_with_threshold(self.commit_threshold);
|
||||
|
||||
// Aggregate all transition changesets and make a list of accounts that have been changed.
|
||||
provider.unwind_account_hashing_range(range)?;
|
||||
|
||||
let mut stage_checkpoint =
|
||||
|
||||
@@ -15,6 +15,7 @@ use reth_stages_api::{
|
||||
EntitiesCheckpoint, ExecInput, ExecOutput, Stage, StageCheckpoint, StageError, StageId,
|
||||
StorageHashingCheckpoint, UnwindInput, UnwindOutput,
|
||||
};
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
@@ -68,7 +69,11 @@ impl Default for StorageHashingStage {
|
||||
|
||||
impl<Provider> Stage<Provider> for StorageHashingStage
|
||||
where
|
||||
Provider: DBProvider<Tx: DbTxMut> + StorageReader + HashingWriter + StatsReader,
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StorageReader
|
||||
+ HashingWriter
|
||||
+ StatsReader
|
||||
+ StorageSettingsCache,
|
||||
{
|
||||
/// Return the id of the stage
|
||||
fn id(&self) -> StageId {
|
||||
@@ -82,6 +87,12 @@ where
|
||||
return Ok(ExecOutput::done(input.checkpoint()))
|
||||
}
|
||||
|
||||
// If use_hashed_state is enabled, execution writes directly to `HashedStorages`,
|
||||
// so this stage becomes a no-op.
|
||||
if provider.cached_storage_settings().use_hashed_state() {
|
||||
return Ok(ExecOutput::done(input.checkpoint().with_block_number(input.target())));
|
||||
}
|
||||
|
||||
let (from_block, to_block) = input.next_block_range().into_inner();
|
||||
|
||||
// if there are more blocks then threshold it is faster to go over Plain state and hash all
|
||||
@@ -176,6 +187,11 @@ where
|
||||
provider: &Provider,
|
||||
input: UnwindInput,
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
// NOTE: this runs in both v1 and v2 mode. In v2 mode, execution writes
|
||||
// directly to `HashedStorages`, but the unwind must still revert those
|
||||
// entries here because `MerkleUnwind` runs after this stage (in unwind
|
||||
// order) and needs `HashedStorages` to reflect the target block state
|
||||
// before it can verify the state root.
|
||||
let (range, unwind_progress, _) =
|
||||
input.unwind_block_range_with_threshold(self.commit_threshold);
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().account_history_in_rocksdb;
|
||||
let use_rocksdb = provider.cached_storage_settings().storage_v2;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
@@ -122,7 +122,7 @@ where
|
||||
|
||||
info!(target: "sync::stages::index_account_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
|
||||
let collector = if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
let collector = if provider.cached_storage_settings().storage_v2 {
|
||||
// Use the provider-based collection that can read from static files.
|
||||
collect_account_history_indices(provider, range.clone(), &self.etl_config)?
|
||||
} else {
|
||||
@@ -666,32 +666,43 @@ mod tests {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_provider::{
|
||||
providers::StaticFileWriter, RocksDBProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Sets up v2 account test data: writes block body indices to MDBX and
|
||||
/// account changesets to static files (matching realistic v2 layout).
|
||||
fn setup_v2_account_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in block_range.clone() {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let mut writer =
|
||||
static_file_provider.latest_writer(StaticFileSegment::AccountChangeSets).unwrap();
|
||||
for block in block_range {
|
||||
writer.append_account_changeset(vec![acc()], block).unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
/// Test that when `account_history_in_rocksdb` is enabled, the stage
|
||||
/// writes account history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
// init
|
||||
let db = TestStageDB::default();
|
||||
|
||||
// Enable RocksDB for account history
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -721,22 +732,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -772,22 +768,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_account_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexAccountHistoryStage::default();
|
||||
@@ -802,18 +783,6 @@ mod tests {
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::AccountChangeSets>(block, acc())?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
|
||||
@@ -107,7 +107,7 @@ where
|
||||
|
||||
let mut range = input.next_block_range();
|
||||
let first_sync = input.checkpoint().block_number == 0;
|
||||
let use_rocksdb = provider.cached_storage_settings().storages_history_in_rocksdb;
|
||||
let use_rocksdb = provider.cached_storage_settings().storage_v2;
|
||||
|
||||
// On first sync we might have history coming from genesis. We clear the table since it's
|
||||
// faster to rebuild from scratch.
|
||||
@@ -125,7 +125,7 @@ where
|
||||
}
|
||||
|
||||
info!(target: "sync::stages::index_storage_history::exec", ?first_sync, ?use_rocksdb, "Collecting indices");
|
||||
let collector = if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
let collector = if provider.cached_storage_settings().storage_v2 {
|
||||
collect_storage_history_indices(provider, range.clone(), &self.etl_config)?
|
||||
} else {
|
||||
collect_history_indices::<_, tables::StorageChangeSets, tables::StoragesHistory, _>(
|
||||
@@ -694,33 +694,51 @@ mod tests {
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
mod rocksdb_tests {
|
||||
use super::*;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_db_api::models::StorageBeforeTx;
|
||||
use reth_provider::{providers::StaticFileWriter, RocksDBProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
/// Sets up v2 storage test data: writes block body indices to MDBX and
|
||||
/// storage changesets to static files (matching realistic v2 layout).
|
||||
fn setup_v2_storage_data(db: &TestStageDB, block_range: std::ops::RangeInclusive<u64>) {
|
||||
db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in block_range.clone() {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let mut writer =
|
||||
static_file_provider.latest_writer(StaticFileSegment::StorageChangeSets).unwrap();
|
||||
for block in block_range {
|
||||
writer
|
||||
.append_storage_changeset(
|
||||
vec![StorageBeforeTx {
|
||||
address: ADDRESS,
|
||||
key: STORAGE_KEY,
|
||||
value: U256::ZERO,
|
||||
}],
|
||||
block,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
/// Test that when `storages_history_in_rocksdb` is enabled, the stage
|
||||
/// writes storage history indices to `RocksDB` instead of MDBX.
|
||||
#[tokio::test]
|
||||
async fn execute_writes_to_rocksdb_when_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -748,25 +766,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_works_when_rocksdb_enabled() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(10), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -803,25 +803,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn unwind_to_zero_keeps_block_zero() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_storage_data(&db, 0..=5);
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -852,25 +834,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn execute_incremental_sync() {
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..=5 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_storage_data(&db, 0..=10);
|
||||
|
||||
let input = ExecInput { target: Some(5), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
@@ -885,21 +849,6 @@ mod tests {
|
||||
let blocks: Vec<u64> = result.unwrap().iter().collect();
|
||||
assert_eq!(blocks, (0..=5).collect::<Vec<_>>());
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 6..=10 {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let input = ExecInput { target: Some(10), checkpoint: Some(StageCheckpoint::new(5)) };
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let out = stage.execute(&provider, input).unwrap();
|
||||
@@ -919,27 +868,8 @@ mod tests {
|
||||
use reth_db_api::models::sharded_key::NUM_OF_INDICES_IN_SHARD;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let num_blocks = (NUM_OF_INDICES_IN_SHARD * 2 + 100) as u64;
|
||||
|
||||
db.commit(|tx| {
|
||||
for block in 0..num_blocks {
|
||||
tx.put::<tables::BlockBodyIndices>(
|
||||
block,
|
||||
StoredBlockBodyIndices { tx_count: 3, ..Default::default() },
|
||||
)?;
|
||||
tx.put::<tables::StorageChangeSets>(
|
||||
block_number_address(block),
|
||||
storage(STORAGE_KEY),
|
||||
)?;
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
setup_v2_storage_data(&db, 0..=num_blocks - 1);
|
||||
|
||||
let input = ExecInput { target: Some(num_blocks - 1), ..Default::default() };
|
||||
let mut stage = IndexStorageHistoryStage::default();
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::{
|
||||
use reth_primitives_traits::{GotExpected, SealedHeader};
|
||||
use reth_provider::{
|
||||
ChangeSetReader, DBProvider, HeaderProvider, ProviderError, StageCheckpointReader,
|
||||
StageCheckpointWriter, StatsReader, StorageChangeSetReader, TrieWriter,
|
||||
StageCheckpointWriter, StatsReader, StorageChangeSetReader, StorageSettingsCache, TrieWriter,
|
||||
};
|
||||
use reth_stages_api::{
|
||||
BlockErrorKind, EntitiesCheckpoint, ExecInput, ExecOutput, MerkleCheckpoint, Stage,
|
||||
@@ -160,6 +160,7 @@ where
|
||||
+ HeaderProvider
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache
|
||||
+ StageCheckpointReader
|
||||
+ StageCheckpointWriter,
|
||||
{
|
||||
|
||||
@@ -540,9 +540,7 @@ mod tests {
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let runner = SenderRecoveryTestRunner::default();
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_senders_in_static_files(true),
|
||||
);
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
let input = ExecInput {
|
||||
target: Some(target),
|
||||
checkpoint: Some(StageCheckpoint::new(stage_progress)),
|
||||
|
||||
@@ -200,7 +200,7 @@ where
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
provider.commit_pending_rocksdb_batches()?;
|
||||
provider.rocksdb_provider().flush(&[Tables::TransactionHashNumbers.name()])?;
|
||||
}
|
||||
@@ -618,9 +618,7 @@ mod tests {
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let input = ExecInput {
|
||||
target: Some(previous_stage),
|
||||
@@ -686,9 +684,7 @@ mod tests {
|
||||
let runner = TransactionLookupTestRunner::default();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
runner.db.factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
// Insert blocks with transactions
|
||||
let blocks = random_block_range(
|
||||
|
||||
@@ -208,7 +208,10 @@ where
|
||||
|
||||
for (idx, changeset_result) in walker.enumerate() {
|
||||
let (BlockNumberAddress((block_number, address)), storage) = changeset_result?;
|
||||
cache.entry(AddressStorageKey((address, storage.key))).or_default().push(block_number);
|
||||
cache
|
||||
.entry(AddressStorageKey((address, storage.key.as_b256())))
|
||||
.or_default()
|
||||
.push(block_number);
|
||||
|
||||
if idx > 0 && idx % interval == 0 && total_changesets > 1000 {
|
||||
info!(target: "sync::stages::index_history", progress = %format!("{:.4}%", (idx as f64 / total_changesets as f64) * 100.0), "Collecting indices");
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use alloy_consensus::{constants::ETH_TO_WEI, Header, TxEip1559, TxReceipt};
|
||||
use alloy_eips::eip1559::INITIAL_BASE_FEE;
|
||||
use alloy_genesis::{Genesis, GenesisAccount};
|
||||
use alloy_primitives::{bytes, Address, Bytes, TxKind, B256, U256};
|
||||
use alloy_primitives::{bytes, keccak256, Address, Bytes, TxKind, B256, U256};
|
||||
use reth_chainspec::{ChainSpecBuilder, ChainSpecProvider, MAINNET};
|
||||
use reth_config::config::StageConfig;
|
||||
use reth_consensus::noop::NoopConsensus;
|
||||
@@ -36,7 +36,7 @@ use reth_stages::sets::DefaultStages;
|
||||
use reth_stages_api::{Pipeline, StageId};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use reth_storage_api::{
|
||||
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettingsCache,
|
||||
ChangeSetReader, StateProvider, StorageChangeSetReader, StorageSettings, StorageSettingsCache,
|
||||
};
|
||||
use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair};
|
||||
use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
|
||||
@@ -79,7 +79,7 @@ fn assert_changesets_queryable(
|
||||
let settings = provider.cached_storage_settings();
|
||||
|
||||
// Verify storage changesets
|
||||
if settings.storage_changesets_in_static_files {
|
||||
if settings.storage_v2 {
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
static_file_provider.initialize_index()?;
|
||||
let storage_changesets =
|
||||
@@ -89,6 +89,11 @@ fn assert_changesets_queryable(
|
||||
"storage changesets should be queryable from static files for blocks {:?}",
|
||||
block_range
|
||||
);
|
||||
|
||||
// Verify keys are in hashed format (v2 mode)
|
||||
for (_, entry) in &storage_changesets {
|
||||
assert!(entry.key.is_hashed(), "v2: storage changeset keys should be tagged as hashed");
|
||||
}
|
||||
} else {
|
||||
let storage_changesets: Vec<_> = provider
|
||||
.tx_ref()
|
||||
@@ -100,10 +105,20 @@ fn assert_changesets_queryable(
|
||||
"storage changesets should be queryable from MDBX for blocks {:?}",
|
||||
block_range
|
||||
);
|
||||
|
||||
// Verify keys are plain (not hashed) in v1 mode
|
||||
for (_, entry) in &storage_changesets {
|
||||
let key = entry.key;
|
||||
assert_ne!(
|
||||
key,
|
||||
keccak256(key),
|
||||
"v1: storage changeset key should be plain (not its own keccak256)"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify account changesets
|
||||
if settings.account_changesets_in_static_files {
|
||||
if settings.storage_v2 {
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
static_file_provider.initialize_index()?;
|
||||
let account_changesets =
|
||||
@@ -201,19 +216,22 @@ where
|
||||
pipeline
|
||||
}
|
||||
|
||||
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
|
||||
/// Shared helper for pipeline forward sync and unwind tests.
|
||||
///
|
||||
/// This test:
|
||||
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
|
||||
/// 2. Each block contains two transactions:
|
||||
/// - ETH transfer to a recipient (account state changes)
|
||||
/// - Counter `increment()` call (storage state changes)
|
||||
/// 3. Runs the full pipeline with ALL stages enabled
|
||||
/// 4. Forward syncs to block 5, unwinds to block 2
|
||||
/// 4. Forward syncs to `num_blocks`, unwinds to `unwind_target`, then re-syncs back to `num_blocks`
|
||||
///
|
||||
/// This exercises both account and storage hashing/history stages.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline() -> eyre::Result<()> {
|
||||
/// When `storage_settings` is `Some`, the pipeline provider factory is configured with the given
|
||||
/// settings before genesis initialization (e.g. v2 storage mode).
|
||||
async fn run_pipeline_forward_and_unwind(
|
||||
storage_settings: Option<StorageSettings>,
|
||||
num_blocks: u64,
|
||||
unwind_target: u64,
|
||||
) -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Generate a keypair for signing transactions
|
||||
@@ -259,7 +277,6 @@ async fn test_pipeline() -> eyre::Result<()> {
|
||||
let evm_config = EthEvmConfig::new(chain_spec.clone());
|
||||
|
||||
// Build blocks by actually executing transactions to get correct state roots
|
||||
let num_blocks = 5u64;
|
||||
let mut blocks: Vec<SealedBlock<Block>> = Vec::new();
|
||||
let mut parent_hash = genesis.hash();
|
||||
|
||||
@@ -384,11 +401,15 @@ async fn test_pipeline() -> eyre::Result<()> {
|
||||
// This is needed because we wrote state during block generation for computing state roots
|
||||
let pipeline_provider_factory =
|
||||
create_test_provider_factory_with_chain_spec(chain_spec.clone());
|
||||
if let Some(settings) = storage_settings {
|
||||
pipeline_provider_factory.set_storage_settings_cache(settings);
|
||||
}
|
||||
init_genesis(&pipeline_provider_factory).expect("init genesis");
|
||||
let pipeline_genesis =
|
||||
pipeline_provider_factory.sealed_header(0)?.expect("genesis should exist");
|
||||
let pipeline_consensus = NoopConsensus::arc();
|
||||
|
||||
let blocks_clone = blocks.clone();
|
||||
let file_client = create_file_client_from_blocks(blocks);
|
||||
let max_block = file_client.max_block().unwrap();
|
||||
let tip = file_client.tip().expect("tip");
|
||||
@@ -417,7 +438,7 @@ async fn test_pipeline() -> eyre::Result<()> {
|
||||
{
|
||||
let provider = pipeline_provider_factory.provider()?;
|
||||
let last_block = provider.last_block_number()?;
|
||||
assert_eq!(last_block, 5, "should have synced 5 blocks");
|
||||
assert_eq!(last_block, num_blocks, "should have synced {num_blocks} blocks");
|
||||
|
||||
for stage_id in [
|
||||
StageId::Headers,
|
||||
@@ -435,29 +456,28 @@ async fn test_pipeline() -> eyre::Result<()> {
|
||||
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
|
||||
assert_eq!(
|
||||
checkpoint.map(|c| c.block_number),
|
||||
Some(5),
|
||||
"{stage_id} checkpoint should be at block 5"
|
||||
Some(num_blocks),
|
||||
"{stage_id} checkpoint should be at block {num_blocks}"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify the counter contract's storage was updated
|
||||
// After 5 blocks with 1 increment each, slot 0 should be 5
|
||||
// After num_blocks blocks with 1 increment each, slot 0 should be num_blocks
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(5)),
|
||||
"Counter storage slot 0 should be 5 after 5 increments"
|
||||
Some(U256::from(num_blocks)),
|
||||
"Counter storage slot 0 should be {num_blocks} after {num_blocks} increments"
|
||||
);
|
||||
}
|
||||
|
||||
// Verify changesets are queryable before unwind
|
||||
// This validates that the #21561 fix works - unwind needs to read changesets from the correct
|
||||
// source
|
||||
assert_changesets_queryable(&pipeline_provider_factory, 1..=5)?;
|
||||
assert_changesets_queryable(&pipeline_provider_factory, 1..=num_blocks)?;
|
||||
|
||||
// Unwind to block 2
|
||||
let unwind_target = 2u64;
|
||||
// Unwind to unwind_target
|
||||
pipeline.unwind(unwind_target, None)?;
|
||||
|
||||
// Verify unwind
|
||||
@@ -484,7 +504,114 @@ async fn test_pipeline() -> eyre::Result<()> {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(unwind_target)),
|
||||
"Counter storage slot 0 should be {unwind_target} after unwinding to block {unwind_target}"
|
||||
);
|
||||
}
|
||||
|
||||
// Re-sync: build a new pipeline starting from unwind_target and sync back to num_blocks
|
||||
let resync_file_client = create_file_client_from_blocks(blocks_clone);
|
||||
let resync_consensus = NoopConsensus::arc();
|
||||
let resync_stages_config = StageConfig::default();
|
||||
|
||||
let unwind_head = pipeline_provider_factory
|
||||
.sealed_header(unwind_target)?
|
||||
.expect("unwind target header should exist");
|
||||
|
||||
let mut resync_header_downloader =
|
||||
ReverseHeadersDownloaderBuilder::new(resync_stages_config.headers)
|
||||
.build(resync_file_client.clone(), resync_consensus.clone())
|
||||
.into_task();
|
||||
resync_header_downloader.update_local_head(unwind_head);
|
||||
resync_header_downloader.update_sync_target(SyncTarget::Tip(tip));
|
||||
|
||||
let mut resync_body_downloader = BodiesDownloaderBuilder::new(resync_stages_config.bodies)
|
||||
.build(resync_file_client, resync_consensus, pipeline_provider_factory.clone())
|
||||
.into_task();
|
||||
resync_body_downloader
|
||||
.set_download_range(unwind_target + 1..=max_block)
|
||||
.expect("set download range");
|
||||
|
||||
let resync_pipeline = build_pipeline(
|
||||
pipeline_provider_factory.clone(),
|
||||
resync_header_downloader,
|
||||
resync_body_downloader,
|
||||
max_block,
|
||||
tip,
|
||||
);
|
||||
|
||||
let (_resync_pipeline, resync_result) = resync_pipeline.run_as_fut(None).await;
|
||||
resync_result?;
|
||||
|
||||
// Verify re-sync
|
||||
{
|
||||
let provider = pipeline_provider_factory.provider()?;
|
||||
let last_block = provider.last_block_number()?;
|
||||
assert_eq!(last_block, num_blocks, "should have re-synced to {num_blocks} blocks");
|
||||
|
||||
for stage_id in [
|
||||
StageId::Headers,
|
||||
StageId::Bodies,
|
||||
StageId::SenderRecovery,
|
||||
StageId::Execution,
|
||||
StageId::AccountHashing,
|
||||
StageId::StorageHashing,
|
||||
StageId::MerkleExecute,
|
||||
StageId::TransactionLookup,
|
||||
StageId::IndexAccountHistory,
|
||||
StageId::IndexStorageHistory,
|
||||
StageId::Finish,
|
||||
] {
|
||||
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
|
||||
assert_eq!(
|
||||
checkpoint.map(|c| c.block_number),
|
||||
Some(num_blocks),
|
||||
"{stage_id} checkpoint should be at block {num_blocks} after re-sync"
|
||||
);
|
||||
}
|
||||
|
||||
let state = provider.latest();
|
||||
let counter_storage = state.storage(CONTRACT_ADDRESS, B256::ZERO)?;
|
||||
assert_eq!(
|
||||
counter_storage,
|
||||
Some(U256::from(num_blocks)),
|
||||
"Counter storage slot 0 should be {num_blocks} after re-sync"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Tests pipeline with ALL stages enabled using both ETH transfers and contract storage changes.
|
||||
///
|
||||
/// This test:
|
||||
/// 1. Pre-funds a signer account and deploys a Counter contract in genesis
|
||||
/// 2. Each block contains two transactions:
|
||||
/// - ETH transfer to a recipient (account state changes)
|
||||
/// - Counter `increment()` call (storage state changes)
|
||||
/// 3. Runs the full pipeline with ALL stages enabled
|
||||
/// 4. Forward syncs to block 5, unwinds to block 2, then re-syncs to block 5
|
||||
///
|
||||
/// This exercises both account and storage hashing/history stages.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline() -> eyre::Result<()> {
|
||||
run_pipeline_forward_and_unwind(None, 5, 2).await
|
||||
}
|
||||
|
||||
/// Same as [`test_pipeline`] but runs with v2 storage settings (`use_hashed_state=true`,
|
||||
/// `is_v2()=true`, etc.).
|
||||
///
|
||||
/// In v2 mode:
|
||||
/// - The execution stage writes directly to `HashedAccounts`/`HashedStorages`
|
||||
/// - `AccountHashingStage` and `StorageHashingStage` are no-ops during forward execution
|
||||
/// - Changesets are stored in static files with pre-hashed storage keys
|
||||
/// - Unwind must still revert hashed state via the hashing stages before `MerkleUnwind` validates
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_pipeline_v2() -> eyre::Result<()> {
|
||||
run_pipeline_forward_and_unwind(Some(StorageSettings::v2()), 5, 2).await
|
||||
}
|
||||
|
||||
@@ -5,208 +5,101 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Storage configuration settings for this node.
|
||||
///
|
||||
/// Controls whether this node uses v2 storage layout (static files + `RocksDB` routing)
|
||||
/// or v1/legacy layout (everything in MDBX).
|
||||
///
|
||||
/// These should be set during `init_genesis` or `init_db` depending on whether we want dictate
|
||||
/// behaviour of new or old nodes respectively.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize, Compact)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Compact, Serialize, Deserialize)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(compact)]
|
||||
pub struct StorageSettings {
|
||||
/// Whether this node always writes receipts to static files.
|
||||
/// Whether this node uses v2 storage layout.
|
||||
///
|
||||
/// If this is set to FALSE AND receipt pruning IS ENABLED, all receipts should be written to DB. Otherwise, they should be written to static files. This ensures that older nodes do not need to migrate their current DB tables to static files. For more, read: <https://github.com/paradigmxyz/reth/issues/18890#issuecomment-3457760097>
|
||||
#[serde(default)]
|
||||
pub receipts_in_static_files: bool,
|
||||
/// Whether this node always writes transaction senders to static files.
|
||||
#[serde(default)]
|
||||
pub transaction_senders_in_static_files: bool,
|
||||
/// Whether `StoragesHistory` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub storages_history_in_rocksdb: bool,
|
||||
/// Whether `TransactionHashNumbers` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub transaction_hash_numbers_in_rocksdb: bool,
|
||||
/// Whether `AccountsHistory` is stored in `RocksDB`.
|
||||
#[serde(default)]
|
||||
pub account_history_in_rocksdb: bool,
|
||||
/// Whether this node should read and write account changesets from static files.
|
||||
#[serde(default)]
|
||||
pub account_changesets_in_static_files: bool,
|
||||
/// Whether this node should read and write storage changesets from static files.
|
||||
#[serde(default)]
|
||||
pub storage_changesets_in_static_files: bool,
|
||||
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
|
||||
/// state representation instead of plain state tables.
|
||||
#[serde(default)]
|
||||
pub use_hashed_state: bool,
|
||||
/// When `true`, enables all v2 storage features:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (accounts, storages, transaction hashes)
|
||||
/// - Account and storage changesets in static files
|
||||
/// - Hashed state tables as canonical state representation
|
||||
///
|
||||
/// When `false`, uses v1/legacy layout (everything in MDBX).
|
||||
pub storage_v2: bool,
|
||||
}
|
||||
|
||||
impl StorageSettings {
|
||||
/// Returns the default base `StorageSettings`.
|
||||
///
|
||||
/// Always returns [`Self::v1()`]. Use the `--storage.v2` CLI flag to opt into
|
||||
/// [`Self::v2()`] at runtime. The `rocksdb` feature only makes the v2 backend
|
||||
/// *available*; it does not activate it by default.
|
||||
/// When the `edge` feature is enabled, returns [`Self::v2()`] so that CI and
|
||||
/// edge builds automatically use v2 storage defaults. Otherwise returns
|
||||
/// [`Self::v1()`]. The `--storage.v2` CLI flag can also opt into v2 at runtime
|
||||
/// regardless of feature flags.
|
||||
pub const fn base() -> Self {
|
||||
Self::v1()
|
||||
#[cfg(feature = "edge")]
|
||||
{
|
||||
Self::v2()
|
||||
}
|
||||
#[cfg(not(feature = "edge"))]
|
||||
{
|
||||
Self::v1()
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for v2 nodes with all storage features enabled:
|
||||
/// - Receipts and transaction senders in static files
|
||||
/// - History indices in `RocksDB` (storages, accounts, transaction hashes)
|
||||
/// - Account and storage changesets in static files
|
||||
/// - Hashed state as canonical state representation
|
||||
///
|
||||
/// Use this when the `--storage.v2` CLI flag is set.
|
||||
pub const fn v2() -> Self {
|
||||
Self {
|
||||
receipts_in_static_files: true,
|
||||
transaction_senders_in_static_files: true,
|
||||
account_changesets_in_static_files: true,
|
||||
storage_changesets_in_static_files: true,
|
||||
storages_history_in_rocksdb: true,
|
||||
transaction_hash_numbers_in_rocksdb: true,
|
||||
account_history_in_rocksdb: true,
|
||||
use_hashed_state: false,
|
||||
}
|
||||
Self { storage_v2: true }
|
||||
}
|
||||
|
||||
/// Creates `StorageSettings` for v1/legacy nodes.
|
||||
///
|
||||
/// This explicitly sets `receipts_in_static_files` and `transaction_senders_in_static_files` to
|
||||
/// `false`, ensuring older nodes continue writing receipts and transaction senders to the
|
||||
/// database when receipt pruning is enabled.
|
||||
/// This keeps all data in MDBX, matching the original storage layout.
|
||||
pub const fn v1() -> Self {
|
||||
Self {
|
||||
receipts_in_static_files: false,
|
||||
transaction_senders_in_static_files: false,
|
||||
storages_history_in_rocksdb: false,
|
||||
transaction_hash_numbers_in_rocksdb: false,
|
||||
account_history_in_rocksdb: false,
|
||||
account_changesets_in_static_files: false,
|
||||
storage_changesets_in_static_files: false,
|
||||
use_hashed_state: false,
|
||||
}
|
||||
Self { storage_v2: false }
|
||||
}
|
||||
|
||||
/// Sets the `receipts_in_static_files` flag to the provided value.
|
||||
pub const fn with_receipts_in_static_files(mut self, value: bool) -> Self {
|
||||
self.receipts_in_static_files = value;
|
||||
self
|
||||
/// Returns `true` if this node uses v2 storage layout.
|
||||
pub const fn is_v2(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `transaction_senders_in_static_files` flag to the provided value.
|
||||
pub const fn with_transaction_senders_in_static_files(mut self, value: bool) -> Self {
|
||||
self.transaction_senders_in_static_files = value;
|
||||
self
|
||||
/// Whether receipts are stored in static files.
|
||||
pub const fn receipts_in_static_files(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `storages_history_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_storages_history_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.storages_history_in_rocksdb = value;
|
||||
self
|
||||
/// Whether transaction senders are stored in static files.
|
||||
pub const fn transaction_senders_in_static_files(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `transaction_hash_numbers_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_transaction_hash_numbers_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.transaction_hash_numbers_in_rocksdb = value;
|
||||
self
|
||||
/// Whether storages history is stored in `RocksDB`.
|
||||
pub const fn storages_history_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `account_history_in_rocksdb` flag to the provided value.
|
||||
pub const fn with_account_history_in_rocksdb(mut self, value: bool) -> Self {
|
||||
self.account_history_in_rocksdb = value;
|
||||
self
|
||||
/// Whether transaction hash numbers are stored in `RocksDB`.
|
||||
pub const fn transaction_hash_numbers_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `account_changesets_in_static_files` flag to the provided value.
|
||||
pub const fn with_account_changesets_in_static_files(mut self, value: bool) -> Self {
|
||||
self.account_changesets_in_static_files = value;
|
||||
self
|
||||
/// Whether account history is stored in `RocksDB`.
|
||||
pub const fn account_history_in_rocksdb(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Sets the `storage_changesets_in_static_files` flag to the provided value.
|
||||
pub const fn with_storage_changesets_in_static_files(mut self, value: bool) -> Self {
|
||||
self.storage_changesets_in_static_files = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the `use_hashed_state` flag to the provided value.
|
||||
pub const fn with_use_hashed_state(mut self, value: bool) -> Self {
|
||||
self.use_hashed_state = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `receipts_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_receipts_in_static_files_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.receipts_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `transaction_senders_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_transaction_senders_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.transaction_senders_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `account_changesets_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_account_changesets_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.account_changesets_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `storage_changesets_in_static_files` if `value` is `Some`.
|
||||
pub const fn with_storage_changesets_in_static_files_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.storage_changesets_in_static_files = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `transaction_hash_numbers_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_transaction_hash_numbers_in_rocksdb_opt(
|
||||
mut self,
|
||||
value: Option<bool>,
|
||||
) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.transaction_hash_numbers_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `storages_history_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_storages_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.storages_history_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets `account_history_in_rocksdb` if `value` is `Some`.
|
||||
pub const fn with_account_history_in_rocksdb_opt(mut self, value: Option<bool>) -> Self {
|
||||
if let Some(v) = value {
|
||||
self.account_history_in_rocksdb = v;
|
||||
}
|
||||
self
|
||||
/// Whether to use hashed state tables (`HashedAccounts`/`HashedStorages`) as the canonical
|
||||
/// state representation instead of plain state tables. Implied by v2 storage layout.
|
||||
pub const fn use_hashed_state(&self) -> bool {
|
||||
self.storage_v2
|
||||
}
|
||||
|
||||
/// Returns `true` if any tables are configured to be stored in `RocksDB`.
|
||||
pub const fn any_in_rocksdb(&self) -> bool {
|
||||
self.transaction_hash_numbers_in_rocksdb ||
|
||||
self.account_history_in_rocksdb ||
|
||||
self.storages_history_in_rocksdb
|
||||
self.storage_v2
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,13 +214,13 @@ where
|
||||
// not the genesis block number. This would cause increment_block(N) to fail.
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
if genesis_block_number > 0 {
|
||||
if genesis_storage_settings.account_changesets_in_static_files {
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::AccountChangeSets)?
|
||||
.user_header_mut()
|
||||
.set_expected_block_start(genesis_block_number);
|
||||
}
|
||||
if genesis_storage_settings.storage_changesets_in_static_files {
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::StorageChangeSets)?
|
||||
.user_header_mut()
|
||||
@@ -259,7 +259,7 @@ where
|
||||
.user_header_mut()
|
||||
.set_block_range(genesis_block_number, genesis_block_number);
|
||||
|
||||
if genesis_storage_settings.transaction_senders_in_static_files {
|
||||
if genesis_storage_settings.storage_v2 {
|
||||
static_file_provider
|
||||
.get_writer(genesis_block_number, StaticFileSegment::TransactionSenders)?
|
||||
.user_header_mut()
|
||||
@@ -1052,7 +1052,7 @@ mod tests {
|
||||
)
|
||||
};
|
||||
|
||||
let (accounts, storages) = if settings.account_history_in_rocksdb {
|
||||
let (accounts, storages) = if settings.storage_v2 {
|
||||
collect_rocksdb(&rocksdb)
|
||||
} else {
|
||||
collect_from_mdbx(&factory)
|
||||
@@ -1075,10 +1075,7 @@ mod tests {
|
||||
init_genesis_with_settings(&factory, StorageSettings::v1()).unwrap();
|
||||
|
||||
// Request different settings - should warn but succeed
|
||||
let result = init_genesis_with_settings(
|
||||
&factory,
|
||||
StorageSettings::v1().with_receipts_in_static_files(true),
|
||||
);
|
||||
let result = init_genesis_with_settings(&factory, StorageSettings::v2());
|
||||
|
||||
// Should succeed (warning is logged, not an error)
|
||||
assert!(result.is_ok());
|
||||
@@ -1087,7 +1084,7 @@ mod tests {
|
||||
#[test]
|
||||
fn allow_same_storage_settings() {
|
||||
let factory = create_test_provider_factory_with_chain_spec(MAINNET.clone());
|
||||
let settings = StorageSettings::v1().with_receipts_in_static_files(true);
|
||||
let settings = StorageSettings::v2();
|
||||
init_genesis_with_settings(&factory, settings).unwrap();
|
||||
|
||||
let result = init_genesis_with_settings(&factory, settings);
|
||||
|
||||
@@ -27,6 +27,8 @@ alloy-primitives.workspace = true
|
||||
# mdbx
|
||||
reth-libmdbx = { workspace = true, optional = true, features = ["return-borrowed", "read-tx-timeouts"] }
|
||||
eyre = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true, features = ["full"] }
|
||||
|
||||
# metrics
|
||||
reth-metrics = { workspace = true, optional = true }
|
||||
@@ -97,6 +99,7 @@ op = [
|
||||
"reth-db-api/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-libmdbx/pageviz", "dep:reth-mdbx-viz", "dep:tokio", "mdbx"]
|
||||
disable-lock = []
|
||||
|
||||
[[bench]]
|
||||
|
||||
@@ -522,6 +522,81 @@ impl DatabaseEnv {
|
||||
self
|
||||
}
|
||||
|
||||
/// Starts the real-time page access visualization server.
|
||||
///
|
||||
/// This enables the C-level page access instrumentation hooks,
|
||||
/// starts a background drainer thread that coalesces events,
|
||||
/// and launches an HTTP+WebSocket server on the given port.
|
||||
///
|
||||
/// The server serves an interactive HTML visualization at `http://localhost:{port}/`
|
||||
/// showing page accesses in real-time with fading highlights:
|
||||
/// - Blue = read, Red = write, Yellow = free
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub fn start_pageviz(&self, port: u16, db_path: std::path::PathBuf) {
|
||||
use reth_libmdbx::pageviz::PageVizDrainer;
|
||||
|
||||
let max_dbi = self.dbis.values().copied().max().unwrap_or(0) as usize;
|
||||
let mut dbi_names = vec![String::new(); max_dbi + 1];
|
||||
dbi_names[0] = "FREE_DBI".to_string();
|
||||
if dbi_names.len() > 1 {
|
||||
dbi_names[1] = "MAIN_DBI".to_string();
|
||||
}
|
||||
for (name, &dbi) in self.dbis.iter() {
|
||||
dbi_names[dbi as usize] = name.to_string();
|
||||
}
|
||||
|
||||
let mut name_to_dbi = std::collections::HashMap::new();
|
||||
for (&name, &dbi) in self.dbis.iter() {
|
||||
name_to_dbi.insert(name, dbi as u8);
|
||||
}
|
||||
|
||||
let mdbx_dat = db_path.join("mdbx.dat");
|
||||
let walk = match reth_mdbx_viz::walker::walk_mdbx(&mdbx_dat, &name_to_dbi) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
reth_tracing::tracing::error!("pageviz: failed to walk B-tree: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let page_count = walk.page_count;
|
||||
let page_size = walk.page_size;
|
||||
|
||||
let mut drainer = PageVizDrainer::new();
|
||||
drainer.enable();
|
||||
let rx = drainer.start(60);
|
||||
|
||||
let config = reth_mdbx_viz::VizConfig {
|
||||
port,
|
||||
page_count: page_count as u64,
|
||||
page_size: page_size as u32,
|
||||
dbi_names,
|
||||
owner_map: walk.owner_map,
|
||||
tree_info: walk.tree_info,
|
||||
db_path: mdbx_dat,
|
||||
};
|
||||
|
||||
// Spawn the viz server on a background tokio runtime
|
||||
std::thread::Builder::new()
|
||||
.name("mdbx-viz-server".into())
|
||||
.spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("failed to build tokio runtime for pageviz");
|
||||
rt.block_on(async {
|
||||
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
|
||||
reth_tracing::tracing::error!("pageviz server error: {e}");
|
||||
}
|
||||
});
|
||||
// Keep drainer alive while server runs
|
||||
drop(drainer);
|
||||
})
|
||||
.expect("failed to spawn pageviz server thread");
|
||||
|
||||
reth_tracing::tracing::info!("pageviz server started on port {port}");
|
||||
}
|
||||
|
||||
/// Creates all the tables defined in [`Tables`], if necessary.
|
||||
///
|
||||
/// This keeps tracks of the created table handles and stores them for better efficiency.
|
||||
|
||||
@@ -45,11 +45,22 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
|
||||
path: P,
|
||||
args: DatabaseArguments,
|
||||
) -> eyre::Result<DatabaseEnv> {
|
||||
let db_path = path.as_ref().to_path_buf();
|
||||
let client_version = args.client_version().clone();
|
||||
let mut db = create_db(path, args)?;
|
||||
db.create_and_track_tables_for::<TS>()?;
|
||||
db.record_client_version(client_version)?;
|
||||
drop_orphan_tables(&db);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
{
|
||||
let port = std::env::var("RETH_PAGEVIZ_PORT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(3141u16);
|
||||
db.start_pageviz(port, db_path);
|
||||
}
|
||||
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ dashmap = { workspace = true, features = ["inline"], optional = true }
|
||||
default = []
|
||||
return-borrowed = []
|
||||
read-tx-timeouts = ["dep:dashmap"]
|
||||
pageviz = ["reth-mdbx-sys/pageviz"]
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
|
||||
@@ -11,3 +11,7 @@ repository.workspace = true
|
||||
[build-dependencies]
|
||||
cc.workspace = true
|
||||
bindgen = { workspace = true, features = ["runtime"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
pageviz = []
|
||||
|
||||
@@ -30,6 +30,11 @@ fn main() {
|
||||
#[cfg(not(debug_assertions))]
|
||||
cc.define("MDBX_DEBUG", "0").define("NDEBUG", None);
|
||||
|
||||
// Enable page visualization instrumentation
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.define("MDBX_PAGEVIZ", "1");
|
||||
}
|
||||
|
||||
// Propagate `-C target-cpu=native`
|
||||
let rustflags = env::var("CARGO_ENCODED_RUSTFLAGS").unwrap();
|
||||
if rustflags.contains("target-cpu=native") &&
|
||||
@@ -38,7 +43,11 @@ fn main() {
|
||||
cc.flag("-march=native");
|
||||
}
|
||||
|
||||
cc.file(mdbx.join("mdbx.c")).compile("libmdbx.a");
|
||||
cc.file(mdbx.join("mdbx.c"));
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.file(mdbx.join("mdbx_pageviz.c"));
|
||||
}
|
||||
cc.compile("libmdbx.a");
|
||||
}
|
||||
|
||||
fn generate_bindings(mdbx: &Path, out_file: &Path) {
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include MDBX_CONFIG_H
|
||||
#endif
|
||||
|
||||
|
||||
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
|
||||
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
|
||||
#undef NDEBUG
|
||||
@@ -46,6 +47,7 @@
|
||||
#define _DARWIN_C_SOURCE
|
||||
#endif /* _DARWIN_C_SOURCE */
|
||||
|
||||
#include "mdbx_pageviz.h"
|
||||
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
|
||||
#define __USE_MINGW_ANSI_STDIO 1
|
||||
#endif /* MinGW */
|
||||
@@ -6193,6 +6195,7 @@ MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const p
|
||||
static inline int page_touch(MDBX_cursor *mc) {
|
||||
page_t *const mp = mc->pg[mc->top];
|
||||
MDBX_txn *txn = mc->txn;
|
||||
MDBX_PAGEVIZ_WRITE(mc, mp->pgno);
|
||||
|
||||
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
|
||||
@@ -19926,6 +19929,9 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t
|
||||
eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
|
||||
|
||||
if (rc == MDBX_SUCCESS) {
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
eASSERT(env, limit_bytes == env->dxb_mmap.limit);
|
||||
eASSERT(env, size_bytes <= env->dxb_mmap.filesize);
|
||||
if (mode == explicit_resize)
|
||||
@@ -20339,6 +20345,10 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bit
|
||||
if (unlikely(err != MDBX_SUCCESS))
|
||||
return err;
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
|
||||
#if defined(MADV_DONTDUMP)
|
||||
err =
|
||||
madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
|
||||
@@ -23090,6 +23100,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
return err;
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -23104,6 +23115,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
tASSERT(txn, lp->flags == P_LOOSE);
|
||||
loose[++count] = lp->pgno;
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -31626,6 +31638,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
|
||||
if (unlikely(r.err != MDBX_SUCCESS))
|
||||
goto bailout;
|
||||
#endif /* MDBX_DISABLE_VALIDATION */
|
||||
MDBX_PAGEVIZ_READ(mc, pgno);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -31842,6 +31855,7 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
|
||||
return ret;
|
||||
|
||||
DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = (uint16_t)flags;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -31870,6 +31884,7 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
|
||||
return ret;
|
||||
|
||||
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = P_LARGE;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -32015,6 +32030,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
|
||||
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
|
||||
tASSERT(txn, mp->pgno != pgno);
|
||||
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
|
||||
MDBX_PAGEVIZ_FREE(mc, mp->pgno);
|
||||
/* Update the parent page, if any, to point to the new page */
|
||||
if (likely(mc->top)) {
|
||||
page_t *parent = mc->pg[mc->top - 1];
|
||||
@@ -32211,6 +32227,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
|
||||
unsigned pageflags /* maybe unknown/zero */) {
|
||||
int rc;
|
||||
MDBX_txn *const txn = mc->txn;
|
||||
MDBX_PAGEVIZ_FREE(mc, pgno);
|
||||
tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));
|
||||
|
||||
/* During deleting entire subtrees, it is reasonable and possible to avoid
|
||||
|
||||
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.c
vendored
Normal file
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.c
vendored
Normal file
@@ -0,0 +1,119 @@
|
||||
#include "mdbx_pageviz.h"
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
|
||||
/* ── Global state ─────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_global = NULL;
|
||||
|
||||
/* ── Thread-local ring index ──────────────────────────────────────────── */
|
||||
|
||||
MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring = MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
|
||||
/* ── Lifecycle ────────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void) {
|
||||
mdbx_pageviz_state_t *state = calloc(1, sizeof(mdbx_pageviz_state_t));
|
||||
if (state)
|
||||
mdbx_pageviz_global = state;
|
||||
return state;
|
||||
}
|
||||
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state) {
|
||||
if (!state)
|
||||
return;
|
||||
if (mdbx_pageviz_global == state)
|
||||
mdbx_pageviz_global = NULL;
|
||||
free(state);
|
||||
}
|
||||
|
||||
/* ── Enable / Disable ─────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 1, memory_order_release);
|
||||
}
|
||||
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 0, memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Drain ────────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count) {
|
||||
if (!state || ring_idx >= MDBX_PAGEVIZ_MAX_RINGS || !out_buf || max_count == 0)
|
||||
return 0;
|
||||
|
||||
mdbx_pageviz_ring_t *ring = &state->rings[ring_idx];
|
||||
uint32_t head = atomic_load_explicit(&ring->published_head,
|
||||
memory_order_acquire);
|
||||
uint32_t tail = atomic_load_explicit(&ring->consumer_tail,
|
||||
memory_order_relaxed);
|
||||
|
||||
if (head == tail)
|
||||
return 0;
|
||||
|
||||
uint32_t avail = head - tail;
|
||||
uint32_t count = avail < max_count ? avail : max_count;
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
uint32_t slot = (tail + i) & MDBX_PAGEVIZ_RING_MASK;
|
||||
out_buf[i] = ring->events[slot];
|
||||
}
|
||||
|
||||
atomic_store_explicit(&ring->consumer_tail, tail + count,
|
||||
memory_order_release);
|
||||
return count;
|
||||
}
|
||||
|
||||
/* ── Queries ──────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state) {
|
||||
return atomic_load_explicit(&state->ring_count, memory_order_relaxed);
|
||||
}
|
||||
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx) {
|
||||
return atomic_load_explicit(&state->rings[ring_idx].dropped,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* ── Mapping info ────────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return;
|
||||
state->mapping.mdbx_page_size = mdbx_page_size;
|
||||
state->mapping.sys_page_size = (uint32_t)sysconf(_SC_PAGESIZE);
|
||||
state->mapping.len = len;
|
||||
/* base written last so consumers see consistent len+page_size first */
|
||||
__atomic_store_n((void *volatile *)&state->mapping.base, base, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return 0;
|
||||
void *b = __atomic_load_n((void *volatile *)&state->mapping.base, __ATOMIC_ACQUIRE);
|
||||
if (!b)
|
||||
return 0;
|
||||
*out_base = b;
|
||||
*out_len = state->mapping.len;
|
||||
*out_mdbx_ps = state->mapping.mdbx_page_size;
|
||||
*out_sys_ps = state->mapping.sys_page_size;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ── Block marker emit (non-inline wrapper for Rust FFI) ──────────────── */
|
||||
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded) {
|
||||
mdbx_pageviz_emit(op, ((uint32_t)gas_encoded << 24) | ((uint32_t)duration_encoded << 16) | (uint32_t)tx_count, block_number);
|
||||
}
|
||||
|
||||
#else /* !MDBX_PAGEVIZ */
|
||||
|
||||
typedef int mdbx_pageviz_empty_tu_;
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
195
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.h
vendored
Normal file
195
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.h
vendored
Normal file
@@ -0,0 +1,195 @@
|
||||
/* mdbx_pageviz.h — Standalone page-access visualization ring buffer for MDBX.
|
||||
*
|
||||
* Self-contained: no dependencies on MDBX internals.
|
||||
* Gate everything behind MDBX_PAGEVIZ; when not defined, all macros expand to
|
||||
* nothing (zero overhead).
|
||||
*
|
||||
* Compatible with C11 and C17. */
|
||||
|
||||
#ifndef MDBX_PAGEVIZ_H
|
||||
#define MDBX_PAGEVIZ_H
|
||||
|
||||
/* ── Disabled stub macros ─────────────────────────────────────────────── */
|
||||
|
||||
#if !defined(MDBX_PAGEVIZ) || !(MDBX_PAGEVIZ)
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) ((void)0)
|
||||
|
||||
#else /* MDBX_PAGEVIZ enabled ───────────────────────────────────────────── */
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* ── Tunables ─────────────────────────────────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_CAPACITY 65536u /* must be power-of-2 */
|
||||
#define MDBX_PAGEVIZ_MAX_RINGS 128u
|
||||
#define MDBX_PAGEVIZ_PUBLISH_INTERVAL 32u
|
||||
|
||||
/* ── Event encoding ───────────────────────────────────────────────────── */
|
||||
|
||||
/* Layout of a 64-bit event word:
|
||||
* bits 63..56 op (1=READ, 2=WRITE, 3=FREE)
|
||||
* bits 55..32 dbi (lower 24 bits of a uint32_t)
|
||||
* bits 31..0 pgno (uint32_t page number) */
|
||||
|
||||
#define MDBX_PAGEVIZ_OP_READ 1
|
||||
#define MDBX_PAGEVIZ_OP_WRITE 2
|
||||
#define MDBX_PAGEVIZ_OP_FREE 3
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_START 4
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_END 5
|
||||
|
||||
#define MDBX_PAGEVIZ_ENCODE(op, dbi, pgno) \
|
||||
(((uint64_t)(op) << 56) | ((uint64_t)((dbi) & 0x00FFFFFFu) << 32) | \
|
||||
(uint64_t)(uint32_t)(pgno))
|
||||
|
||||
#define MDBX_PAGEVIZ_DECODE_OP(ev) ((uint8_t)((ev) >> 56))
|
||||
#define MDBX_PAGEVIZ_DECODE_DBI(ev) ((uint32_t)(((ev) >> 32) & 0x00FFFFFFu))
|
||||
#define MDBX_PAGEVIZ_DECODE_PGNO(ev) ((uint32_t)(ev))
|
||||
|
||||
/* ── Ring-buffer mask (power-of-2 capacity) ───────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_MASK (MDBX_PAGEVIZ_RING_CAPACITY - 1u)
|
||||
|
||||
/* ── Platform TLS ─────────────────────────────────────────────────────── */
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
# define MDBX_PAGEVIZ_TLS __declspec(thread)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
# define MDBX_PAGEVIZ_TLS __thread
|
||||
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
# define MDBX_PAGEVIZ_TLS _Thread_local
|
||||
#else
|
||||
# error "No thread-local storage support detected"
|
||||
#endif
|
||||
|
||||
/* ── Structures ───────────────────────────────────────────────────────── */
|
||||
|
||||
typedef struct mdbx_pageviz_ring {
|
||||
uint64_t events[MDBX_PAGEVIZ_RING_CAPACITY];
|
||||
_Atomic uint32_t published_head; /* consumer reads (acquire) */
|
||||
uint32_t local_head; /* producer-only, no atomic needed */
|
||||
_Atomic uint32_t consumer_tail; /* consumer updates after draining */
|
||||
_Atomic uint64_t dropped; /* unused, kept for ABI compat */
|
||||
uint32_t _pad[10]; /* pad to separate cache lines */
|
||||
} mdbx_pageviz_ring_t;
|
||||
|
||||
typedef struct mdbx_pageviz_mapping {
|
||||
volatile void *base;
|
||||
volatile size_t len;
|
||||
volatile uint32_t mdbx_page_size;
|
||||
volatile uint32_t sys_page_size;
|
||||
} mdbx_pageviz_mapping_t;
|
||||
|
||||
typedef struct mdbx_pageviz_state {
|
||||
mdbx_pageviz_ring_t rings[MDBX_PAGEVIZ_MAX_RINGS];
|
||||
_Atomic uint32_t ring_count; /* how many rings registered */
|
||||
_Atomic uint32_t enabled; /* runtime enable/disable */
|
||||
mdbx_pageviz_mapping_t mapping;
|
||||
} mdbx_pageviz_state_t;
|
||||
|
||||
/* ── Global state (defined in .c) ─────────────────────────────────────── */
|
||||
|
||||
extern mdbx_pageviz_state_t *mdbx_pageviz_global;
|
||||
|
||||
/* ── Thread-local ring index (defined in .c) ──────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_TLS_UNREGISTERED UINT32_MAX
|
||||
|
||||
extern MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring;
|
||||
|
||||
/* ── Inline helpers (private) ─────────────────────────────────────────── */
|
||||
|
||||
static inline uint32_t mdbx_pageviz_register_ring_(mdbx_pageviz_state_t *s) {
|
||||
uint32_t idx =
|
||||
atomic_fetch_add_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
if (idx >= MDBX_PAGEVIZ_MAX_RINGS) {
|
||||
atomic_fetch_sub_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
return MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
}
|
||||
mdbx_pageviz_ring_t *r = &s->rings[idx];
|
||||
atomic_store_explicit(&r->published_head, 0, memory_order_relaxed);
|
||||
r->local_head = 0;
|
||||
atomic_store_explicit(&r->consumer_tail, 0, memory_order_relaxed);
|
||||
atomic_store_explicit(&r->dropped, 0, memory_order_relaxed);
|
||||
return idx;
|
||||
}
|
||||
|
||||
/* ── Hot-path emit (inlined) ──────────────────────────────────────────── */
|
||||
|
||||
static inline void mdbx_pageviz_emit(uint8_t op, uint32_t dbi,
|
||||
uint32_t pgno) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (__builtin_expect(
|
||||
state == NULL ||
|
||||
!atomic_load_explicit(&state->enabled, memory_order_relaxed),
|
||||
1))
|
||||
return;
|
||||
|
||||
uint32_t idx = mdbx_pageviz_tls_ring;
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0)) {
|
||||
idx = mdbx_pageviz_register_ring_(state);
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0))
|
||||
return;
|
||||
mdbx_pageviz_tls_ring = idx;
|
||||
}
|
||||
|
||||
mdbx_pageviz_ring_t *r = &state->rings[idx];
|
||||
|
||||
/* Block (spin) until the consumer drains enough space. */
|
||||
while (__builtin_expect(
|
||||
(uint32_t)(r->local_head -
|
||||
atomic_load_explicit(&r->consumer_tail, memory_order_acquire))
|
||||
>= MDBX_PAGEVIZ_RING_CAPACITY, 0)) {
|
||||
/* Flush our writes so the consumer can make progress. */
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
|
||||
__builtin_ia32_pause();
|
||||
#elif defined(__aarch64__) || defined(_M_ARM64)
|
||||
__asm__ volatile("yield");
|
||||
#endif
|
||||
}
|
||||
|
||||
r->events[r->local_head & MDBX_PAGEVIZ_RING_MASK] =
|
||||
MDBX_PAGEVIZ_ENCODE(op, dbi, pgno);
|
||||
r->local_head++;
|
||||
|
||||
if ((r->local_head & (MDBX_PAGEVIZ_PUBLISH_INTERVAL - 1u)) == 0)
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Macro hooks (inserted into mdbx.c) ──────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_READ, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_WRITE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
|
||||
/* ── Public API (defined in .c) ───────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void);
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state);
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count);
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state);
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx);
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size);
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps);
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded);
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
#endif /* MDBX_PAGEVIZ_H */
|
||||
@@ -36,6 +36,9 @@ mod flags;
|
||||
mod transaction;
|
||||
mod txn_manager;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub mod pageviz;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_utils {
|
||||
use super::*;
|
||||
|
||||
226
crates/storage/libmdbx-rs/src/pageviz.rs
Normal file
226
crates/storage/libmdbx-rs/src/pageviz.rs
Normal file
@@ -0,0 +1,226 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc, Arc,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum PageOp {
|
||||
Read = 1,
|
||||
Write = 2,
|
||||
Free = 3,
|
||||
BlockStart = 4,
|
||||
BlockEnd = 5,
|
||||
}
|
||||
|
||||
impl PageOp {
|
||||
fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
1 => Some(Self::Read),
|
||||
2 => Some(Self::Write),
|
||||
3 => Some(Self::Free),
|
||||
4 => Some(Self::BlockStart),
|
||||
5 => Some(Self::BlockEnd),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_block_marker(self) -> bool {
|
||||
matches!(self, Self::BlockStart | Self::BlockEnd)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PageEvent {
|
||||
pub pgno: u32,
|
||||
pub dbi: u32,
|
||||
pub op: PageOp,
|
||||
}
|
||||
|
||||
impl PageEvent {
|
||||
fn decode(raw: u64) -> Option<Self> {
|
||||
let op_byte = (raw >> 56) as u8;
|
||||
let op = PageOp::from_u8(op_byte)?;
|
||||
let dbi = ((raw >> 32) & 0x00FF_FFFF) as u32;
|
||||
let pgno = raw as u32;
|
||||
Some(Self { pgno, dbi, op })
|
||||
}
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn mdbx_pageviz_create() -> *mut std::ffi::c_void;
|
||||
fn mdbx_pageviz_destroy(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_enable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_disable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_drain(
|
||||
state: *mut std::ffi::c_void,
|
||||
ring_idx: u32,
|
||||
out_buf: *mut u64,
|
||||
max_count: u32,
|
||||
) -> u32;
|
||||
fn mdbx_pageviz_ring_count(state: *mut std::ffi::c_void) -> u32;
|
||||
fn mdbx_pageviz_dropped(state: *mut std::ffi::c_void, ring_idx: u32) -> u64;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizStats {
|
||||
pub ring_count: u32,
|
||||
pub total_dropped: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct StatePtr(*mut std::ffi::c_void);
|
||||
|
||||
// SAFETY: The C state is thread-safe (uses atomics internally).
|
||||
unsafe impl Send for StatePtr {}
|
||||
unsafe impl Sync for StatePtr {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizDrainer {
|
||||
state: StatePtr,
|
||||
running: Arc<AtomicBool>,
|
||||
handle: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
const DRAIN_BUF_LEN: usize = 4096;
|
||||
const OVERLOAD_THRESHOLD: usize = 200_000;
|
||||
const READ_SAMPLE_RATE: usize = 8;
|
||||
|
||||
impl PageVizDrainer {
|
||||
pub fn new() -> Self {
|
||||
let state = StatePtr(unsafe { mdbx_pageviz_create() });
|
||||
Self { state, running: Arc::new(AtomicBool::new(false)), handle: None }
|
||||
}
|
||||
|
||||
pub fn enable(&self) {
|
||||
unsafe { mdbx_pageviz_enable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn disable(&self) {
|
||||
unsafe { mdbx_pageviz_disable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn start(&mut self, tick_hz: u32) -> mpsc::Receiver<Vec<PageEvent>> {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let running = self.running.clone();
|
||||
running.store(true, Ordering::SeqCst);
|
||||
|
||||
let state_addr = self.state.0 as usize;
|
||||
let tick_interval = Duration::from_micros(1_000_000 / u64::from(tick_hz));
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("mdbx-pageviz".into())
|
||||
.spawn(move || {
|
||||
let state = state_addr as *mut std::ffi::c_void;
|
||||
let mut buf = [0u64; DRAIN_BUF_LEN];
|
||||
let mut coalesced: HashMap<u32, PageEvent> = HashMap::new();
|
||||
let mut markers: Vec<PageEvent> = Vec::new();
|
||||
let mut read_counter: usize = 0;
|
||||
|
||||
while running.load(Ordering::Relaxed) {
|
||||
let tick_start = Instant::now();
|
||||
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(state) };
|
||||
|
||||
for ring_idx in 0..ring_count {
|
||||
let dropped = unsafe { mdbx_pageviz_dropped(state, ring_idx) };
|
||||
if dropped > 0 {
|
||||
tracing::warn!(
|
||||
target: "libmdbx::pageviz",
|
||||
ring_idx,
|
||||
dropped,
|
||||
"pageviz ring dropped events"
|
||||
);
|
||||
}
|
||||
|
||||
loop {
|
||||
let count = unsafe {
|
||||
mdbx_pageviz_drain(
|
||||
state,
|
||||
ring_idx,
|
||||
buf.as_mut_ptr(),
|
||||
DRAIN_BUF_LEN as u32,
|
||||
)
|
||||
};
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
for &raw in &buf[..count as usize] {
|
||||
if let Some(evt) = PageEvent::decode(raw) {
|
||||
if evt.op.is_block_marker() {
|
||||
markers.push(evt);
|
||||
} else {
|
||||
coalesced.insert(evt.pgno, evt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !coalesced.is_empty() || !markers.is_empty() {
|
||||
let mut events: Vec<PageEvent> = if coalesced.len() > OVERLOAD_THRESHOLD {
|
||||
coalesced
|
||||
.drain()
|
||||
.filter(|(_, evt)| {
|
||||
if evt.op == PageOp::Read {
|
||||
read_counter += 1;
|
||||
read_counter % READ_SAMPLE_RATE == 0
|
||||
} else {
|
||||
true
|
||||
}
|
||||
})
|
||||
.map(|(_, evt)| evt)
|
||||
.collect()
|
||||
} else {
|
||||
coalesced.drain().map(|(_, evt)| evt).collect()
|
||||
};
|
||||
|
||||
events.extend(markers.drain(..));
|
||||
|
||||
if tx.send(events).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = tick_start.elapsed();
|
||||
if let Some(remaining) = tick_interval.checked_sub(elapsed) {
|
||||
std::thread::sleep(remaining);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn mdbx-pageviz thread");
|
||||
|
||||
self.handle = Some(handle);
|
||||
rx
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) {
|
||||
self.running.store(false, Ordering::SeqCst);
|
||||
if let Some(handle) = self.handle.take() {
|
||||
let _ = handle.join();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> PageVizStats {
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(self.state.0) };
|
||||
let mut total_dropped = 0u64;
|
||||
for ring_idx in 0..ring_count {
|
||||
total_dropped += unsafe { mdbx_pageviz_dropped(self.state.0, ring_idx) };
|
||||
}
|
||||
PageVizStats { ring_count, total_dropped }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PageVizDrainer {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
unsafe {
|
||||
mdbx_pageviz_disable(self.state.0);
|
||||
mdbx_pageviz_destroy(self.state.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
1381
crates/storage/mdbx-viz/Cargo.lock
generated
Normal file
1381
crates/storage/mdbx-viz/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
23
crates/storage/mdbx-viz/Cargo.toml
Normal file
23
crates/storage/mdbx-viz/Cargo.toml
Normal file
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "reth-mdbx-viz"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Real-time MDBX page access visualization server"
|
||||
|
||||
[dependencies]
|
||||
reth-libmdbx = { workspace = true, features = ["pageviz"] }
|
||||
reth-mdbx-sys = { workspace = true, features = ["pageviz"] }
|
||||
axum = { version = "0.7", features = ["ws"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
tracing.workspace = true
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
rayon.workspace = true
|
||||
eyre.workspace = true
|
||||
libc = "0.2"
|
||||
952
crates/storage/mdbx-viz/src/index.html
Normal file
952
crates/storage/mdbx-viz/src/index.html
Normal file
@@ -0,0 +1,952 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>MDBX Live Viz</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
html,body{background:#0d1117;color:#c9d1d9;font-family:'SF Mono','Fira Code','Cascadia Code',monospace;height:100%;overflow:hidden}
|
||||
body{display:flex;flex-direction:row;height:100%}
|
||||
#main-panel{flex:1;display:flex;flex-direction:column;padding:16px;overflow:hidden;min-width:0}
|
||||
#side-panel{flex:0 0 540px;border-left:1px solid #30363d;overflow-y:auto;padding:16px;background:#161b22}
|
||||
h1{color:#f0f6fc;font-size:18px;margin-bottom:2px}
|
||||
.subtitle{color:#8b949e;font-size:12px;margin-bottom:10px}
|
||||
.sub-row{display:flex;align-items:baseline;gap:12px;margin-bottom:6px;flex-wrap:wrap}
|
||||
.sub-row .subtitle{margin-bottom:0;flex-shrink:0}
|
||||
.sub-row .viz-controls{margin-bottom:0;margin-left:auto}
|
||||
.controls{margin-bottom:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
||||
.controls button{background:#21262d;border:1px solid #30363d;color:#c9d1d9;padding:3px 10px;border-radius:4px;cursor:pointer;font-family:inherit;font-size:12px}
|
||||
.controls button:hover{background:#30363d}
|
||||
.zoom-info{font-size:11px;color:#8b949e}
|
||||
.status-bar{display:flex;gap:12px;align-items:center;margin-bottom:6px;font-size:11px}
|
||||
.dot{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:4px;vertical-align:middle}
|
||||
.dot.connected{background:#4ade80}.dot.disconnected{background:#f87171}.dot.connecting{background:#facc15}
|
||||
.stat-label{color:#8b949e}.stat-value{color:#c9d1d9;font-weight:600}
|
||||
#canvas-wrap{border:1px solid #30363d;border-radius:6px;overflow:hidden;background:#000;position:relative;cursor:crosshair;flex:1;min-height:0}
|
||||
canvas{display:block;image-rendering:pixelated;width:100%;height:100%}
|
||||
#tooltip{position:fixed;background:#1c2128;border:1px solid #30363d;border-radius:6px;padding:8px 12px;font-size:12px;pointer-events:none;display:none;z-index:100;box-shadow:0 4px 12px rgba(0,0,0,.4);max-width:360px}
|
||||
#tooltip .tt-row{display:flex;align-items:center;gap:6px}
|
||||
#tooltip .tt-swatch{width:10px;height:10px;border-radius:2px;flex-shrink:0;border:1px solid rgba(255,255,255,0.2)}
|
||||
#tooltip .tt-name{color:#f0f6fc;font-weight:600}
|
||||
#tooltip .tt-detail{color:#8b949e;margin-top:2px}
|
||||
#side-panel h2{color:#f0f6fc;font-size:14px;margin-bottom:8px}
|
||||
.viz-controls{margin-bottom:12px;font-size:11px}
|
||||
.viz-controls label{display:inline-flex;align-items:center;gap:4px;margin-right:10px;cursor:pointer;color:#8b949e}
|
||||
.viz-controls label:hover{color:#c9d1d9}
|
||||
.viz-controls input[type=checkbox]{accent-color:#58a6ff}
|
||||
.viz-controls input[type=range]{width:100px;accent-color:#58a6ff}
|
||||
.fade-val{color:#c9d1d9;min-width:35px;display:inline-block}
|
||||
table{width:100%;border-collapse:collapse;font-size:11px;background:#0d1117;border:1px solid #30363d;border-radius:6px;overflow:hidden;table-layout:fixed}
|
||||
td,th{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
th{text-align:left;padding:6px 8px;background:#1c2128;color:#8b949e;font-weight:500;border-bottom:1px solid #30363d;font-size:10px;text-transform:uppercase;position:sticky;top:0}
|
||||
td{padding:5px 8px;border-bottom:1px solid #21262d}
|
||||
tr.tbl-row{cursor:pointer;transition:background 0.1s}
|
||||
tr.tbl-row:hover td{background:#1c2128}
|
||||
tr.tbl-row.active td{background:#1c2128;outline:1px solid #58a6ff}
|
||||
.pct-bar{display:inline-block;height:6px;border-radius:3px;margin-right:6px;vertical-align:middle}
|
||||
.swatch-cell{width:12px;height:12px;border-radius:2px;flex-shrink:0}
|
||||
#loading{display:flex;flex-direction:column;align-items:center;justify-content:center;height:100%;color:#8b949e;font-size:14px;gap:12px}
|
||||
#load-bar-wrap{width:280px;height:6px;background:#21262d;border-radius:3px;overflow:hidden;display:none}
|
||||
#load-bar{height:100%;width:0;background:#58a6ff;border-radius:3px;transition:width 0.15s}
|
||||
#flame-wrap{border:1px solid #30363d;border-radius:0 0 6px 6px;margin-top:-1px;background:#0d1117}
|
||||
#flame-wrap canvas{display:block}
|
||||
.spark-row{display:flex;align-items:center;gap:6px;margin-bottom:3px;font-size:10px}
|
||||
.spark-label{width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
|
||||
.spark-cv{border:1px solid #21262d;border-radius:2px;background:#0d1117}
|
||||
.spark-val{color:#c9d1d9;min-width:30px;text-align:right}
|
||||
.tree-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:4px;font-size:10px;cursor:pointer}
|
||||
.tree-card:hover{border-color:#58a6ff}
|
||||
.tree-card .tree-name{color:#f0f6fc;font-weight:600;font-size:11px}
|
||||
.tree-card .tree-stat{color:#8b949e;margin-top:2px}
|
||||
.tree-card .tree-bar{display:flex;gap:1px;margin-top:3px;height:6px;border-radius:2px;overflow:hidden}
|
||||
.tree-card .tree-seg{height:100%}
|
||||
.cache-stat{display:flex;justify-content:space-between;align-items:center;font-size:10px;padding:2px 0}
|
||||
.cache-bar{height:4px;border-radius:2px;background:#21262d;flex:1;margin:0 6px;position:relative;overflow:hidden}
|
||||
.cache-fill{height:100%;border-radius:2px;background:#4ade80;position:absolute;left:0;top:0}
|
||||
.cache-pct{color:#4ade80;font-weight:600;min-width:36px;text-align:right}
|
||||
.cache-label{color:#8b949e;min-width:80px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
.fault-cnt{color:#d946ef;font-size:10px;margin-left:4px}
|
||||
.block-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:3px;font-size:10px;cursor:pointer;transition:border-color 0.15s}
|
||||
.block-card:hover{border-color:#58a6ff}
|
||||
.block-card.replaying{border-color:#f59e0b;background:#1e1b13}
|
||||
.block-card.recording{border-color:#4ade80;animation:pulse-border 1s infinite}
|
||||
.block-card .block-num{color:#f0f6fc;font-weight:600;font-size:11px}
|
||||
.block-card .block-stat{color:#8b949e;margin-top:1px}
|
||||
@keyframes pulse-border{0%,100%{border-color:#4ade80}50%{border-color:#166534}}
|
||||
#block-live{cursor:pointer;font-size:10px;margin-left:6px;user-select:none}
|
||||
.utbl-row{display:flex;align-items:center;gap:6px;padding:3px 4px;font-size:10px;cursor:pointer}
|
||||
.utbl-row:hover{background:#1c2128}
|
||||
.utbl-row.active{background:#1c2128;outline:1px solid #58a6ff}
|
||||
.utbl-sw{width:8px;height:8px;border-radius:2px;flex-shrink:0}
|
||||
.utbl-name{width:110px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
|
||||
.utbl-spark{flex-shrink:0}
|
||||
.utbl-cache{flex:1;min-width:0}
|
||||
.utbl-cache-bar-wrap{width:100%;height:6px;background:#21262d;border-radius:3px;overflow:hidden}
|
||||
.utbl-cache-fill{height:100%;border-radius:3px;transition:width 0.3s}
|
||||
.utbl-cache-pct{text-align:center;font-size:9px;color:#4ade80;font-weight:600;margin-top:1px;line-height:1}
|
||||
.utbl-diff{width:52px;text-align:right;font-size:9px}
|
||||
.utbl-faults{width:36px;text-align:right;color:#d946ef;font-size:9px}
|
||||
#block-panel::-webkit-scrollbar,#side-panel::-webkit-scrollbar{width:6px}
|
||||
#block-panel::-webkit-scrollbar-track,#side-panel::-webkit-scrollbar-track{background:#161b22}
|
||||
#block-panel::-webkit-scrollbar-thumb,#side-panel::-webkit-scrollbar-thumb{background:#30363d;border-radius:3px}
|
||||
#block-panel::-webkit-scrollbar-thumb:hover,#side-panel::-webkit-scrollbar-thumb:hover{background:#484f58}
|
||||
#block-panel,#side-panel{scrollbar-width:thin;scrollbar-color:#30363d #161b22}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="main-panel">
|
||||
<h1>MDBX Live Viz</h1>
|
||||
<div class="sub-row">
|
||||
<div class="subtitle" id="subtitle">Loading...</div>
|
||||
<div class="viz-controls">
|
||||
<label><input type="checkbox" id="chk-read" checked> <span style="color:#3c82f6">■</span> Reads</label>
|
||||
<label><input type="checkbox" id="chk-write" checked> <span style="color:#ef4444">■</span> Writes</label>
|
||||
<label><input type="checkbox" id="chk-free" checked> <span style="color:#eab308">■</span> Frees</label>
|
||||
<label><input type="checkbox" id="chk-fixw"> Fixed Width</label>
|
||||
<label><input type="checkbox" id="chk-trails" checked> Trails</label>
|
||||
<label><input type="checkbox" id="chk-heatmap"> Heatmap</label>
|
||||
<label><input type="checkbox" id="chk-flame"> Timeline</label>
|
||||
<label><input type="checkbox" id="chk-cache"> Page Cache</label>
|
||||
<label>Fade: <input type="range" id="fade-slider" min="200" max="8000" value="2000" step="100"> <span class="fade-val" id="fade-val">2.0s</span></label>
|
||||
<label>Trail W: <input type="range" id="trail-width" min="0.5" max="6" value="1.0" step="0.5"> <span class="fade-val" id="trail-w-val">1.0</span></label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="status-bar">
|
||||
<span><span class="dot connecting" id="ws-dot"></span><span id="ws-status">Connecting</span></span>
|
||||
<span class="stat-label">Events/s: <span class="stat-value" id="evt-rate">0</span></span>
|
||||
<span class="stat-label">Total: <span class="stat-value" id="evt-total">0</span></span>
|
||||
<span class="stat-label">Active: <span class="stat-value" id="active-count">0</span></span>
|
||||
<span class="stat-label">FPS: <span class="stat-value" id="fps-counter">-</span></span>
|
||||
</div>
|
||||
<div class="status-bar">
|
||||
<span class="stat-label" style="color:#3c82f6">R: <span class="stat-value" id="cnt-read">0</span></span>
|
||||
<span class="stat-label" style="color:#ef4444">W: <span class="stat-value" id="cnt-write">0</span></span>
|
||||
<span class="stat-label" style="color:#eab308">F: <span class="stat-value" id="cnt-free">0</span></span>
|
||||
<span class="stat-label">Latest: <span class="stat-value" id="evt-latest">-</span></span>
|
||||
<span id="lag-warn" style="display:none;color:#f87171;font-weight:600">⚠ Lagged <span id="lag-count">0</span> msgs</span>
|
||||
</div>
|
||||
<div class="controls">
|
||||
<button id="btn-zin">Zoom +</button>
|
||||
<button id="btn-zout">Zoom −</button>
|
||||
<button id="btn-reset">Reset</button>
|
||||
<button id="btn-clear">Clear Highlights</button>
|
||||
<span class="zoom-info" id="zoom-info"></span>
|
||||
</div>
|
||||
<div id="canvas-wrap">
|
||||
<div id="loading"><span id="load-text">Loading page map…</span><div id="load-bar-wrap"><div id="load-bar"></div></div></div>
|
||||
<canvas id="map" style="display:none"></canvas>
|
||||
</div>
|
||||
<div id="flame-wrap" style="height:0;overflow:hidden;transition:height 0.2s">
|
||||
<canvas id="flame"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div id="side-panel">
|
||||
<h2>Blocks <span id="block-live" style="color:#4ade80" onclick="window._goLive&&window._goLive()">● LIVE</span></h2>
|
||||
<div id="block-panel" style="max-height:172px;overflow-y:auto"></div>
|
||||
<h2 style="margin-top:12px">Tables <span id="cache-total" style="color:#8b949e;font-weight:400;font-size:11px"></span> <span id="cache-ago" style="color:#8b949e;font-weight:400;font-size:10px"></span></h2>
|
||||
<div id="unified-tables"></div>
|
||||
<h2 style="margin-top:12px">Table Details</h2>
|
||||
<table><thead><tr><th></th><th>Table</th><th>Pages</th><th>Size</th><th>%</th></tr></thead><tbody id="tbl-body"></tbody></table>
|
||||
<h2 style="margin-top:12px">B-Tree Structure</h2>
|
||||
<div id="tree-info"></div>
|
||||
</div>
|
||||
<div id="tooltip">
|
||||
<div class="tt-row"><div class="tt-swatch" id="tt-swatch"></div><div class="tt-name" id="tt-name"></div></div>
|
||||
<div class="tt-detail" id="tt-detail"></div>
|
||||
</div>
|
||||
<script>
|
||||
(function(){
|
||||
"use strict";
|
||||
function hsl2r(h,s,l){h/=360;s/=100;l/=100;let r,g,b;if(!s){r=g=b=l}else{const q=l<.5?l*(1+s):l+s-l*s,p=2*l-q;const f=(p,q,t)=>{if(t<0)t+=1;if(t>1)t-=1;if(t<1/6)return p+(q-p)*6*t;if(t<.5)return q;if(t<2/3)return p+(q-p)*(2/3-t)*6;return p};r=f(p,q,h+1/3);g=f(p,q,h);b=f(p,q,h-1/3)}return[Math.round(r*255),Math.round(g*255),Math.round(b*255)]}
|
||||
const DBI_HUES=new Float32Array(256);
|
||||
for(let i=2;i<254;i++)DBI_HUES[i]=((i-2)*137.508)%360;
|
||||
const PAL=new Array(256),PAL_DESAT=new Array(256),PAL_DARK=new Array(256),PAL_DD=new Array(256);
|
||||
PAL[0]=hsl2r(220,15,82);PAL_DESAT[0]=hsl2r(220,5,72);PAL_DARK[0]=hsl2r(220,15,28);PAL_DD[0]=hsl2r(220,5,22);
|
||||
PAL[1]=hsl2r(220,30,24);PAL_DESAT[1]=hsl2r(220,8,22);PAL_DARK[1]=hsl2r(220,30,12);PAL_DD[1]=hsl2r(220,8,12);
|
||||
for(let i=2;i<254;i++){const h=DBI_HUES[i];PAL[i]=hsl2r(h,70,55);PAL_DESAT[i]=hsl2r(h,15,50);PAL_DARK[i]=hsl2r(h,70,22);PAL_DD[i]=hsl2r(h,15,20)}
|
||||
PAL[0xFE]=[20,20,23];PAL_DESAT[0xFE]=[20,20,23];PAL_DARK[0xFE]=[12,12,14];PAL_DD[0xFE]=[12,12,14];
|
||||
PAL[0xFF]=[28,28,32];PAL_DESAT[0xFF]=[28,28,32];PAL_DARK[0xFF]=[16,16,18];PAL_DD[0xFF]=[16,16,18];
|
||||
const FLASH_TINT={1:[140,180,255],2:[255,140,140],3:[255,220,100]};
|
||||
let NP=0,PS=4096,GC=0,GR=0,DBIS=[],DBI_IDX={},oMap=null,lOp=null,lTs=null,FADE=2000;
|
||||
let cntR=0,cntW=0,cntF=0;
|
||||
let sR=true,sW=true,sF=true,fW=false,showTrails=true,trailW=1.0;
|
||||
const TRAIL_CAP=4096;let trailBuf=new Array(TRAIL_CAP);let trailStart=0,trailN=0;
|
||||
const TRC={1:"60,130,246",2:"255,80,80",3:"234,179,8"};
|
||||
let activePgs=0;
|
||||
const activeSet=new Set();
|
||||
let oMapDirty=false;
|
||||
let unrefCount=0;
|
||||
let outR=0,outG=0,outB=0;
|
||||
let cacheTotalCached=0,cacheCachedPer=null,cachePrevPer=null,cachePrevTotal=0,cacheStale=true;
|
||||
let hmMode=false,hmCounts=null;
|
||||
let cacheMode=false,resMap=null,faultCounts={},faultWindow=[],faultCurrent={};
|
||||
let cacheLastUpdate=0;
|
||||
let zbCv=null,zbCx=null,zbDirty=true,zbW=0,zbH=0,zbPX=0,zbPY=0,zbSc=0,zbFlt=false;
|
||||
const BASE_LUT=new Uint32Array(256*32*2);
|
||||
function buildLUT(){
|
||||
for(let dbi=0;dbi<256;dbi++){for(let hq=0;hq<32;hq++){const ht=hq/31;for(let c=0;c<2;c++){
|
||||
let cf,cd;
|
||||
if(c){cf=PAL[dbi];cd=PAL_DESAT[dbi]}else{cf=PAL_DARK[dbi];cd=PAL_DD[dbi]}
|
||||
if(!cf)cf=PAL[255];if(!cd)cd=PAL_DESAT[255];
|
||||
const r=Math.round(cd[0]+(cf[0]-cd[0])*ht);
|
||||
const g=Math.round(cd[1]+(cf[1]-cd[1])*ht);
|
||||
const b=Math.round(cd[2]+(cf[2]-cd[2])*ht);
|
||||
BASE_LUT[(dbi*32+hq)*2+c]=(255<<24)|(b<<16)|(g<<8)|r;
|
||||
}}}
|
||||
}
|
||||
buildLUT();
|
||||
function heatQ(pg){if(!hmMode||!hmCounts)return 31;const cnt=hmCounts[pg];return cnt>0?Math.min(31,Math.floor(Math.log2(cnt+1)*31/16)):0}
|
||||
function rollingFaults(dbi){let s=0;for(const w of faultWindow)s+=(w[dbi]||0);return s}
|
||||
function markZbDirty(){zbDirty=true}
|
||||
let blockRecording=null,blockHistory=[],replayMode=false,replayBlockRef=null,replayTimer=null,liveSnapshot=null;
|
||||
const BLOCK_HISTORY_CAP=15;
|
||||
let blockPanelDirty=false;
|
||||
let showFlame=false;const FLAME_SECS=120;let flameData={},flameHead=0;
|
||||
const SPARK_SECS=60;let sparkData={},sparkAccum={};
|
||||
const cv=document.getElementById("map"),cx=cv.getContext("2d"),wr=document.getElementById("canvas-wrap");
|
||||
const ldEl=document.getElementById("loading"),zmEl=document.getElementById("zoom-info");
|
||||
const tt=document.getElementById("tooltip"),ttSw=document.getElementById("tt-swatch");
|
||||
const ttNm=document.getElementById("tt-name"),ttDt=document.getElementById("tt-detail");
|
||||
const tb=document.getElementById("tbl-body");
|
||||
let W=0,H=0,iD=null,oC=null,oCx=null,pX=0,pY=0,sc=1,mS=.01;const xS=128;
|
||||
const sel=new Set();let tEv=0,rEv=0,lRT=performance.now();
|
||||
function cl(v,a,b){return Math.max(a,Math.min(b,v))}
|
||||
function growTo(newNP){
|
||||
if(newNP<=NP)return;
|
||||
const oldNP=NP;
|
||||
const nOm=new Uint8Array(newNP);nOm.set(oMap);nOm.fill(0xFF,oldNP);oMap=nOm;
|
||||
const nOp=new Uint8Array(newNP);nOp.set(lOp);lOp=nOp;
|
||||
const nTs=new Float64Array(newNP);nTs.set(lTs);lTs=nTs;
|
||||
if(hmCounts){const nHm=new Uint32Array(newNP);nHm.set(hmCounts);hmCounts=nHm}
|
||||
if(resMap){const nRm=new Uint8Array(newNP);nRm.set(resMap);resMap=nRm}
|
||||
unrefCount+=newNP-oldNP;
|
||||
NP=newNP;GR=Math.ceil(NP/GC);
|
||||
oMapDirty=true;
|
||||
iD=null;oC=null;markZbDirty();
|
||||
}
|
||||
function oX(){return 0}
|
||||
function oY(){return Math.max(0,(H-GR*sc)/2)}
|
||||
function cP(){const vw=W/sc,vh=H/sc;pX=cl(pX,0,Math.max(0,GC-vw));pY=cl(pY,0,Math.max(0,GR-vh))}
|
||||
function s2w(sx,sy){return{wx:pX+(sx-oX())/sc,wy:pY+(sy-oY())/sc}}
|
||||
function w2p(wx,wy){const px=Math.floor(wx),py=Math.floor(wy);if(px<0||py<0||px>=GC||py>=GR)return-1;const i=py*GC+px;return i<NP?i:-1}
|
||||
|
||||
function trailPush(from,to,op,ts){
|
||||
const idx=(trailStart+trailN)%TRAIL_CAP;
|
||||
if(!trailBuf[idx])trailBuf[idx]={from,to,op,ts};
|
||||
else{trailBuf[idx].from=from;trailBuf[idx].to=to;trailBuf[idx].op=op;trailBuf[idx].ts=ts}
|
||||
if(trailN<TRAIL_CAP)trailN++;
|
||||
else trailStart=(trailStart+1)%TRAIL_CAP;
|
||||
}
|
||||
function trailExpire(now){
|
||||
while(trailN>0){
|
||||
const t=trailBuf[trailStart];
|
||||
if(now-t.ts>=FADE){trailStart=(trailStart+1)%TRAIL_CAP;trailN--}
|
||||
else break;
|
||||
}
|
||||
}
|
||||
function pCBase(pg,flt){
|
||||
if(pg<0||pg>=NP)return false;
|
||||
const ow=oMap[pg];
|
||||
if(flt&&!sel.has(ow)){outR=25;outG=25;outB=28;return true}
|
||||
const hq=heatQ(pg);
|
||||
const cached=(cacheMode&&resMap&&pg<resMap.length)?resMap[pg]?1:0:1;
|
||||
const rgba=BASE_LUT[(ow*32+hq)*2+cached];
|
||||
outR=rgba&0xFF;outG=(rgba>>8)&0xFF;outB=(rgba>>16)&0xFF;
|
||||
return true;
|
||||
}
|
||||
function pC(pg,flt,now){
|
||||
if(!pCBase(pg,flt))return false;
|
||||
const op=lOp[pg];
|
||||
if(op&&((op===1&&sR)||(op===2&&sW)||(op===3&&sF))){
|
||||
const age=now-lTs[pg];
|
||||
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);activeSet.delete(pg)}
|
||||
else{const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
|
||||
outR=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
|
||||
outG=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
|
||||
outB=Math.min(255,Math.round(outB+(fl[2]-outB)*fi))}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function pg2s(pg){const col=pg%GC,row=Math.floor(pg/GC);return{sx:oX()+(col-pX+.5)*sc,sy:oY()+(row-pY+.5)*sc}}
|
||||
function drawTrails(now){
|
||||
if(!showTrails||trailN===0)return;
|
||||
cx.lineCap="round";cx.lineWidth=trailW;
|
||||
const paths={1:[],2:[],3:[]};
|
||||
for(let j=0;j<trailN;j++){
|
||||
const idx=(trailStart+j)%TRAIL_CAP;
|
||||
const t=trailBuf[idx];
|
||||
const age=now-t.ts;if(age>=FADE)continue;
|
||||
if((t.op===1&&!sR)||(t.op===2&&!sW)||(t.op===3&&!sF))continue;
|
||||
const a=1-Math.pow(age/FADE,2);
|
||||
const f=pg2s(t.from),e=pg2s(t.to);
|
||||
(paths[t.op]||(paths[t.op]=[])).push({fx:f.sx,fy:f.sy,ex:e.sx,ey:e.sy,a});
|
||||
}
|
||||
for(const op of [1,2,3]){
|
||||
const segs=paths[op];if(!segs||!segs.length)continue;
|
||||
const groups={};
|
||||
for(const s of segs){
|
||||
const ak=Math.round(s.a*10);
|
||||
if(!groups[ak])groups[ak]=[];
|
||||
groups[ak].push(s);
|
||||
}
|
||||
const c=TRC[op]||TRC[1];
|
||||
for(const [ak,gs] of Object.entries(groups)){
|
||||
cx.strokeStyle=`rgba(${c},${(ak/10*0.6).toFixed(2)})`;
|
||||
cx.beginPath();
|
||||
for(const s of gs){cx.moveTo(s.fx,s.fy);cx.lineTo(s.ex,s.ey)}
|
||||
cx.stroke();
|
||||
}
|
||||
}
|
||||
}
|
||||
function rebuildZoomBase(flt){
|
||||
if(!zbCv){zbCv=document.createElement("canvas");zbCx=zbCv.getContext("2d")}
|
||||
zbCv.width=W;zbCv.height=H;zbW=W;zbH=H;zbPX=pX;zbPY=pY;zbSc=sc;zbFlt=flt;
|
||||
const zd=zbCx.createImageData(W,H),d=zd.data,ox=oX(),oy=oY(),ips=1/sc;
|
||||
for(let sy=0;sy<H;sy++){
|
||||
const wy0=Math.max(0,Math.floor(pY+(sy-oy)*ips)),wy1=Math.min(GR-1,Math.floor(pY+(sy-oy+1)*ips));
|
||||
for(let sx=0;sx<W;sx++){
|
||||
const wx0=Math.max(0,Math.floor(pX+(sx-ox)*ips)),wx1=Math.min(GC-1,Math.floor(pX+(sx-ox+1)*ips));
|
||||
const o=(sy*W+sx)<<2;
|
||||
if(wx1<0||wy1<0||wx0>=GC||wy0>=GR){d[o]=0;d[o+1]=0;d[o+2]=0;d[o+3]=255;continue}
|
||||
const cpx=Math.floor((wx0+wx1)/2),cpy=Math.floor((wy0+wy1)/2);
|
||||
const ci=cpy*GC+cpx;
|
||||
if(ci>=0&&ci<NP&&pCBase(ci,flt)){d[o]=outR;d[o+1]=outG;d[o+2]=outB}
|
||||
d[o+3]=255;
|
||||
}
|
||||
}
|
||||
zbCx.putImageData(zd,0,0);
|
||||
}
|
||||
function drawActiveOverlay(now,flt){
|
||||
if(activeSet.size===0)return;
|
||||
const ox=oX(),oy=oY(),ips=1/sc;
|
||||
const expired=[];
|
||||
for(const pg of activeSet){
|
||||
const op=lOp[pg];
|
||||
if(!op){expired.push(pg);continue}
|
||||
if((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF))continue;
|
||||
const age=now-lTs[pg];
|
||||
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);expired.push(pg);continue}
|
||||
pCBase(pg,flt);
|
||||
const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
|
||||
const r=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
|
||||
const g=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
|
||||
const b=Math.min(255,Math.round(outB+(fl[2]-outB)*fi));
|
||||
const col=pg%GC,row=Math.floor(pg/GC);
|
||||
const sx=ox+(col-pX)*sc,sy=oy+(row-pY)*sc;
|
||||
if(sx<-sc||sy<-sc||sx>W||sy>H)continue;
|
||||
const psz=Math.max(1,Math.ceil(sc));
|
||||
cx.fillStyle=`rgb(${r},${g},${b})`;
|
||||
cx.fillRect(Math.floor(sx),Math.floor(sy),psz,psz);
|
||||
}
|
||||
for(const pg of expired)activeSet.delete(pg);
|
||||
}
|
||||
function render(){
|
||||
if(!oMap||W<1||H<1)return;
|
||||
const flt=sel.size>0,now=performance.now();
|
||||
if(sc>=1){
|
||||
const vw=W/sc,vh=H/sc,x0=Math.floor(pX),y0=Math.floor(pY);
|
||||
const x1=Math.min(GC,Math.ceil(pX+vw)+1),y1=Math.min(GR,Math.ceil(pY+vh)+1);
|
||||
const vW=Math.max(1,x1-x0),vH=Math.max(1,y1-y0);
|
||||
if(!oC){oC=document.createElement("canvas");oCx=oC.getContext("2d")}
|
||||
if(oC.width!==vW||oC.height!==vH){oC.width=vW;oC.height=vH}
|
||||
const od=oCx.createImageData(vW,vH),d=od.data;
|
||||
for(let y=0;y<vH;y++){const gy=y0+y;for(let x=0;x<vW;x++){
|
||||
const i=gy*GC+(x0+x),o=(y*vW+x)<<2;
|
||||
const ok=(i>=0&&i<NP)?pC(i,flt,now):false;
|
||||
if(ok){d[o]=outR;d[o+1]=outG;d[o+2]=outB}else{d[o]=0;d[o+1]=0;d[o+2]=0}d[o+3]=255;
|
||||
}}
|
||||
oCx.putImageData(od,0,0);cx.imageSmoothingEnabled=false;cx.clearRect(0,0,W,H);
|
||||
cx.drawImage(oC,oX()-(pX-x0)*sc,oY()-(pY-y0)*sc,vW*sc,vH*sc);
|
||||
drawTrails(now);
|
||||
zmEl.textContent=sc.toFixed(sc>=10?0:1)+" px/page";
|
||||
}else{
|
||||
if(zbDirty||!zbCv||zbW!==W||zbH!==H||zbPX!==pX||zbPY!==pY||zbSc!==sc||zbFlt!==flt){
|
||||
rebuildZoomBase(flt);zbDirty=false;
|
||||
}
|
||||
cx.clearRect(0,0,W,H);
|
||||
cx.drawImage(zbCv,0,0);
|
||||
drawActiveOverlay(now,flt);
|
||||
drawTrails(now);
|
||||
const pp=1/sc;zmEl.textContent="1 px = "+pp.toFixed(pp>=10?0:1)+" pages";
|
||||
}
|
||||
}
|
||||
let anim=false;
|
||||
let fpsFrames=0,fpsLast=performance.now();
|
||||
function rLoop(){
|
||||
render();let act=0;const now=performance.now();
|
||||
act+=activePgs;
|
||||
if(showTrails){trailExpire(now);act+=trailN}
|
||||
document.getElementById("active-count").textContent=act.toLocaleString();
|
||||
fpsFrames++;
|
||||
const fpsNow=performance.now();
|
||||
if(fpsNow-fpsLast>=500){
|
||||
document.getElementById("fps-counter").textContent=Math.round(fpsFrames/((fpsNow-fpsLast)/1000));
|
||||
fpsFrames=0;fpsLast=fpsNow;
|
||||
}
|
||||
if(act>0&&!(replayMode&&!replayTimer))requestAnimationFrame(rLoop);else{anim=false;document.getElementById("fps-counter").textContent="-"}
|
||||
}
|
||||
function kick(){if(!anim){anim=true;requestAnimationFrame(rLoop)}}
|
||||
let schedPending=false;
|
||||
function sched(){if(!schedPending){schedPending=true;requestAnimationFrame(()=>{schedPending=false;render()})}}
|
||||
function szCv(){const w=wr.clientWidth,h=wr.clientHeight;if(w<1||h<1)return;W=w;H=h;cv.width=W;cv.height=H;iD=null}
|
||||
function fmtSz(mb){if(mb>=1024)return(mb/1024).toFixed(1)+" GB";if(mb>=1)return mb.toFixed(1)+" MB";return(mb*1024).toFixed(0)+" KB"}
|
||||
function bldTbl(){
|
||||
tb.innerHTML="";const mx=Math.max(...DBIS.map(d=>d.pct),.1);
|
||||
DBIS.forEach(d=>{const tr=document.createElement("tr");tr.className="tbl-row";tr.dataset.dbiIndex=d.dbi_index;
|
||||
const c=PAL[d.dbi_index]||PAL[255];
|
||||
tr.innerHTML=`<td><div class="swatch-cell" style="background:rgb(${c[0]},${c[1]},${c[2]})"></div></td><td>${d.name}</td><td>${d.pages.toLocaleString()}</td><td>${fmtSz(d.size_mb)}</td><td><span class="pct-bar" style="width:${d.pct/mx*60}px;background:rgb(${c[0]},${c[1]},${c[2]})"></span>${d.pct}%</td>`;
|
||||
tr.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
tb.appendChild(tr);
|
||||
});
|
||||
}
|
||||
function uAR(){document.querySelectorAll(".tbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")});document.querySelectorAll(".utbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")})}
|
||||
function reflow(){if(!fW)return;GC=Math.max(1,Math.floor(W/sc));GR=Math.ceil(NP/GC);pX=0;cP()}
|
||||
function boot(){szCv();const aspect=W>0&&H>0?W/H:1;GC=Math.max(1,Math.ceil(Math.sqrt(NP*aspect)));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);sc=mS;pX=0;pY=0;render()}
|
||||
function zA(ns,sx,sy){ns=cl(ns,mS,xS);if(Math.abs(ns-sc)<1e-6)return;const b=s2w(sx,sy);sc=ns;if(fW){reflow();pY=b.wy-(sy-oY())/sc;cP()}else{pX=b.wx-(sx-oX())/sc;pY=b.wy-(sy-oY())/sc;cP()}sched()}
|
||||
document.getElementById("btn-zin").addEventListener("click",()=>zA(sc*1.3,W>>1,H>>1));
|
||||
document.getElementById("btn-zout").addEventListener("click",()=>zA(sc/1.3,W>>1,H>>1));
|
||||
document.getElementById("btn-reset").addEventListener("click",()=>{sc=mS;pX=0;pY=0;sel.clear();uAR();markZbDirty();render()});
|
||||
document.getElementById("btn-clear").addEventListener("click",()=>{if(lOp)lOp.fill(0);if(hmCounts)hmCounts.fill(0);activePgs=0;activeSet.clear();markZbDirty();sched()});
|
||||
wr.addEventListener("wheel",e=>{e.preventDefault();const r=cv.getBoundingClientRect();zA(sc*Math.exp(-(e.deltaMode===1?e.deltaY*16:e.deltaY)*.002),(e.clientX-r.left)/r.width*W,(e.clientY-r.top)/r.height*H)},{passive:false});
|
||||
let drag=false,lCX=0,lCY=0;
|
||||
cv.addEventListener("pointerdown",e=>{drag=true;lCX=e.clientX;lCY=e.clientY;cv.setPointerCapture(e.pointerId);cv.style.cursor="grabbing"});
|
||||
cv.addEventListener("pointermove",e=>{
|
||||
if(drag){if(!fW)pX-=(e.clientX-lCX)/sc;pY-=(e.clientY-lCY)/sc;lCX=e.clientX;lCY=e.clientY;cP();sched();tt.style.display="none";return}
|
||||
const r=cv.getBoundingClientRect(),sx=(e.clientX-r.left)/r.width*W,sy=(e.clientY-r.top)/r.height*H;
|
||||
const w=s2w(sx,sy),pg=w2p(w.wx,w.wy);if(pg<0){tt.style.display="none";return}
|
||||
const ow=oMap[pg],inf=DBI_IDX[ow],c=PAL[ow]||PAL[255];
|
||||
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
let ex="";const op=lOp[pg];if(op===1)ex=" [READ]";else if(op===2)ex=" [WRITE]";else if(op===3)ex=" [FREE]";
|
||||
ttNm.textContent=`Page ${pg.toLocaleString()} \u2014 ${inf?inf.name:"idx "+ow}${ex}`;
|
||||
let dtxt=inf?`${inf.pages.toLocaleString()} pages, ${inf.pct}%`:"";
|
||||
if(hmCounts&&pg<hmCounts.length&&hmCounts[pg]>0)dtxt+=(dtxt?" \u00b7 ":"")+hmCounts[pg].toLocaleString()+" hits";
|
||||
if(resMap&&pg<resMap.length){
|
||||
const cached=resMap[pg];
|
||||
dtxt+=(dtxt?" \u00b7 ":"")+(cached?'<span style="color:#28c840;font-weight:600">CACHED</span>':'<span style="color:#f04040;font-weight:600">NOT CACHED</span>');
|
||||
}
|
||||
ttDt.innerHTML=dtxt;
|
||||
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
|
||||
});
|
||||
cv.addEventListener("pointerup",()=>{drag=false;cv.style.cursor="crosshair"});
|
||||
cv.addEventListener("pointercancel",()=>{drag=false;cv.style.cursor="crosshair"});
|
||||
cv.addEventListener("mouseleave",()=>{tt.style.display="none"});
|
||||
document.getElementById("chk-read").addEventListener("change",e=>{sR=e.target.checked;sched()});
|
||||
document.getElementById("chk-write").addEventListener("change",e=>{sW=e.target.checked;sched()});
|
||||
document.getElementById("chk-free").addEventListener("change",e=>{sF=e.target.checked;sched()});
|
||||
document.getElementById("fade-slider").addEventListener("input",e=>{FADE=parseInt(e.target.value);document.getElementById("fade-val").textContent=(FADE/1000).toFixed(1)+"s"});
|
||||
document.getElementById("chk-trails").addEventListener("change",e=>{showTrails=e.target.checked;if(!showTrails){trailStart=0;trailN=0}sched()});
|
||||
document.getElementById("trail-width").addEventListener("input",e=>{trailW=parseFloat(e.target.value);document.getElementById("trail-w-val").textContent=trailW.toFixed(1);sched()});
|
||||
document.getElementById("chk-heatmap").addEventListener("change",e=>{hmMode=e.target.checked;markZbDirty();sched()});
|
||||
document.getElementById("chk-cache").addEventListener("change",e=>{cacheMode=e.target.checked;markZbDirty();sched()});
|
||||
document.getElementById("chk-flame").addEventListener("change",e=>{
|
||||
showFlame=e.target.checked;const fw=document.getElementById("flame-wrap");
|
||||
const at=DBIS.filter(d=>d.pages>0);fw.style.height=showFlame?Math.min(at.length*8+24,200)+"px":"0";
|
||||
if(showFlame){szFlame();drawFlame()}
|
||||
});
|
||||
document.getElementById("chk-fixw").addEventListener("change",e=>{fW=e.target.checked;if(fW){reflow()}else{GC=Math.ceil(Math.sqrt(NP));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);pX=0;cP()}sched()});
|
||||
window.addEventListener("resize",()=>{if(!oMap)return;szCv();reflow();mS=Math.min(W/GC,H/GR,1);if(sc<mS)sc=mS;cP();render();if(showFlame){szFlame();drawFlame()}});
|
||||
const wD=document.getElementById("ws-dot"),wS=document.getElementById("ws-status");
|
||||
function connectWS(){
|
||||
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws`);
|
||||
ws.binaryType="arraybuffer";
|
||||
ws.onopen=()=>{wD.className="dot connected";wS.textContent="Connected"};
|
||||
ws.onclose=()=>{wD.className="dot disconnected";wS.textContent="Reconnecting...";setTimeout(connectWS,1000)};
|
||||
ws.onerror=()=>ws.close();
|
||||
let lagTimer=0;
|
||||
ws.onmessage=msg=>{
|
||||
if(typeof msg.data==="string"){try{const j=JSON.parse(msg.data);if(j.lagged){const el=document.getElementById("lag-warn"),lc=document.getElementById("lag-count");lc.textContent=j.lagged.toLocaleString();el.style.display="inline";clearTimeout(lagTimer);lagTimer=setTimeout(()=>{el.style.display="none"},5000)}}catch(e){}return}
|
||||
if(!oMap)return;const v=new DataView(msg.data);const now=performance.now();let cnt=0;
|
||||
let lastPg=-1,lastDbi=0,lastOp=0,prevPg=-1;
|
||||
{let scan=0,maxPg=NP;while(scan+8<=msg.data.byteLength){const pg=v.getUint32(scan,true),op=v.getUint8(scan+6);scan+=8;if(op>=4)continue;if(pg>=maxPg)maxPg=pg+1}if(maxPg>NP)growTo(maxPg)}
|
||||
let off=0;
|
||||
while(off+8<=msg.data.byteLength){
|
||||
const pg=v.getUint32(off,true),dbi=v.getUint16(off+4,true),op=v.getUint8(off+6),rsv=v.getUint8(off+7);off+=8;
|
||||
if(op===4){if(!replayMode){blockRecording={number:pg,events:[],before:{},faults:0,startTime:performance.now()};blockPanelDirty=true}continue}
|
||||
if(op===5){
|
||||
if(!replayMode&&blockRecording&&blockRecording.number===pg){
|
||||
const txCount=dbi&0xFF,gasEnc=(dbi>>8)&0xFF;
|
||||
blockRecording.txCount=txCount;
|
||||
blockRecording.duration=rsv>0?Math.pow(2,rsv/20.0):0;
|
||||
blockRecording.gasUsed=gasEnc>0?Math.pow(2,gasEnc/10.0):0;
|
||||
let r=0,w=0,f=0;
|
||||
for(const e of blockRecording.events){if(e.op===1)r++;else if(e.op===2)w++;else if(e.op===3)f++}
|
||||
blockRecording.reads=r;blockRecording.writes=w;blockRecording.frees=f;
|
||||
blockRecording.uniquePages=Object.keys(blockRecording.before).length;
|
||||
blockRecording.panelSnapshot={
|
||||
dbiPages:DBIS.map(d=>d.pages),
|
||||
cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,
|
||||
cachePrevPer:cachePrevPer?cachePrevPer.slice():null,
|
||||
cacheTotalCached,cachePrevTotal,unrefCount,
|
||||
faultWindow:faultWindow.map(w=>({...w})),
|
||||
faultCurrent:{...faultCurrent}
|
||||
};
|
||||
blockHistory.push(blockRecording);
|
||||
if(blockHistory.length>BLOCK_HISTORY_CAP)blockHistory.shift();
|
||||
blockRecording=null;
|
||||
blockPanelDirty=true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if(blockRecording){
|
||||
if(!(pg in blockRecording.before))blockRecording.before[pg]={om:oMap[pg],rm:resMap?resMap[pg]:0};
|
||||
blockRecording.events.push({pg,dbi,op});
|
||||
}
|
||||
if(pg>=NP)continue;cnt++;
|
||||
if(op===1)cntR++;else if(op===2)cntW++;else if(op===3)cntF++;
|
||||
if(cacheMode&&resMap&&op===1&&pg<resMap.length&&!resMap[pg]){faultCounts[dbi]=(faultCounts[dbi]||0)+1;faultCurrent[dbi]=(faultCurrent[dbi]||0)+1;if(blockRecording){blockRecording.faults=(blockRecording.faults||0)+1}}
|
||||
if(resMap&&pg<resMap.length&&op!==3)resMap[pg]=1;
|
||||
if(hmCounts)hmCounts[pg]++;
|
||||
if(sparkAccum[dbi]!==undefined)sparkAccum[dbi]++;
|
||||
if(op===2&&dbi>0&&dbi<0xFE){const old=oMap[pg];if(old!==dbi){oMap[pg]=dbi;oMapDirty=true;
|
||||
const di=DBI_IDX[dbi];if(di)di.pages++;
|
||||
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
|
||||
if(old===0xFF)unrefCount--;
|
||||
}}
|
||||
if(op===3){const old=oMap[pg];if(old!==1){oMap[pg]=1;oMapDirty=true;
|
||||
const di=DBI_IDX[1];if(di)di.pages++;
|
||||
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
|
||||
if(old===0xFF)unrefCount--;
|
||||
}}
|
||||
if(replayMode)continue;
|
||||
const prev=lOp[pg],prevLive=prev&&(now-lTs[pg])<FADE;
|
||||
if(!prevLive||op>=prev){
|
||||
if(prev===0)activePgs++;
|
||||
lOp[pg]=op;lTs[pg]=now;
|
||||
activeSet.add(pg);
|
||||
}
|
||||
if(showTrails&&prevPg>=0&&prevPg!==pg&&!((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF)))trailPush(prevPg,pg,op,now);
|
||||
prevPg=pg;
|
||||
if(op>=2){lastPg=pg;lastDbi=dbi;lastOp=op}else if(lastPg<0){lastPg=pg;lastDbi=dbi;lastOp=op}
|
||||
}
|
||||
tEv+=cnt;rEv+=cnt;document.getElementById("evt-total").textContent=tEv.toLocaleString();
|
||||
document.getElementById("cnt-read").textContent=cntR.toLocaleString();
|
||||
document.getElementById("cnt-write").textContent=cntW.toLocaleString();
|
||||
document.getElementById("cnt-free").textContent=cntF.toLocaleString();
|
||||
if(lastPg>=0){const inf=DBI_IDX[lastDbi];const nm=inf?inf.name:"dbi"+lastDbi;const opN=lastOp===1?"R":lastOp===2?"W":"F";document.getElementById("evt-latest").textContent=opN+" pg"+lastPg.toLocaleString()+" "+nm}
|
||||
const el=now-lRT;if(el>=1000){document.getElementById("evt-rate").textContent=Math.round(rEv/(el/1000)).toLocaleString();rEv=0;lRT=now}
|
||||
if(cnt>0&&!replayMode)kick();
|
||||
};
|
||||
}
|
||||
function szFlame(){
|
||||
const fw=document.getElementById("flame-wrap"),fc=document.getElementById("flame");
|
||||
if(!fw||!fc)return;fc.width=fw.clientWidth||300;fc.height=fw.clientHeight||100;
|
||||
}
|
||||
function drawFlame(){
|
||||
const fc=document.getElementById("flame");if(!fc)return;
|
||||
const ctx=fc.getContext("2d"),cw=fc.width,ch=fc.height;ctx.clearRect(0,0,cw,ch);
|
||||
const at=DBIS.filter(d=>d.pages>0);if(!at.length)return;
|
||||
const lw=42,rw=cw-lw,rowH=Math.max(4,Math.floor((ch-14)/Math.max(1,at.length)));
|
||||
ctx.font="7px monospace";
|
||||
for(let r=0;r<at.length;r++){
|
||||
const d=at[r],fd=flameData[d.dbi_index];if(!fd)continue;
|
||||
const c=PAL[d.dbi_index]||PAL[255],y=r*rowH;
|
||||
let mx=1;for(let s=0;s<FLAME_SECS;s++){const v=fd[s];if(v>mx)mx=v}
|
||||
for(let s=0;s<FLAME_SECS;s++){
|
||||
const idx=((flameHead-FLAME_SECS+s)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
|
||||
const ops=fd[idx];if(ops<=0)continue;
|
||||
const x=lw+(s/FLAME_SECS)*rw,w=Math.ceil(rw/FLAME_SECS);
|
||||
const a=Math.min(1,Math.log2(ops+1)/Math.log2(mx+1));
|
||||
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},${a.toFixed(2)})`;
|
||||
ctx.fillRect(x,y,w,rowH-1);
|
||||
}
|
||||
ctx.fillStyle="#8b949e";ctx.fillText(d.name.slice(0,6),2,y+rowH-2);
|
||||
}
|
||||
const ay=at.length*rowH+2;ctx.fillStyle="#30363d";ctx.fillRect(lw,ay,rw,1);
|
||||
ctx.fillStyle="#8b949e";ctx.font="7px monospace";
|
||||
for(let s=30;s<FLAME_SECS;s+=30){
|
||||
const x=lw+((FLAME_SECS-s)/FLAME_SECS)*rw;
|
||||
ctx.fillText("-"+s+"s",x,ay+10);
|
||||
}
|
||||
}
|
||||
function buildUnifiedPanel(){
|
||||
const el=document.getElementById("unified-tables");if(!el)return;el.innerHTML="";
|
||||
const hdr=document.createElement("div");hdr.className="utbl-row";hdr.style.cssText="cursor:default;border-bottom:1px solid #30363d;margin-bottom:2px;padding-bottom:4px";
|
||||
hdr.innerHTML='<div class="utbl-sw" style="visibility:hidden"></div><div class="utbl-name" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Table</div><div class="utbl-spark" style="width:80px;text-align:center;color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Activity</div><div class="utbl-cache" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px;text-align:center">Cache</div><div class="utbl-diff" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Diff</div><div class="utbl-faults" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px" title="Page faults over last 3 residency updates">Fault<span style="font-size:7px;text-transform:none"> (3)</span></div>';
|
||||
el.appendChild(hdr);
|
||||
DBIS.forEach(d=>{
|
||||
if(d.pages<=0&&d.dbi_index>2)return;
|
||||
const c=PAL[d.dbi_index]||PAL[255];
|
||||
const row=document.createElement("div");row.className="utbl-row";
|
||||
row.dataset.dbiIndex=d.dbi_index;
|
||||
const sw=document.createElement("div");sw.className="utbl-sw";
|
||||
sw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
const nm=document.createElement("div");nm.className="utbl-name";nm.textContent=d.name;
|
||||
const cv=document.createElement("canvas");cv.className="utbl-spark";cv.width=80;cv.height=16;
|
||||
cv.style.cssText="width:80px;height:16px;border:1px solid #21262d;border-radius:2px;background:#0d1117";
|
||||
const cp=document.createElement("div");cp.className="utbl-cache";
|
||||
const cpBar=document.createElement("div");cpBar.className="utbl-cache-bar-wrap";
|
||||
const cpFill=document.createElement("div");cpFill.className="utbl-cache-fill";cpFill.style.width="0%";cpFill.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
cpBar.appendChild(cpFill);
|
||||
const cpPct=document.createElement("div");cpPct.className="utbl-cache-pct";
|
||||
cp.append(cpBar,cpPct);
|
||||
const cd=document.createElement("div");cd.className="utbl-diff";
|
||||
const ft=document.createElement("div");ft.className="utbl-faults";
|
||||
row.append(sw,nm,cv,cp,cd,ft);
|
||||
row.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
el.appendChild(row);
|
||||
d._sparkCv=cv;d._sparkCtx=cv.getContext("2d");
|
||||
d._cacheEl=cp;d._cacheFill=cpFill;d._cachePct=cpPct;d._diffEl=cd;d._faultEl=ft;
|
||||
});
|
||||
}
|
||||
function drawSparkline(d){
|
||||
const cv=d._sparkCv,ctx=d._sparkCtx;if(!cv||!ctx)return;
|
||||
const sd=sparkData[d.dbi_index];if(!sd)return;
|
||||
const w=cv.width,h=cv.height,c=PAL[d.dbi_index]||PAL[255];
|
||||
ctx.clearRect(0,0,w,h);
|
||||
let mx=1;for(let i=0;i<SPARK_SECS;i++){const v=sd.ring[i];if(v>mx)mx=v}
|
||||
const latest=sd.ring[((sd.head-1)%SPARK_SECS+SPARK_SECS)%SPARK_SECS];
|
||||
ctx.beginPath();ctx.moveTo(0,h);
|
||||
for(let i=0;i<SPARK_SECS;i++){
|
||||
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
|
||||
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
|
||||
if(i===0)ctx.lineTo(x,y);else ctx.lineTo(x,y);
|
||||
}
|
||||
ctx.lineTo(w,h);ctx.closePath();
|
||||
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},0.3)`;ctx.fill();
|
||||
ctx.beginPath();
|
||||
for(let i=0;i<SPARK_SECS;i++){
|
||||
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
|
||||
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
|
||||
if(i===0)ctx.moveTo(x,y);else ctx.lineTo(x,y);
|
||||
}
|
||||
ctx.strokeStyle=`rgba(${c[0]},${c[1]},${c[2]},0.8)`;ctx.lineWidth=1;ctx.stroke();
|
||||
if(d._sparkVal)d._sparkVal.textContent=latest.toLocaleString();
|
||||
}
|
||||
function buildTreeInfo(info){
|
||||
const el=document.getElementById("tree-info");if(!el)return;el.innerHTML="";
|
||||
info.forEach(ti=>{
|
||||
if(ti.height===0&&ti.branch_pages===0&&ti.leaf_pages===0&&ti.large_pages===0)return;
|
||||
const card=document.createElement("div");card.className="tree-card";
|
||||
const total=ti.branch_pages+ti.leaf_pages+ti.large_pages;
|
||||
const ipl=ti.leaf_pages>0?Math.round(ti.items/ti.leaf_pages):0;
|
||||
let html=`<div class="tree-name">${ti.name} <span style="color:#8b949e;font-weight:400">depth ${ti.height}</span></div>`;
|
||||
html+=`<div class="tree-stat">B:${ti.branch_pages.toLocaleString()} L:${ti.leaf_pages.toLocaleString()} O:${ti.large_pages.toLocaleString()} items:${ti.items.toLocaleString()}</div>`;
|
||||
if(total>0){
|
||||
const bp=Math.max(1,Math.round(ti.branch_pages/total*100)),lp=Math.max(1,Math.round(ti.leaf_pages/total*100)),op=Math.max(1,100-bp-lp);
|
||||
html+=`<div class="tree-bar"><div class="tree-seg" style="width:${bp}%;background:#3c82f6"></div><div class="tree-seg" style="width:${lp}%;background:#4ade80"></div><div class="tree-seg" style="width:${op}%;background:#f58231"></div></div>`;
|
||||
}
|
||||
if(ipl>0)html+=`<div class="tree-stat">~${ipl.toLocaleString()} items/leaf</div>`;
|
||||
card.innerHTML=html;
|
||||
card.addEventListener("click",()=>{const i=ti.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
el.appendChild(card);
|
||||
});
|
||||
}
|
||||
function fmtGas(g){if(g>=1e6)return(g/1e6).toFixed(1)+"M";if(g>=1e3)return(g/1e3).toFixed(0)+"K";return g.toFixed(0)}
|
||||
function updateBlockPanel(){
|
||||
blockPanelDirty=false;
|
||||
const el=document.getElementById("block-panel");if(!el)return;
|
||||
el.innerHTML="";
|
||||
if(blockRecording){
|
||||
const c=document.createElement("div");c.className="block-card recording";
|
||||
const elapsed=performance.now()-blockRecording.startTime;
|
||||
const elStr=elapsed<1000?elapsed.toFixed(0)+"ms":(elapsed/1000).toFixed(1)+"s";
|
||||
c.innerHTML=`<div class="block-num">⏱ #${blockRecording.number.toLocaleString()} recording\u2026</div><div class="block-stat">${blockRecording.events.length.toLocaleString()} events ${elStr}</div>`;
|
||||
el.appendChild(c);
|
||||
}
|
||||
for(let i=blockHistory.length-1;i>=0;i--){
|
||||
const b=blockHistory[i],isReplaying=replayMode&&replayBlockRef===b;
|
||||
const c=document.createElement("div");c.className="block-card"+(isReplaying?" replaying":"");
|
||||
const total=b.reads+b.writes+b.frees;
|
||||
const dur=b.duration>0?(b.duration<1000?b.duration.toFixed(0)+"ms":(b.duration/1000).toFixed(1)+"s"):"";
|
||||
const gas=b.gasUsed>0?fmtGas(b.gasUsed)+"gas":"";
|
||||
c.innerHTML=`<div class="block-num">▶ #${b.number.toLocaleString()} ${b.txCount}tx${dur?" "+dur:""}${gas?" "+gas:""} ${total.toLocaleString()} ops</div><div class="block-stat"><span style="color:#3c82f6">R:${b.reads.toLocaleString()}</span> <span style="color:#ef4444">W:${b.writes.toLocaleString()}</span> <span style="color:#eab308">F:${b.frees.toLocaleString()}</span>${b.faults?` <span style="color:#d946ef">${b.faults}f</span>`:""} ${(b.uniquePages||0).toLocaleString()} pages</div>`;
|
||||
c.addEventListener("click",()=>startReplay(b));
|
||||
el.appendChild(c);
|
||||
}
|
||||
}
|
||||
function snapPanelState(){
|
||||
return{dbiPages:DBIS.map(d=>d.pages),cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,cachePrevPer:cachePrevPer?cachePrevPer.slice():null,cacheTotalCached,cachePrevTotal,unrefCount,faultWindow:faultWindow.map(w=>({...w})),faultCurrent:{...faultCurrent}};
|
||||
}
|
||||
function applyPanelSnap(snap){
|
||||
DBIS.forEach((d,i)=>{d.pages=snap.dbiPages[i]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
if(snap.cacheCachedPer&&cacheCachedPer){for(let i=0;i<cacheCachedPer.length;i++)cacheCachedPer[i]=snap.cacheCachedPer[i]||0}
|
||||
cachePrevPer=snap.cachePrevPer?snap.cachePrevPer.slice():null;
|
||||
cacheTotalCached=snap.cacheTotalCached;cachePrevTotal=snap.cachePrevTotal;
|
||||
unrefCount=snap.unrefCount;
|
||||
faultWindow=snap.faultWindow.map(w=>({...w}));faultCurrent={...snap.faultCurrent};
|
||||
cacheStale=true;bldTbl();updateCacheStats();
|
||||
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
}
|
||||
function startReplay(block){
|
||||
if(replayMode&&replayBlockRef===block&&replayTimer)return;
|
||||
if(!liveSnapshot)liveSnapshot=snapPanelState();
|
||||
replayMode=true;replayBlockRef=block;
|
||||
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
|
||||
lOp.fill(0);activeSet.clear();activePgs=0;
|
||||
if(hmCounts)hmCounts.fill(0);
|
||||
trailStart=0;trailN=0;
|
||||
for(const pgStr in block.before){
|
||||
const pg=parseInt(pgStr,10);if(pg>=NP)continue;
|
||||
const snap=block.before[pg];
|
||||
oMap[pg]=snap.om;
|
||||
if(resMap&&pg<resMap.length)resMap[pg]=snap.rm;
|
||||
}
|
||||
if(block.panelSnapshot)applyPanelSnap(block.panelSnapshot);
|
||||
markZbDirty();sched();
|
||||
document.getElementById("block-live").style.color="#8b949e";
|
||||
updateBlockPanel();
|
||||
const events=block.events;
|
||||
const batchSize=Math.max(1,Math.ceil(events.length/60));
|
||||
let idx=0;
|
||||
function step(){
|
||||
if(!replayMode){replayTimer=null;return}
|
||||
const end=Math.min(idx+batchSize,events.length);
|
||||
const now=performance.now();
|
||||
for(let i=idx;i<end;i++){
|
||||
const e=events[i];
|
||||
if(e.pg>=NP)continue;
|
||||
if(hmCounts)hmCounts[e.pg]++;
|
||||
if(cacheMode&&resMap&&e.op===1&&e.pg<resMap.length&&!resMap[e.pg]){faultCounts[e.dbi]=(faultCounts[e.dbi]||0)+1;faultCurrent[e.dbi]=(faultCurrent[e.dbi]||0)+1}
|
||||
if(resMap&&e.pg<resMap.length&&e.op!==3)resMap[e.pg]=1;
|
||||
lOp[e.pg]=e.op;lTs[e.pg]=now;
|
||||
if(!activeSet.has(e.pg)){activeSet.add(e.pg);activePgs++}
|
||||
if(e.op===2&&e.dbi>0&&e.dbi<0xFE)oMap[e.pg]=e.dbi;
|
||||
if(showTrails&&i>0&&!((e.op===1&&!sR)||(e.op===2&&!sW)||(e.op===3&&!sF))){const p=events[i-1];if(p.pg!==e.pg)trailPush(p.pg,e.pg,e.op,now)}
|
||||
}
|
||||
idx=end;
|
||||
markZbDirty();kick();
|
||||
if(idx<events.length){replayTimer=requestAnimationFrame(step)}
|
||||
else{
|
||||
replayTimer=null;
|
||||
const pin=performance.now()+FADE*10;
|
||||
for(const pg of activeSet)lTs[pg]=pin;
|
||||
markZbDirty();sched();
|
||||
updateBlockPanel();
|
||||
}
|
||||
}
|
||||
replayTimer=requestAnimationFrame(step);
|
||||
}
|
||||
function goLive(){
|
||||
replayMode=false;replayBlockRef=null;
|
||||
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
|
||||
lOp.fill(0);activeSet.clear();activePgs=0;
|
||||
trailStart=0;trailN=0;
|
||||
if(liveSnapshot){applyPanelSnap(liveSnapshot);liveSnapshot=null}
|
||||
oMapDirty=true;cacheStale=true;
|
||||
markZbDirty();sched();
|
||||
document.getElementById("block-live").style.color="#4ade80";
|
||||
updateBlockPanel();updateCacheStats();
|
||||
}
|
||||
window._goLive=goLive;
|
||||
function connectResidencyWS(){
|
||||
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws_residency`);
|
||||
ws.binaryType="arraybuffer";
|
||||
ws.onclose=()=>setTimeout(connectResidencyWS,3000);
|
||||
ws.onerror=()=>ws.close();
|
||||
ws.onmessage=msg=>{
|
||||
if(!msg.data||msg.data.byteLength<1)return;
|
||||
if(replayMode)return;
|
||||
const v=new DataView(msg.data);
|
||||
const msgType=v.getUint8(0);
|
||||
if(msgType===0){
|
||||
const src=new Uint8Array(msg.data,1);
|
||||
const newLen=src.length;
|
||||
if(!resMap||resMap.length!==newLen){
|
||||
const oldMap=resMap;const oldLen=oldMap?oldMap.length:0;
|
||||
resMap=new Uint8Array(newLen);
|
||||
if(oldMap){
|
||||
const shared=Math.min(oldLen,newLen,NP);
|
||||
for(let i=0;i<shared;i++){
|
||||
const ns=src[i],os=oldMap[i];
|
||||
if(ns!==os){
|
||||
const dbi=oMap?oMap[i]:0;
|
||||
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[i]=ns;
|
||||
}
|
||||
for(let i=shared;i<newLen;i++){
|
||||
resMap[i]=src[i];
|
||||
if(src[i]){cacheTotalCached++;if(cacheCachedPer&&oMap){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
|
||||
}
|
||||
}else{
|
||||
resMap.set(src);
|
||||
cacheTotalCached=0;if(cacheCachedPer)cacheCachedPer.fill(0);
|
||||
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
|
||||
for(let i=0;i<lim;i++){
|
||||
if(src[i]){cacheTotalCached++;if(cacheCachedPer){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
|
||||
for(let i=0;i<lim;i++){
|
||||
const ns=src[i],os=resMap[i];
|
||||
if(ns!==os){
|
||||
const dbi=oMap?oMap[i]:0;
|
||||
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[i]=ns;
|
||||
}
|
||||
}
|
||||
cacheStale=true;
|
||||
cacheLastUpdate=performance.now();
|
||||
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
|
||||
if(cacheMode)markZbDirty();
|
||||
updateCacheStats();
|
||||
if(cacheMode)sched();
|
||||
}else if(msgType===1){
|
||||
if(!resMap)return;
|
||||
const spanCount=v.getUint32(1,true);
|
||||
let off=5;
|
||||
for(let s=0;s<spanCount;s++){
|
||||
const start=v.getUint32(off,true);off+=4;
|
||||
const len=v.getUint32(off,true);off+=4;
|
||||
const state=v.getUint8(off);off+=1;
|
||||
for(let p=start;p<start+len&&p<resMap.length;p++){
|
||||
const oldState=resMap[p];
|
||||
if(oldState!==state){
|
||||
const dbi=oMap?oMap[p]:0;
|
||||
if(state&&!oldState){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!state&&oldState){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[p]=state;
|
||||
}
|
||||
}
|
||||
cacheStale=true;
|
||||
cacheLastUpdate=performance.now();
|
||||
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
|
||||
if(cacheMode)markZbDirty();
|
||||
updateCacheStats();
|
||||
if(cacheMode)sched();
|
||||
}
|
||||
};
|
||||
}
|
||||
let cacheStatsPending=false;
|
||||
function updateCacheStats(){
|
||||
if(!cacheStale||!cacheCachedPer)return;
|
||||
if(cacheStatsPending)return;
|
||||
cacheStatsPending=true;
|
||||
requestAnimationFrame(()=>{
|
||||
cacheStatsPending=false;
|
||||
if(!cacheStale||!cacheCachedPer)return;
|
||||
cacheStale=false;
|
||||
const limit=Math.min(resMap?resMap.length:0,NP);
|
||||
const totalPct=limit>0?(cacheTotalCached/limit*100).toFixed(1):"0";
|
||||
const prevTotalPct=limit>0?(cachePrevTotal/limit*100):0;
|
||||
const totalDiff=limit>0?(cacheTotalCached/limit*100-prevTotalPct):0;
|
||||
const tdS=cachePrevPer?(totalDiff>=0?"+":"")+totalDiff.toFixed(2)+"%":"";
|
||||
const tdC=totalDiff>0?"#4ade80":totalDiff<0?"#f87171":"#8b949e";
|
||||
document.getElementById("cache-total").innerHTML=`${cacheTotalCached.toLocaleString()}/${limit.toLocaleString()} (${totalPct}%) ${cachePrevPer?`<span style="color:${tdC}">${tdS}</span>`:""}`;
|
||||
DBIS.forEach(d=>{
|
||||
if(!d._cacheEl)return;
|
||||
const total=d.pages;if(total<=0){if(d._cachePct)d._cachePct.textContent="-";if(d._cacheFill)d._cacheFill.style.width="0%";d._diffEl.textContent="";d._faultEl.textContent="";return}
|
||||
const cached=cacheCachedPer[d.dbi_index]||0;
|
||||
const pct=total>0?(cached/total*100):0;
|
||||
if(d._cacheFill)d._cacheFill.style.width=pct.toFixed(1)+"%";
|
||||
if(d._cachePct){d._cachePct.textContent=pct.toFixed(1)+"%";d._cachePct.style.color=pct>50?"#4ade80":"#f87171"}
|
||||
const prevCached=cachePrevPer?cachePrevPer[d.dbi_index]||0:0;
|
||||
const prevPct=total>0?(prevCached/total*100):0;
|
||||
const diff=cachePrevPer?(pct-prevPct):0;
|
||||
if(cachePrevPer){
|
||||
d._diffEl.textContent=(diff>=0?"+":"")+diff.toFixed(2)+"%";
|
||||
d._diffEl.style.color=diff>0?"#4ade80":diff<0?"#f87171":"#8b949e";
|
||||
}
|
||||
const fc=rollingFaults(d.dbi_index);
|
||||
d._faultEl.textContent=fc>0?fc+"f":"";
|
||||
});
|
||||
cachePrevPer=cacheCachedPer.slice();
|
||||
cachePrevTotal=cacheTotalCached;
|
||||
});
|
||||
}
|
||||
async function init(){
|
||||
try{
|
||||
const r=await fetch("/api/info"),info=await r.json();NP=info.page_count;PS=info.page_size;
|
||||
DBIS=info.dbi_names.map((n,i)=>({name:n,dbi_index:i,pages:0,pct:0,size_mb:0}));
|
||||
DBI_IDX={};DBIS.forEach(d=>{DBI_IDX[d.dbi_index]=d});
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 loading owner map\u2026`;
|
||||
}catch(e){document.getElementById("subtitle").textContent="Failed: "+e.message}
|
||||
try{
|
||||
const lt=document.getElementById("load-text"),lbw=document.getElementById("load-bar-wrap"),lb=document.getElementById("load-bar");
|
||||
lt.textContent="Downloading owner map\u2026";lbw.style.display="block";
|
||||
const r=await fetch("/api/owner_map");
|
||||
if(r.ok){
|
||||
const total=parseInt(r.headers.get("content-length")||"0",10)||NP;
|
||||
const reader=r.body.getReader();
|
||||
const buf=new Uint8Array(total||NP);let pos=0;
|
||||
for(;;){const{done,value}=await reader.read();if(done)break;buf.set(value,pos);pos+=value.length;
|
||||
const pct=total>0?Math.min(100,pos/total*100):0;
|
||||
lb.style.width=pct.toFixed(1)+"%";
|
||||
lt.textContent=`Downloading owner map\u2026 ${(pos/(1024*1024)).toFixed(1)}/${(total/(1024*1024)).toFixed(1)} MB`;
|
||||
}
|
||||
oMap=buf.length===pos?buf:buf.slice(0,pos);
|
||||
lt.textContent="Processing owner map\u2026";lb.style.width="100%";
|
||||
await new Promise(r=>setTimeout(r,0));
|
||||
const ct={};for(let i=0;i<NP;i++){const o=oMap[i];ct[o]=(ct[o]||0)+1}
|
||||
DBIS.forEach(d=>{d.pages=ct[d.dbi_index]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
}else{oMap=new Uint8Array(NP);oMap.fill(0xFF)}
|
||||
}catch(e){oMap=new Uint8Array(NP);oMap.fill(0xFF)}
|
||||
unrefCount=oMap.reduce((n,b)=>n+(b===0xFF?1:0),0);const ref=NP-unrefCount;
|
||||
const gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${ref.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
lOp=new Uint8Array(NP);lTs=new Float64Array(NP);hmCounts=new Uint32Array(NP);
|
||||
cacheCachedPer=new Array(DBIS.length).fill(0);
|
||||
DBIS.forEach(d=>{sparkData[d.dbi_index]={ring:new Array(SPARK_SECS).fill(0),head:0};sparkAccum[d.dbi_index]=0;flameData[d.dbi_index]=new Array(FLAME_SECS).fill(0)});
|
||||
bldTbl();buildUnifiedPanel();
|
||||
let treeInfo=[];
|
||||
try{const r=await fetch("/api/tree_info");if(r.ok)treeInfo=await r.json()}catch(e){}
|
||||
if(treeInfo.length>0)buildTreeInfo(treeInfo);
|
||||
ldEl.style.display="none";cv.style.display="block";boot();connectWS();connectResidencyWS();
|
||||
const flCv=document.getElementById("flame");
|
||||
flCv.addEventListener("mousemove",e=>{
|
||||
if(!showFlame)return;
|
||||
const at=DBIS.filter(d=>d.pages>0);if(!at.length){tt.style.display="none";return}
|
||||
const rect=flCv.getBoundingClientRect();
|
||||
const mx=e.clientX-rect.left,my=e.clientY-rect.top;
|
||||
const lw=42,rw=flCv.width-lw;
|
||||
const rowH=Math.max(4,Math.floor((flCv.height-14)/Math.max(1,at.length)));
|
||||
const ri=Math.floor(my/rowH);
|
||||
if(ri<0||ri>=at.length||mx<lw){tt.style.display="none";return}
|
||||
const d=at[ri],c=PAL[d.dbi_index]||PAL[255];
|
||||
const secIdx=Math.floor((mx-lw)/rw*FLAME_SECS);
|
||||
const idx=((flameHead-FLAME_SECS+secIdx)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
|
||||
const fd=flameData[d.dbi_index];
|
||||
const ops=fd?fd[idx]:0;
|
||||
const ago=FLAME_SECS-secIdx;
|
||||
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
ttNm.textContent=d.name;
|
||||
ttDt.innerHTML=`${ops.toLocaleString()} ops/s \u00b7 ${ago}s ago`;
|
||||
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
|
||||
});
|
||||
flCv.addEventListener("mouseleave",()=>{tt.style.display="none"});
|
||||
setInterval(()=>{
|
||||
DBIS.forEach(d=>{
|
||||
const v=sparkAccum[d.dbi_index]||0;
|
||||
const sd=sparkData[d.dbi_index];if(sd){sd.ring[sd.head%SPARK_SECS]=v;sd.head++}
|
||||
const fd=flameData[d.dbi_index];if(fd)fd[flameHead%FLAME_SECS]=v;
|
||||
sparkAccum[d.dbi_index]=0;
|
||||
});
|
||||
flameHead++;
|
||||
DBIS.forEach(d=>drawSparkline(d));
|
||||
if(showFlame)drawFlame();
|
||||
},1000);
|
||||
setInterval(()=>{if(!oMap||!oMapDirty||replayMode)return;oMapDirty=false;
|
||||
DBIS.forEach(d=>{d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
bldTbl();uAR();markZbDirty();sched();
|
||||
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
},2000);
|
||||
setInterval(()=>{
|
||||
if(!cacheLastUpdate)return;
|
||||
const ago=Math.round((performance.now()-cacheLastUpdate)/1000);
|
||||
let txt;
|
||||
if(ago<60)txt=ago+"s ago";
|
||||
else if(ago<3600)txt=Math.floor(ago/60)+"m "+ago%60+"s ago";
|
||||
else txt=Math.floor(ago/3600)+"h ago";
|
||||
document.getElementById("cache-ago").textContent=txt;
|
||||
},1000);
|
||||
setInterval(()=>{if(blockPanelDirty)updateBlockPanel()},250);
|
||||
setInterval(()=>{
|
||||
if(activeSet.size<5000)return;
|
||||
const now=performance.now();
|
||||
for(const pg of activeSet){
|
||||
if(!lOp[pg]||now-lTs[pg]>=FADE){activeSet.delete(pg);if(lOp[pg]){lOp[pg]=0;activePgs=Math.max(0,activePgs-1)}}
|
||||
}
|
||||
},5000);
|
||||
}
|
||||
init();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
577
crates/storage/mdbx-viz/src/lib.rs
Normal file
577
crates/storage/mdbx-viz/src/lib.rs
Normal file
@@ -0,0 +1,577 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
mpsc, Arc, RwLock,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
State,
|
||||
},
|
||||
http::{header, StatusCode},
|
||||
response::{Html, IntoResponse},
|
||||
routing::get,
|
||||
Router,
|
||||
};
|
||||
use reth_libmdbx::pageviz::{PageEvent, PageOp};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
pub mod walker;
|
||||
|
||||
unsafe extern "C" {
|
||||
fn mdbx_pageviz_emit_block_marker(op: u8, block_number: u32, tx_count: u16, duration_encoded: u8, gas_encoded: u8);
|
||||
}
|
||||
|
||||
pub fn pageviz_emit_block_marker(block_number: u64, start: bool, tx_count: u16, duration_ms: u32, gas_used: u64) {
|
||||
let dur_enc = if duration_ms == 0 { 0u8 } else { ((duration_ms as f64).log2() * 20.0).round().min(255.0) as u8 };
|
||||
let gas_enc = if gas_used == 0 { 0u8 } else { ((gas_used as f64).log2() * 10.0).round().min(255.0) as u8 };
|
||||
unsafe {
|
||||
let op = if start { 4u8 } else { 5u8 };
|
||||
mdbx_pageviz_emit_block_marker(op, block_number as u32, tx_count, dur_enc, gas_enc);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Serialize)]
|
||||
struct ResidencyStats {
|
||||
total_pages: u64,
|
||||
cached_pages: u64,
|
||||
pct: f64,
|
||||
per_table: Vec<TableResidency>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Serialize)]
|
||||
struct TableResidency {
|
||||
dbi: usize,
|
||||
name: String,
|
||||
total: u64,
|
||||
cached: u64,
|
||||
pct: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState {
|
||||
info: VizInfo,
|
||||
owner_map: Arc<RwLock<Vec<u8>>>,
|
||||
tree_info: Arc<Vec<walker::TreeInfo>>,
|
||||
subscribers: Arc<AtomicUsize>,
|
||||
event_tx: broadcast::Sender<Vec<u8>>,
|
||||
residency: Arc<RwLock<Vec<u8>>>,
|
||||
residency_stats: Arc<RwLock<ResidencyStats>>,
|
||||
residency_tx: broadcast::Sender<Vec<u8>>,
|
||||
residency_subscribers: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct VizInfo {
|
||||
pub page_count: u64,
|
||||
pub page_size: u32,
|
||||
pub dbi_names: Vec<String>,
|
||||
}
|
||||
|
||||
pub struct VizConfig {
|
||||
pub port: u16,
|
||||
pub page_count: u64,
|
||||
pub page_size: u32,
|
||||
pub dbi_names: Vec<String>,
|
||||
pub owner_map: Vec<u8>,
|
||||
pub tree_info: Vec<walker::TreeInfo>,
|
||||
pub db_path: PathBuf,
|
||||
}
|
||||
|
||||
const WIRE_EVENT_SIZE: usize = 8;
|
||||
|
||||
fn encode_events(events: &[PageEvent]) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(events.len() * WIRE_EVENT_SIZE);
|
||||
for ev in events {
|
||||
buf.extend_from_slice(&ev.pgno.to_le_bytes());
|
||||
if ev.op.is_block_marker() {
|
||||
let tx_lo = (ev.dbi & 0xFF) as u8;
|
||||
let gas = ((ev.dbi >> 24) & 0xFF) as u8;
|
||||
buf.extend_from_slice(&u16::to_le_bytes((gas as u16) << 8 | tx_lo as u16));
|
||||
buf.push(ev.op as u8);
|
||||
buf.push(((ev.dbi >> 16) & 0xFF) as u8);
|
||||
} else {
|
||||
buf.extend_from_slice(&(ev.dbi as u16).to_le_bytes());
|
||||
buf.push(ev.op as u8);
|
||||
buf.push(0u8);
|
||||
}
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
struct ReadOnlyMapping {
|
||||
base: *mut libc::c_void,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl ReadOnlyMapping {
|
||||
fn new(path: &std::path::Path) -> Option<Self> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let len = file.metadata().ok()?.len() as usize;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
let base = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if base == libc::MAP_FAILED {
|
||||
return None;
|
||||
}
|
||||
Some(Self { base, len })
|
||||
}
|
||||
|
||||
fn remap(&mut self, path: &std::path::Path) -> bool {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
let file = match std::fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(_) => return false,
|
||||
};
|
||||
let new_len = match file.metadata() {
|
||||
Ok(m) => m.len() as usize,
|
||||
Err(_) => return false,
|
||||
};
|
||||
if new_len == 0 {
|
||||
return false;
|
||||
}
|
||||
unsafe {
|
||||
libc::munmap(self.base, self.len);
|
||||
}
|
||||
let base = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
new_len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if base == libc::MAP_FAILED {
|
||||
return false;
|
||||
}
|
||||
self.base = base;
|
||||
self.len = new_len;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReadOnlyMapping {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
libc::munmap(self.base, self.len);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for ReadOnlyMapping {}
|
||||
|
||||
pub async fn start_viz_server(
|
||||
config: VizConfig,
|
||||
rx: mpsc::Receiver<Vec<PageEvent>>,
|
||||
) -> std::io::Result<()> {
|
||||
let (event_tx, _) = broadcast::channel::<Vec<u8>>(256);
|
||||
let (residency_tx, _) = broadcast::channel::<Vec<u8>>(64);
|
||||
|
||||
let owner_map = Arc::new(RwLock::new(config.owner_map));
|
||||
let subscribers = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let state = AppState {
|
||||
info: VizInfo {
|
||||
page_count: config.page_count,
|
||||
page_size: config.page_size,
|
||||
dbi_names: config.dbi_names,
|
||||
},
|
||||
owner_map: owner_map.clone(),
|
||||
tree_info: Arc::new(config.tree_info),
|
||||
subscribers: subscribers.clone(),
|
||||
event_tx: event_tx.clone(),
|
||||
residency: Arc::new(RwLock::new(Vec::new())),
|
||||
residency_stats: Arc::new(RwLock::new(ResidencyStats::default())),
|
||||
residency_tx: residency_tx.clone(),
|
||||
residency_subscribers: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
|
||||
let bridge_tx = event_tx.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("viz-bridge".into())
|
||||
.spawn(move || {
|
||||
while let Ok(events) = rx.recv() {
|
||||
if events.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
{
|
||||
let mut map = owner_map.write().unwrap();
|
||||
for ev in &events {
|
||||
let pg = ev.pgno as usize;
|
||||
if ev.op == PageOp::Write && ev.dbi > 0 && ev.dbi < 0xFE && pg < map.len()
|
||||
{
|
||||
map[pg] = ev.dbi as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if subscribers.load(Ordering::Relaxed) > 0 {
|
||||
let encoded = encode_events(&events);
|
||||
let _ = bridge_tx.send(encoded);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn viz-bridge thread");
|
||||
|
||||
let res_map = state.residency.clone();
|
||||
let res_stats = state.residency_stats.clone();
|
||||
let res_tx = state.residency_tx.clone();
|
||||
let res_subs = state.residency_subscribers.clone();
|
||||
let own_map = state.owner_map.clone();
|
||||
let info_clone = state.info.clone();
|
||||
let db_path = config.db_path;
|
||||
let mdbx_ps = config.page_size;
|
||||
let sys_ps = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u32;
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("residency-poller".into())
|
||||
.spawn(move || {
|
||||
let mut mapping: Option<ReadOnlyMapping> = None;
|
||||
let mut prev: Vec<u8> = Vec::new();
|
||||
|
||||
if db_path.as_os_str().is_empty() {
|
||||
tracing::warn!("residency-poller: no db_path configured, residency polling disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
|
||||
if res_subs.load(Ordering::Relaxed) == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if mapping.is_none() {
|
||||
match ReadOnlyMapping::new(&db_path) {
|
||||
Some(m) => {
|
||||
tracing::info!(
|
||||
"residency-poller: mapped {} ({} bytes, mdbx_ps={}, sys_ps={})",
|
||||
db_path.display(),
|
||||
m.len,
|
||||
mdbx_ps,
|
||||
sys_ps,
|
||||
);
|
||||
mapping = Some(m);
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"residency-poller: failed to mmap {}",
|
||||
db_path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let file_len = match std::fs::metadata(&db_path) {
|
||||
Ok(m) => m.len() as usize,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let m = mapping.as_mut().unwrap();
|
||||
if file_len > m.len {
|
||||
tracing::info!(
|
||||
"residency-poller: file grew {} -> {}, remapping",
|
||||
m.len,
|
||||
file_len,
|
||||
);
|
||||
if !m.remap(&db_path) {
|
||||
tracing::warn!("residency-poller: remap failed");
|
||||
mapping = None;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let m = mapping.as_ref().unwrap();
|
||||
let base = m.base;
|
||||
let len = m.len;
|
||||
|
||||
if mdbx_ps == 0 || sys_ps == 0 || len == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let sys_pages = (len + sys_ps as usize - 1) / sys_ps as usize;
|
||||
let mdbx_page_count = len / mdbx_ps as usize;
|
||||
|
||||
let mut mincore_vec = vec![0u8; sys_pages];
|
||||
let ret = unsafe { libc::mincore(base, len, mincore_vec.as_mut_ptr()) };
|
||||
if ret != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut cur = vec![0u8; mdbx_page_count];
|
||||
if mdbx_ps == sys_ps {
|
||||
for i in 0..mdbx_page_count.min(sys_pages) {
|
||||
cur[i] = mincore_vec[i] & 1;
|
||||
}
|
||||
} else if mdbx_ps > sys_ps {
|
||||
let ratio = mdbx_ps as usize / sys_ps as usize;
|
||||
for i in 0..mdbx_page_count {
|
||||
let base_idx = i * ratio;
|
||||
let mut all_resident = true;
|
||||
for j in 0..ratio {
|
||||
if base_idx + j >= sys_pages
|
||||
|| (mincore_vec[base_idx + j] & 1) == 0
|
||||
{
|
||||
all_resident = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
cur[i] = if all_resident { 1 } else { 0 };
|
||||
}
|
||||
} else {
|
||||
let ratio = sys_ps as usize / mdbx_ps as usize;
|
||||
for i in 0..mdbx_page_count {
|
||||
let sys_idx = i / ratio;
|
||||
cur[i] = if sys_idx < sys_pages {
|
||||
mincore_vec[sys_idx] & 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut rm = res_map.write().unwrap();
|
||||
rm.clear();
|
||||
rm.extend_from_slice(&cur);
|
||||
}
|
||||
|
||||
{
|
||||
let omap = own_map.read().unwrap();
|
||||
let max_dbi = info_clone.dbi_names.len();
|
||||
let mut total_per = vec![0u64; max_dbi];
|
||||
let mut cached_per = vec![0u64; max_dbi];
|
||||
let mut total_cached = 0u64;
|
||||
let limit = mdbx_page_count.min(omap.len());
|
||||
|
||||
for i in 0..limit {
|
||||
let dbi = omap[i] as usize;
|
||||
if dbi < max_dbi {
|
||||
total_per[dbi] += 1;
|
||||
if cur[i] == 1 {
|
||||
cached_per[dbi] += 1;
|
||||
total_cached += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut stats = res_stats.write().unwrap();
|
||||
stats.total_pages = mdbx_page_count as u64;
|
||||
stats.cached_pages = total_cached;
|
||||
stats.pct = if mdbx_page_count > 0 {
|
||||
(total_cached as f64 / mdbx_page_count as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
stats.per_table = (0..max_dbi)
|
||||
.filter(|&d| total_per[d] > 0)
|
||||
.map(|d| TableResidency {
|
||||
dbi: d,
|
||||
name: info_clone.dbi_names.get(d).cloned().unwrap_or_default(),
|
||||
total: total_per[d],
|
||||
cached: cached_per[d],
|
||||
pct: if total_per[d] > 0 {
|
||||
(cached_per[d] as f64 / total_per[d] as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
if prev.len() != cur.len() {
|
||||
let mut buf = Vec::with_capacity(1 + cur.len());
|
||||
buf.push(0u8);
|
||||
buf.extend_from_slice(&cur);
|
||||
let _ = res_tx.send(buf);
|
||||
tracing::info!(
|
||||
"residency-poller: broadcast full snapshot ({} pages)",
|
||||
cur.len(),
|
||||
);
|
||||
} else {
|
||||
let mut spans: Vec<u8> = Vec::new();
|
||||
let mut span_count: u32 = 0;
|
||||
spans.extend_from_slice(&[1u8]);
|
||||
spans.extend_from_slice(&[0u8; 4]);
|
||||
|
||||
let mut i = 0;
|
||||
while i < cur.len() {
|
||||
if cur[i] != prev[i] {
|
||||
let state_val = cur[i];
|
||||
let start = i as u32;
|
||||
let mut end = i + 1;
|
||||
while end < cur.len()
|
||||
&& cur[end] != prev[end]
|
||||
&& cur[end] == state_val
|
||||
{
|
||||
end += 1;
|
||||
}
|
||||
let run_len = (end - i) as u32;
|
||||
spans.extend_from_slice(&start.to_le_bytes());
|
||||
spans.extend_from_slice(&run_len.to_le_bytes());
|
||||
spans.push(state_val);
|
||||
span_count += 1;
|
||||
i = end;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if span_count > 0 {
|
||||
spans[1..5].copy_from_slice(&span_count.to_le_bytes());
|
||||
let _ = res_tx.send(spans);
|
||||
}
|
||||
}
|
||||
|
||||
prev = cur;
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn residency-poller thread");
|
||||
|
||||
let app = Router::new()
|
||||
.route("/", get(index_handler))
|
||||
.route("/ws", get(ws_handler))
|
||||
.route("/api/info", get(info_handler))
|
||||
.route("/api/owner_map", get(owner_map_handler))
|
||||
.route("/api/tree_info", get(tree_info_handler))
|
||||
.route("/api/residency", get(residency_handler))
|
||||
.route("/api/residency_stats", get(residency_stats_handler))
|
||||
.route("/ws_residency", get(ws_residency_handler))
|
||||
.with_state(state);
|
||||
|
||||
let addr = format!("0.0.0.0:{}", config.port);
|
||||
tracing::info!("mdbx-viz server listening on {addr}");
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&addr).await?;
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn index_handler() -> Html<&'static str> {
|
||||
Html(include_str!("index.html"))
|
||||
}
|
||||
|
||||
async fn info_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
axum::Json(state.info)
|
||||
}
|
||||
|
||||
async fn owner_map_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let map = state.owner_map.read().unwrap();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/octet-stream")],
|
||||
map.clone(),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn tree_info_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
axum::Json(state.tree_info.as_ref().clone())
|
||||
}
|
||||
|
||||
async fn ws_handler(
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<AppState>,
|
||||
) -> impl IntoResponse {
|
||||
ws.on_upgrade(move |socket| handle_ws(socket, state))
|
||||
}
|
||||
|
||||
async fn handle_ws(mut socket: WebSocket, state: AppState) {
|
||||
state.subscribers.fetch_add(1, Ordering::Relaxed);
|
||||
let mut rx = state.event_tx.subscribe();
|
||||
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(data) => {
|
||||
if socket.send(Message::Binary(data.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!("ws client lagged by {n} messages");
|
||||
let msg = format!("{{\"lagged\":{n}}}");
|
||||
if socket.send(Message::Text(msg)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
|
||||
state.subscribers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn residency_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let map = state.residency.read().unwrap();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/octet-stream")],
|
||||
map.clone(),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn residency_stats_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let stats = state.residency_stats.read().unwrap();
|
||||
axum::Json(stats.clone())
|
||||
}
|
||||
|
||||
async fn ws_residency_handler(
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<AppState>,
|
||||
) -> impl IntoResponse {
|
||||
ws.on_upgrade(move |socket| handle_ws_residency(socket, state))
|
||||
}
|
||||
|
||||
async fn handle_ws_residency(mut socket: WebSocket, state: AppState) {
|
||||
state.residency_subscribers.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
{
|
||||
let msg = {
|
||||
let map = state.residency.read().unwrap();
|
||||
if !map.is_empty() {
|
||||
let mut buf = Vec::with_capacity(1 + map.len());
|
||||
buf.push(0u8);
|
||||
buf.extend_from_slice(&map);
|
||||
Some(buf)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(msg) = msg {
|
||||
let _ = socket.send(Message::Binary(msg.into())).await;
|
||||
}
|
||||
}
|
||||
|
||||
let mut rx = state.residency_tx.subscribe();
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(data) => {
|
||||
if socket.send(Message::Binary(data.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
|
||||
state.residency_subscribers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
93
crates/storage/mdbx-viz/src/main.rs
Normal file
93
crates/storage/mdbx-viz/src/main.rs
Normal file
@@ -0,0 +1,93 @@
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::Parser;
|
||||
use reth_libmdbx::pageviz::{PageEvent, PageOp};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "reth-mdbx-viz", about = "Real-time MDBX page visualization")]
|
||||
struct Args {
|
||||
/// Path to the .bin owner map file (generated by mdbx-viz Python tool)
|
||||
#[arg(short, long)]
|
||||
bin: Option<String>,
|
||||
|
||||
/// Page size in bytes
|
||||
#[arg(long, default_value = "4096")]
|
||||
page_size: u32,
|
||||
|
||||
/// Port to serve on
|
||||
#[arg(short, long, default_value = "3141")]
|
||||
port: u16,
|
||||
|
||||
/// Generate fake events for testing
|
||||
#[arg(long)]
|
||||
demo: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "info".into()),
|
||||
)
|
||||
.init();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
let (owner_map, page_count) = if let Some(ref bin_path) = args.bin {
|
||||
let data = std::fs::read(bin_path).expect("failed to read .bin file");
|
||||
let pc = data.len() as u64;
|
||||
tracing::info!("loaded owner map: {pc} pages from {bin_path}");
|
||||
(data, pc)
|
||||
} else {
|
||||
tracing::info!("no --bin provided, starting with empty owner map");
|
||||
(vec![0xFFu8; 1_000_000], 1_000_000)
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
|
||||
if args.demo {
|
||||
let np = page_count as u32;
|
||||
std::thread::Builder::new()
|
||||
.name("demo-events".into())
|
||||
.spawn(move || {
|
||||
let mut pgno: u32 = 0;
|
||||
loop {
|
||||
let mut batch = Vec::with_capacity(256);
|
||||
for _ in 0..256 {
|
||||
let p = pgno % np;
|
||||
batch.push(PageEvent {
|
||||
pgno: p,
|
||||
dbi: (p % 8) + 2,
|
||||
op: match p % 10 {
|
||||
0..=6 => PageOp::Read,
|
||||
7..=8 => PageOp::Write,
|
||||
_ => PageOp::Free,
|
||||
},
|
||||
});
|
||||
pgno = pgno.wrapping_add(37);
|
||||
}
|
||||
if tx.send(batch).is_err() {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(33));
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn demo thread");
|
||||
}
|
||||
|
||||
let config = reth_mdbx_viz::VizConfig {
|
||||
port: args.port,
|
||||
page_count,
|
||||
page_size: args.page_size,
|
||||
dbi_names: vec![],
|
||||
tree_info: vec![],
|
||||
owner_map,
|
||||
db_path: std::path::PathBuf::new(),
|
||||
};
|
||||
|
||||
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
|
||||
tracing::error!("server error: {e}");
|
||||
}
|
||||
}
|
||||
757
crates/storage/mdbx-viz/src/walker.rs
Normal file
757
crates/storage/mdbx-viz/src/walker.rs
Normal file
@@ -0,0 +1,757 @@
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io::{self, Read, Write},
|
||||
os::unix::io::AsRawFd,
|
||||
path::Path,
|
||||
sync::atomic::{AtomicU8, AtomicU64, Ordering},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const PAGEHDRSZ: usize = 20;
|
||||
const NODESIZE: usize = 8;
|
||||
|
||||
const P_BRANCH: u16 = 0x01;
|
||||
const P_LEAF: u16 = 0x02;
|
||||
const P_LARGE: u16 = 0x04;
|
||||
const P_META: u16 = 0x08;
|
||||
const P_DUPFIX: u16 = 0x20;
|
||||
|
||||
const N_BIG: u8 = 0x01;
|
||||
const N_TREE: u8 = 0x02;
|
||||
const N_DUP: u8 = 0x04;
|
||||
|
||||
const P_INVALID: u32 = 0xFFFFFFFF;
|
||||
const UNREFERENCED: u8 = 0xFF;
|
||||
const DBI_META: u8 = 0xFE;
|
||||
|
||||
const META_GEO: usize = 0x14;
|
||||
const META_FREE_DB: usize = 0x28;
|
||||
const META_MAIN_DB: usize = 0x58;
|
||||
const META_TXNID_A: usize = 0x08;
|
||||
const META_TXNID_B: usize = 0xB0;
|
||||
const GEO_NOW: usize = 0x0C;
|
||||
|
||||
const TREE_ROOT: usize = 0x08;
|
||||
const TREE_BRANCH_PAGES: usize = 0x0C;
|
||||
const TREE_LEAF_PAGES: usize = 0x10;
|
||||
const TREE_LARGE_PAGES: usize = 0x14;
|
||||
const TREE_ITEMS: usize = 0x20;
|
||||
|
||||
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
|
||||
|
||||
const CACHE_MAGIC: u64 = 0x5056495A_43414348; // "PVIZC ACH"
|
||||
const CACHE_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TreeInfo {
|
||||
pub name: String,
|
||||
pub dbi_index: u8,
|
||||
pub height: u16,
|
||||
pub branch_pages: u32,
|
||||
pub leaf_pages: u32,
|
||||
pub large_pages: u32,
|
||||
pub items: u64,
|
||||
pub root_pgno: u32,
|
||||
}
|
||||
|
||||
pub struct WalkResult {
|
||||
pub owner_map: Vec<u8>,
|
||||
pub page_count: usize,
|
||||
pub page_size: usize,
|
||||
pub tree_info: Vec<TreeInfo>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
struct TreeDescriptor {
|
||||
flags: u16,
|
||||
height: u16,
|
||||
dupfix_size: u32,
|
||||
root: u32,
|
||||
branch_pages: u32,
|
||||
leaf_pages: u32,
|
||||
large_pages: u32,
|
||||
sequence: u64,
|
||||
items: u64,
|
||||
mod_txnid: u64,
|
||||
}
|
||||
|
||||
struct DBIInfo {
|
||||
name: String,
|
||||
dbi_index: u8,
|
||||
tree: TreeDescriptor,
|
||||
}
|
||||
|
||||
struct MmapFile {
|
||||
ptr: *mut u8,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl MmapFile {
|
||||
fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let len = file.metadata()?.len() as usize;
|
||||
if len == 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "empty file"));
|
||||
}
|
||||
let ptr = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if ptr == libc::MAP_FAILED {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(Self { ptr: ptr as *mut u8, len })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page(&self, pgno: usize, ps: usize) -> Option<&[u8]> {
|
||||
let off = pgno.checked_mul(ps)?;
|
||||
if off + ps > self.len {
|
||||
return None;
|
||||
}
|
||||
Some(unsafe { std::slice::from_raw_parts(self.ptr.add(off), ps) })
|
||||
}
|
||||
|
||||
fn evict(&self) {
|
||||
unsafe {
|
||||
libc::madvise(self.ptr as *mut libc::c_void, self.len, libc::MADV_DONTNEED);
|
||||
}
|
||||
tracing::info!(size_gb = self.len / (1024 * 1024 * 1024), "evicted walker mmap from page cache");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MmapFile {
|
||||
fn drop(&mut self) {
|
||||
unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.len); }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for MmapFile {}
|
||||
unsafe impl Sync for MmapFile {}
|
||||
|
||||
#[inline]
|
||||
fn u16_le(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
u64::from_le_bytes([
|
||||
buf[off], buf[off + 1], buf[off + 2], buf[off + 3],
|
||||
buf[off + 4], buf[off + 5], buf[off + 6], buf[off + 7],
|
||||
])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_flags(buf: &[u8]) -> u16 {
|
||||
u16_le(buf, 0x0A)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_nkeys(buf: &[u8]) -> usize {
|
||||
u16_le(buf, 0x0C) as usize / 2
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_overflow_count(buf: &[u8]) -> u32 {
|
||||
u32_le(buf, 0x0C)
|
||||
}
|
||||
|
||||
fn parse_tree_descriptor(buf: &[u8], off: usize) -> TreeDescriptor {
|
||||
TreeDescriptor {
|
||||
flags: u16_le(buf, off),
|
||||
height: u16_le(buf, off + 0x02),
|
||||
dupfix_size: u32_le(buf, off + 0x04),
|
||||
root: u32_le(buf, off + TREE_ROOT),
|
||||
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
|
||||
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
|
||||
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
|
||||
sequence: u64_le(buf, off + 0x18),
|
||||
items: u64_le(buf, off + TREE_ITEMS),
|
||||
mod_txnid: u64_le(buf, off + 0x28),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn claim(owner_map: &[AtomicU8], pgno: usize, dbi_index: u8, conflicts: &AtomicU64) -> bool {
|
||||
if pgno >= owner_map.len() {
|
||||
return false;
|
||||
}
|
||||
match owner_map[pgno].compare_exchange(UNREFERENCED, dbi_index, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => true,
|
||||
Err(existing) => {
|
||||
if existing != dbi_index {
|
||||
conflicts.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_tree_claim(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
root_pgno: u32,
|
||||
page_count: usize,
|
||||
dbi_index: u8,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
) {
|
||||
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut stack: Vec<usize> = vec![root_pgno as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_index, conflicts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if op < page_count {
|
||||
claim(owner_map, op, dbi_index, conflicts);
|
||||
}
|
||||
}
|
||||
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
|
||||
if data_off + TREE_ROOT + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let sub_root = u32_le(buf, data_off + TREE_ROOT);
|
||||
if sub_root != P_INVALID {
|
||||
stack.push(sub_root as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if flags & P_LARGE != 0 {
|
||||
let ov_count = page_overflow_count(buf) as usize;
|
||||
for op in (pgno + 1)..pgno + ov_count {
|
||||
if op < page_count {
|
||||
claim(owner_map, op, dbi_index, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn discover_named_dbis(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
main_root: u32,
|
||||
page_count: usize,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
dbi_main: u8,
|
||||
dbi_start: u8,
|
||||
) -> Vec<DBIInfo> {
|
||||
let mut named = Vec::new();
|
||||
if main_root == P_INVALID || main_root as usize >= page_count {
|
||||
return named;
|
||||
}
|
||||
|
||||
let mut next_index = dbi_start;
|
||||
let mut stack: Vec<usize> = vec![main_root as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_main, conflicts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_TREE != 0 {
|
||||
let key_off = node_off + NODESIZE;
|
||||
if key_off + ksize > ps || data_off + 0x30 > ps {
|
||||
continue;
|
||||
}
|
||||
let key_bytes = &buf[key_off..key_off + ksize];
|
||||
let name = match key_bytes.iter().position(|&b| b == 0) {
|
||||
Some(nul) => &key_bytes[..nul],
|
||||
None => key_bytes,
|
||||
};
|
||||
let name = String::from_utf8_lossy(name).into_owned();
|
||||
let tree = parse_tree_descriptor(buf, data_off);
|
||||
named.push(DBIInfo { name, dbi_index: next_index, tree });
|
||||
next_index = next_index.wrapping_add(1);
|
||||
} else if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
claim(owner_map, op, dbi_main, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
named
|
||||
}
|
||||
|
||||
fn mark_free_pages(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
free_root: u32,
|
||||
page_count: usize,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
dbi_free: u8,
|
||||
) -> u64 {
|
||||
let mut marked: u64 = 0;
|
||||
if free_root == P_INVALID || free_root as usize >= page_count {
|
||||
return marked;
|
||||
}
|
||||
|
||||
let mut stack: Vec<usize> = vec![free_root as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_free, conflicts) {
|
||||
continue;
|
||||
}
|
||||
marked += 1;
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let dsize = u32_le(buf, node_off) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if claim(owner_map, op, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
let ov_data_off = PAGEHDRSZ;
|
||||
if dsize >= 4 && ov_data_off + 4 <= ps {
|
||||
let pnl_count = u32_le(ov_buf, ov_data_off) as usize;
|
||||
let max_entries = (ps - ov_data_off - 4) / 4;
|
||||
let pnl_count = pnl_count.min(max_entries);
|
||||
for j in 0..pnl_count {
|
||||
let off = ov_data_off + 4 + j * 4;
|
||||
if off + 4 > ps { break; }
|
||||
let fp = u32_le(ov_buf, off) as usize;
|
||||
if fp < page_count {
|
||||
if claim(owner_map, fp, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dsize >= 4 && data_off + 4 <= ps {
|
||||
let pnl_count = u32_le(buf, data_off) as usize;
|
||||
let max_entries = (ps - data_off - 4) / 4;
|
||||
let pnl_count = pnl_count.min(max_entries);
|
||||
for j in 0..pnl_count {
|
||||
let off = data_off + 4 + j * 4;
|
||||
if off + 4 > ps { break; }
|
||||
let fp = u32_le(buf, off) as usize;
|
||||
if fp < page_count {
|
||||
if claim(owner_map, fp, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
marked
|
||||
}
|
||||
|
||||
fn cache_path(mdbx_path: &Path) -> std::path::PathBuf {
|
||||
mdbx_path.with_file_name("mdbx_pageviz_cache.bin")
|
||||
}
|
||||
|
||||
fn try_load_cache(
|
||||
path: &Path,
|
||||
file_size: u64,
|
||||
txnid: u64,
|
||||
page_size: usize,
|
||||
page_count: usize,
|
||||
) -> Option<WalkResult> {
|
||||
let cp = cache_path(path);
|
||||
let mut f = std::fs::File::open(&cp).ok()?;
|
||||
let mut hdr = [0u8; 40];
|
||||
f.read_exact(&mut hdr).ok()?;
|
||||
|
||||
let magic = u64::from_le_bytes(hdr[0..8].try_into().unwrap());
|
||||
let ver = u32::from_le_bytes(hdr[8..12].try_into().unwrap());
|
||||
let c_txnid = u64::from_le_bytes(hdr[12..20].try_into().unwrap());
|
||||
let c_pc = u64::from_le_bytes(hdr[20..28].try_into().unwrap());
|
||||
let c_ps = u32::from_le_bytes(hdr[28..32].try_into().unwrap());
|
||||
let c_fsz = u64::from_le_bytes(hdr[32..40].try_into().unwrap());
|
||||
|
||||
if magic != CACHE_MAGIC || ver != CACHE_VERSION {
|
||||
return None;
|
||||
}
|
||||
if c_txnid != txnid || c_pc as usize != page_count || c_ps as usize != page_size || c_fsz != file_size {
|
||||
tracing::info!(
|
||||
cache_txnid = c_txnid, current_txnid = txnid,
|
||||
"cache stale, will re-walk"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut ti_len_buf = [0u8; 4];
|
||||
f.read_exact(&mut ti_len_buf).ok()?;
|
||||
let ti_len = u32::from_le_bytes(ti_len_buf) as usize;
|
||||
|
||||
let mut ti_buf = vec![0u8; ti_len];
|
||||
f.read_exact(&mut ti_buf).ok()?;
|
||||
let tree_info: Vec<TreeInfo> = serde_json::from_slice(&ti_buf).ok()?;
|
||||
|
||||
let mut owner_map = vec![0u8; page_count];
|
||||
f.read_exact(&mut owner_map).ok()?;
|
||||
|
||||
tracing::info!(page_count, "loaded owner_map from cache");
|
||||
Some(WalkResult { owner_map, page_count, page_size, tree_info })
|
||||
}
|
||||
|
||||
fn save_cache(
|
||||
path: &Path,
|
||||
file_size: u64,
|
||||
txnid: u64,
|
||||
result: &WalkResult,
|
||||
) {
|
||||
let cp = cache_path(path);
|
||||
let tmp = cp.with_extension("tmp");
|
||||
|
||||
let write_inner = || -> io::Result<()> {
|
||||
let mut f = std::fs::File::create(&tmp)?;
|
||||
f.write_all(&CACHE_MAGIC.to_le_bytes())?;
|
||||
f.write_all(&CACHE_VERSION.to_le_bytes())?;
|
||||
f.write_all(&txnid.to_le_bytes())?;
|
||||
f.write_all(&(result.page_count as u64).to_le_bytes())?;
|
||||
f.write_all(&(result.page_size as u32).to_le_bytes())?;
|
||||
f.write_all(&file_size.to_le_bytes())?;
|
||||
|
||||
let ti_json = serde_json::to_vec(&result.tree_info).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
f.write_all(&(ti_json.len() as u32).to_le_bytes())?;
|
||||
f.write_all(&ti_json)?;
|
||||
f.write_all(&result.owner_map)?;
|
||||
f.flush()?;
|
||||
Ok(())
|
||||
};
|
||||
|
||||
match write_inner() {
|
||||
Ok(()) => {
|
||||
if let Err(e) = std::fs::rename(&tmp, &cp) {
|
||||
tracing::warn!("failed to rename cache file: {e}");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
} else {
|
||||
tracing::info!(
|
||||
path = %cp.display(),
|
||||
size_mb = result.owner_map.len() / (1024 * 1024),
|
||||
"saved owner_map cache"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to write cache: {e}");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn walk_mdbx(path: &Path, name_to_dbi: &HashMap<&str, u8>) -> eyre::Result<WalkResult> {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let mmap = MmapFile::open(path)?;
|
||||
let file_size = mmap.len as u64;
|
||||
|
||||
let meta0 = mmap.page(0, 4096).ok_or_else(|| eyre::eyre!("cannot read page 0"))?;
|
||||
let pflags = page_flags(meta0);
|
||||
eyre::ensure!(pflags & P_META != 0, "page 0 missing P_META flag");
|
||||
|
||||
let magic = u64_le(meta0, PAGEHDRSZ);
|
||||
eyre::ensure!((magic >> 8) == MDBX_MAGIC, "MDBX magic mismatch");
|
||||
|
||||
let candidates: &[usize] = &[4096, 8192, 16384, 32768, 65536, 1024, 2048];
|
||||
let geo_now_raw = u32_le(meta0, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
|
||||
|
||||
let mut ps = 4096usize;
|
||||
for &candidate in candidates {
|
||||
let mapped = geo_now_raw * candidate;
|
||||
if mapped >= mmap.len / 2 && mapped <= mmap.len * 4 {
|
||||
ps = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(page_size = ps, file_size_gb = file_size / (1024 * 1024 * 1024), "detected page size");
|
||||
|
||||
let meta_pages: Vec<&[u8]> = (0..3)
|
||||
.map(|i| mmap.page(i, ps).ok_or_else(|| eyre::eyre!("cannot read meta page {i}")))
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
let mut best_idx = 0usize;
|
||||
let mut best_txnid = 0u64;
|
||||
|
||||
for (i, meta) in meta_pages.iter().enumerate() {
|
||||
let m = u64_le(meta, PAGEHDRSZ);
|
||||
if (m >> 8) != MDBX_MAGIC {
|
||||
continue;
|
||||
}
|
||||
let txnid_a = u64_le(meta, PAGEHDRSZ + META_TXNID_A);
|
||||
let txnid_b = u64_le(meta, PAGEHDRSZ + META_TXNID_B);
|
||||
if txnid_a == txnid_b && txnid_a > best_txnid {
|
||||
best_txnid = txnid_a;
|
||||
best_idx = i;
|
||||
}
|
||||
}
|
||||
|
||||
eyre::ensure!(best_txnid > 0, "no valid meta page found");
|
||||
|
||||
let best = meta_pages[best_idx];
|
||||
let mb = PAGEHDRSZ;
|
||||
let geo_now = u32_le(best, mb + META_GEO + GEO_NOW) as usize;
|
||||
let page_count = geo_now;
|
||||
|
||||
tracing::info!(best_meta = best_idx, txnid = best_txnid, page_count, "selected best meta");
|
||||
|
||||
if let Some(cached) = try_load_cache(path, file_size, best_txnid, ps, page_count) {
|
||||
let elapsed = t0.elapsed();
|
||||
tracing::info!(elapsed_ms = elapsed.as_millis() as u64, "walk skipped (cache hit)");
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
let free_tree = parse_tree_descriptor(best, mb + META_FREE_DB);
|
||||
let main_tree = parse_tree_descriptor(best, mb + META_MAIN_DB);
|
||||
|
||||
tracing::info!(
|
||||
page_count,
|
||||
free_root = free_tree.root,
|
||||
main_root = main_tree.root,
|
||||
"parsed meta, starting walk"
|
||||
);
|
||||
|
||||
let owner_map: Vec<AtomicU8> = (0..page_count).map(|_| AtomicU8::new(UNREFERENCED)).collect();
|
||||
let conflicts = AtomicU64::new(0);
|
||||
|
||||
for pgno in 0..3.min(page_count) {
|
||||
owner_map[pgno].store(DBI_META, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let dbi_free: u8 = 1;
|
||||
let dbi_main: u8 = 2;
|
||||
let dbi_start: u8 = 3;
|
||||
|
||||
tracing::info!("discovering named DBIs via MainDB walk");
|
||||
let mut named_dbis =
|
||||
discover_named_dbis(&mmap, ps, main_tree.root, page_count, &owner_map, &conflicts, dbi_main, dbi_start);
|
||||
tracing::info!(count = named_dbis.len(), "found named DBIs");
|
||||
|
||||
for dbi in &mut named_dbis {
|
||||
if let Some(&mapped_idx) = name_to_dbi.get(dbi.name.as_str()) {
|
||||
dbi.dbi_index = mapped_idx;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("walking FreeDB");
|
||||
let free_marked =
|
||||
mark_free_pages(&mmap, ps, free_tree.root, page_count, &owner_map, &conflicts, dbi_free);
|
||||
tracing::info!(marked = free_marked, "FreeDB walk complete");
|
||||
|
||||
let walk_tasks: Vec<(u32, u8)> = named_dbis
|
||||
.iter()
|
||||
.filter(|d| d.tree.root != P_INVALID)
|
||||
.map(|d| (d.tree.root, d.dbi_index))
|
||||
.collect();
|
||||
|
||||
tracing::info!(count = walk_tasks.len(), "walking named DBIs in parallel");
|
||||
|
||||
walk_tasks
|
||||
.par_iter()
|
||||
.for_each(|&(root, dbi_idx)| {
|
||||
walk_tree_claim(&mmap, ps, root, page_count, dbi_idx, &owner_map, &conflicts);
|
||||
});
|
||||
|
||||
let final_conflicts = conflicts.load(Ordering::Relaxed);
|
||||
let owner_map_vec: Vec<u8> = owner_map.iter().map(|a| a.load(Ordering::Relaxed)).collect();
|
||||
let assigned = owner_map_vec.iter().filter(|&&b| b != UNREFERENCED).count();
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
tracing::info!(
|
||||
elapsed_ms = elapsed.as_millis() as u64,
|
||||
assigned,
|
||||
unreferenced = page_count - assigned,
|
||||
conflicts = final_conflicts,
|
||||
"walk complete"
|
||||
);
|
||||
|
||||
let mut tree_info = Vec::new();
|
||||
|
||||
tree_info.push(TreeInfo {
|
||||
name: "FreeDB".to_string(),
|
||||
dbi_index: dbi_free,
|
||||
height: free_tree.height,
|
||||
branch_pages: free_tree.branch_pages,
|
||||
leaf_pages: free_tree.leaf_pages,
|
||||
large_pages: free_tree.large_pages,
|
||||
items: free_tree.items,
|
||||
root_pgno: free_tree.root,
|
||||
});
|
||||
|
||||
tree_info.push(TreeInfo {
|
||||
name: "MainDB".to_string(),
|
||||
dbi_index: dbi_main,
|
||||
height: main_tree.height,
|
||||
branch_pages: main_tree.branch_pages,
|
||||
leaf_pages: main_tree.leaf_pages,
|
||||
large_pages: main_tree.large_pages,
|
||||
items: main_tree.items,
|
||||
root_pgno: main_tree.root,
|
||||
});
|
||||
|
||||
for dbi in &named_dbis {
|
||||
tree_info.push(TreeInfo {
|
||||
name: dbi.name.clone(),
|
||||
dbi_index: dbi.dbi_index,
|
||||
height: dbi.tree.height,
|
||||
branch_pages: dbi.tree.branch_pages,
|
||||
leaf_pages: dbi.tree.leaf_pages,
|
||||
large_pages: dbi.tree.large_pages,
|
||||
items: dbi.tree.items,
|
||||
root_pgno: dbi.tree.root,
|
||||
});
|
||||
}
|
||||
|
||||
let result = WalkResult { owner_map: owner_map_vec, page_count, page_size: ps, tree_info };
|
||||
|
||||
save_cache(path, file_size, best_txnid, &result);
|
||||
|
||||
mmap.evict();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
@@ -5,8 +5,7 @@ use crate::ProviderResult;
|
||||
use alloy_primitives::BlockNumber;
|
||||
use reth_db::models::AccountBeforeTx;
|
||||
use reth_db_api::models::BlockNumberAddress;
|
||||
use reth_primitives_traits::StorageEntry;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, ChangesetEntry, StorageChangeSetReader};
|
||||
use std::ops::{Bound, RangeBounds};
|
||||
|
||||
/// Iterator that walks account changesets from static files in a block range.
|
||||
@@ -110,7 +109,7 @@ pub struct StaticFileStorageChangesetWalker<P> {
|
||||
/// Current block being processed
|
||||
current_block: BlockNumber,
|
||||
/// Changesets for current block
|
||||
current_changesets: Vec<(BlockNumberAddress, StorageEntry)>,
|
||||
current_changesets: Vec<(BlockNumberAddress, ChangesetEntry)>,
|
||||
/// Index within current block's changesets
|
||||
changeset_index: usize,
|
||||
}
|
||||
@@ -144,7 +143,7 @@ impl<P> Iterator for StaticFileStorageChangesetWalker<P>
|
||||
where
|
||||
P: StorageChangeSetReader,
|
||||
{
|
||||
type Item = ProviderResult<(BlockNumberAddress, StorageEntry)>;
|
||||
type Item = ProviderResult<(BlockNumberAddress, ChangesetEntry)>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if let Some(changeset) = self.current_changesets.get(self.changeset_index).copied() {
|
||||
|
||||
@@ -162,7 +162,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
Ok(EitherWriter::StaticFile(
|
||||
provider
|
||||
.get_static_file_writer(block_number, StaticFileSegment::AccountChangeSets)?,
|
||||
@@ -183,7 +183,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
P: DBProvider + NodePrimitivesProvider + StorageSettingsCache + StaticFileProviderFactory,
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
Ok(EitherWriter::StaticFile(
|
||||
provider
|
||||
.get_static_file_writer(block_number, StaticFileSegment::StorageChangeSets)?,
|
||||
@@ -206,7 +206,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
pub fn receipts_destination<P: DBProvider + StorageSettingsCache>(
|
||||
provider: &P,
|
||||
) -> EitherWriterDestination {
|
||||
let receipts_in_static_files = provider.cached_storage_settings().receipts_in_static_files;
|
||||
let receipts_in_static_files = provider.cached_storage_settings().storage_v2;
|
||||
let prune_modes = provider.prune_modes_ref();
|
||||
|
||||
if !receipts_in_static_files && prune_modes.has_receipts_pruning() ||
|
||||
@@ -225,7 +225,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
pub fn account_changesets_destination<P: DBProvider + StorageSettingsCache>(
|
||||
provider: &P,
|
||||
) -> EitherWriterDestination {
|
||||
if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
EitherWriterDestination::StaticFile
|
||||
} else {
|
||||
EitherWriterDestination::Database
|
||||
@@ -238,7 +238,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
pub fn storage_changesets_destination<P: DBProvider + StorageSettingsCache>(
|
||||
provider: &P,
|
||||
) -> EitherWriterDestination {
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
EitherWriterDestination::StaticFile
|
||||
} else {
|
||||
EitherWriterDestination::Database
|
||||
@@ -255,7 +255,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherWriter::RocksDB(_rocksdb_batch));
|
||||
}
|
||||
|
||||
@@ -272,7 +272,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherWriter::RocksDB(_rocksdb_batch));
|
||||
}
|
||||
|
||||
@@ -291,7 +291,7 @@ impl<'a> EitherWriter<'a, (), ()> {
|
||||
P::Tx: DbTxMut,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherWriter::RocksDB(_rocksdb_batch));
|
||||
}
|
||||
|
||||
@@ -764,7 +764,7 @@ impl<'a> EitherReader<'a, (), ()> {
|
||||
P::Tx: DbTx,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherReader::RocksDB(
|
||||
_rocksdb_tx.expect("storages_history_in_rocksdb requires rocksdb tx"),
|
||||
));
|
||||
@@ -786,7 +786,7 @@ impl<'a> EitherReader<'a, (), ()> {
|
||||
P::Tx: DbTx,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherReader::RocksDB(
|
||||
_rocksdb_tx.expect("transaction_hash_numbers_in_rocksdb requires rocksdb tx"),
|
||||
));
|
||||
@@ -808,7 +808,7 @@ impl<'a> EitherReader<'a, (), ()> {
|
||||
P::Tx: DbTx,
|
||||
{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
return Ok(EitherReader::RocksDB(
|
||||
_rocksdb_tx.expect("account_history_in_rocksdb requires rocksdb tx"),
|
||||
));
|
||||
@@ -1046,7 +1046,7 @@ impl EitherWriterDestination {
|
||||
P: StorageSettingsCache,
|
||||
{
|
||||
// Write senders to static files only if they're explicitly enabled
|
||||
if provider.cached_storage_settings().transaction_senders_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
Self::StaticFile
|
||||
} else {
|
||||
Self::Database
|
||||
@@ -1059,7 +1059,7 @@ impl EitherWriterDestination {
|
||||
P: StorageSettingsCache,
|
||||
{
|
||||
// Write account changesets to static files only if they're explicitly enabled
|
||||
if provider.cached_storage_settings().account_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
Self::StaticFile
|
||||
} else {
|
||||
Self::Database
|
||||
@@ -1072,7 +1072,7 @@ impl EitherWriterDestination {
|
||||
P: StorageSettingsCache,
|
||||
{
|
||||
// Write storage changesets to static files only if they're explicitly enabled
|
||||
if provider.cached_storage_settings().storage_changesets_in_static_files {
|
||||
if provider.cached_storage_settings().storage_v2 {
|
||||
Self::StaticFile
|
||||
} else {
|
||||
Self::Database
|
||||
@@ -1127,9 +1127,7 @@ mod tests {
|
||||
writer.commit().unwrap();
|
||||
}
|
||||
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(true),
|
||||
);
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
@@ -1161,10 +1159,11 @@ mod tests {
|
||||
];
|
||||
|
||||
for transaction_senders_in_static_files in [false, true] {
|
||||
factory.set_storage_settings_cache(
|
||||
factory.set_storage_settings_cache(if transaction_senders_in_static_files {
|
||||
StorageSettings::v2()
|
||||
} else {
|
||||
StorageSettings::v1()
|
||||
.with_transaction_senders_in_static_files(transaction_senders_in_static_files),
|
||||
);
|
||||
});
|
||||
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
let mut writer = EitherWriter::new_senders(&provider, 0).unwrap();
|
||||
@@ -1234,9 +1233,7 @@ mod rocksdb_tests {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let hash1 = B256::from([1u8; 32]);
|
||||
let hash2 = B256::from([2u8; 32]);
|
||||
@@ -1278,9 +1275,7 @@ mod rocksdb_tests {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let hash = B256::from([1u8; 32]);
|
||||
let tx_num = 100u64;
|
||||
@@ -1832,9 +1827,7 @@ mod rocksdb_tests {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
// Enable RocksDB for transaction hash numbers
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let hash1 = B256::from([1u8; 32]);
|
||||
let hash2 = B256::from([2u8; 32]);
|
||||
@@ -1892,9 +1885,7 @@ mod rocksdb_tests {
|
||||
fn test_settings_mismatch_panics() {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
factory.set_storage_settings_cache(
|
||||
StorageSettings::v1().with_account_history_in_rocksdb(true),
|
||||
);
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
let _ = EitherReader::<(), ()>::new_accounts_history(&provider, None);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user