mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
12 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
784b4e9dd2 | ||
|
|
2badbe3f3f | ||
|
|
f2cada2660 | ||
|
|
a2006c5f86 | ||
|
|
a95cf8fe8e | ||
|
|
852551b790 | ||
|
|
483eaae67c | ||
|
|
f76a748d92 | ||
|
|
fb1039cf7a | ||
|
|
ffa681795f | ||
|
|
cd7f43bdf0 | ||
|
|
3de7589755 |
128
Cargo.lock
generated
128
Cargo.lock
generated
@@ -1520,6 +1520,64 @@ version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.7.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum-core",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"itoa",
|
||||
"matchit",
|
||||
"memchr",
|
||||
"mime",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
"serde_urlencoded",
|
||||
"sha1",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.24.0",
|
||||
"tower",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum-core"
|
||||
version = "0.4.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"mime",
|
||||
"pin-project-lite",
|
||||
"rustversion",
|
||||
"sync_wrapper",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backon"
|
||||
version = "1.6.0"
|
||||
@@ -5793,6 +5851,12 @@ dependencies = [
|
||||
"regex-automata",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "matchit"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.8.0"
|
||||
@@ -7762,11 +7826,13 @@ dependencies = [
|
||||
"humantime",
|
||||
"itertools 0.14.0",
|
||||
"lz4",
|
||||
"memmap2",
|
||||
"metrics",
|
||||
"parking_lot",
|
||||
"proptest",
|
||||
"proptest-arbitrary-interop",
|
||||
"ratatui",
|
||||
"rayon",
|
||||
"reqwest",
|
||||
"reth-chainspec",
|
||||
"reth-cli",
|
||||
@@ -7979,6 +8045,7 @@ dependencies = [
|
||||
"reth-db-api",
|
||||
"reth-fs-util",
|
||||
"reth-libmdbx",
|
||||
"reth-mdbx-viz",
|
||||
"reth-metrics",
|
||||
"reth-nippy-jar",
|
||||
"reth-primitives-traits",
|
||||
@@ -7993,6 +8060,7 @@ dependencies = [
|
||||
"sysinfo",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
@@ -8393,6 +8461,7 @@ dependencies = [
|
||||
"reth-evm-ethereum",
|
||||
"reth-execution-types",
|
||||
"reth-exex-types",
|
||||
"reth-mdbx-viz",
|
||||
"reth-metrics",
|
||||
"reth-network-p2p",
|
||||
"reth-node-ethereum",
|
||||
@@ -9029,6 +9098,24 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reth-mdbx-viz"
|
||||
version = "1.11.0"
|
||||
dependencies = [
|
||||
"axum",
|
||||
"clap",
|
||||
"eyre",
|
||||
"libc",
|
||||
"rayon",
|
||||
"reth-libmdbx",
|
||||
"reth-mdbx-sys",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber 0.3.22",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reth-metrics"
|
||||
version = "1.11.0"
|
||||
@@ -11475,6 +11562,17 @@ dependencies = [
|
||||
"zmij",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_path_to_error"
|
||||
version = "0.1.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_qs"
|
||||
version = "0.8.5"
|
||||
@@ -12267,6 +12365,18 @@ dependencies = [
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"tokio",
|
||||
"tungstenite 0.24.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-tungstenite"
|
||||
version = "0.26.2"
|
||||
@@ -12706,6 +12816,24 @@ version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"data-encoding",
|
||||
"http",
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.8.5",
|
||||
"sha1",
|
||||
"thiserror 1.0.69",
|
||||
"utf-8",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tungstenite"
|
||||
version = "0.26.2"
|
||||
|
||||
@@ -110,6 +110,7 @@ members = [
|
||||
"crates/storage/errors/",
|
||||
"crates/storage/libmdbx-rs/",
|
||||
"crates/storage/libmdbx-rs/mdbx-sys/",
|
||||
"crates/storage/mdbx-viz/",
|
||||
"crates/storage/nippy-jar/",
|
||||
"crates/storage/provider/",
|
||||
"crates/storage/storage-api/",
|
||||
@@ -377,6 +378,7 @@ reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" }
|
||||
reth-ipc = { path = "crates/rpc/ipc" }
|
||||
reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
|
||||
reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
|
||||
reth-mdbx-viz = { path = "crates/storage/mdbx-viz" }
|
||||
reth-metrics = { path = "crates/metrics" }
|
||||
reth-net-banlist = { path = "crates/net/banlist" }
|
||||
reth-net-nat = { path = "crates/net/nat" }
|
||||
|
||||
@@ -190,6 +190,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
pageviz = ["reth-db/pageviz", "reth-node-builder/pageviz"]
|
||||
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
|
||||
edge = ["rocksdb"]
|
||||
|
||||
|
||||
@@ -87,6 +87,8 @@ tokio-stream.workspace = true
|
||||
reqwest.workspace = true
|
||||
url.workspace = true
|
||||
metrics.workspace = true
|
||||
memmap2.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
# io
|
||||
fdlimit.workspace = true
|
||||
|
||||
564
crates/cli/commands/src/db/gen_ownermap.rs
Normal file
564
crates/cli/commands/src/db/gen_ownermap.rs
Normal file
@@ -0,0 +1,564 @@
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use memmap2::Mmap;
|
||||
use rayon::prelude::*;
|
||||
use reth_db_api::tables::Tables;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
fs,
|
||||
io::Write,
|
||||
path::PathBuf,
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const PAGEHDRSZ: usize = 20;
|
||||
const NODESIZE: usize = 8;
|
||||
const P_BRANCH: u16 = 0x01;
|
||||
const P_LEAF: u16 = 0x02;
|
||||
const P_LARGE: u16 = 0x04;
|
||||
const P_META: u16 = 0x08;
|
||||
const P_DUPFIX: u16 = 0x20;
|
||||
const N_BIG: u8 = 0x01;
|
||||
const N_TREE: u8 = 0x02;
|
||||
const N_DUP: u8 = 0x04;
|
||||
const P_INVALID: u32 = 0xFFFFFFFF;
|
||||
const UNREFERENCED: u8 = 0xFF;
|
||||
const DBI_META: u8 = 0xFE;
|
||||
|
||||
const META_GEO: usize = 0x14;
|
||||
const META_FREE_DB: usize = 0x28;
|
||||
const META_MAIN_DB: usize = 0x58;
|
||||
const META_TXNID_A: usize = 0x08;
|
||||
const META_TXNID_B: usize = 0xB0;
|
||||
const GEO_NOW: usize = 0x0C;
|
||||
const TREE_ROOT: usize = 0x08;
|
||||
const TREE_BRANCH_PAGES: usize = 0x0C;
|
||||
const TREE_LEAF_PAGES: usize = 0x10;
|
||||
const TREE_LARGE_PAGES: usize = 0x14;
|
||||
const TREE_ITEMS: usize = 0x20;
|
||||
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
|
||||
|
||||
fn u16_le(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
fn u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
fn u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
u64::from_le_bytes(buf[off..off + 8].try_into().unwrap())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct TreeDescriptor {
|
||||
root: u32,
|
||||
branch_pages: u32,
|
||||
leaf_pages: u32,
|
||||
large_pages: u32,
|
||||
items: u64,
|
||||
}
|
||||
|
||||
impl TreeDescriptor {
|
||||
fn parse(buf: &[u8], off: usize) -> Self {
|
||||
Self {
|
||||
root: u32_le(buf, off + TREE_ROOT),
|
||||
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
|
||||
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
|
||||
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
|
||||
items: u64_le(buf, off + TREE_ITEMS),
|
||||
}
|
||||
}
|
||||
|
||||
fn total_pages(&self) -> u64 {
|
||||
self.branch_pages as u64 + self.leaf_pages as u64 + self.large_pages as u64
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct DBIInfo {
|
||||
name: String,
|
||||
dbi_index: u8,
|
||||
tree: TreeDescriptor,
|
||||
}
|
||||
|
||||
fn page_flags(buf: &[u8], page_off: usize) -> u16 {
|
||||
u16_le(buf, page_off + 0x0A)
|
||||
}
|
||||
|
||||
fn page_nkeys(buf: &[u8], page_off: usize) -> usize {
|
||||
u16_le(buf, page_off + 0x0C) as usize / 2
|
||||
}
|
||||
|
||||
fn page_overflow_count(buf: &[u8], page_off: usize) -> u32 {
|
||||
u32_le(buf, page_off + 0x0C)
|
||||
}
|
||||
|
||||
fn mark(owner_map: &mut [u8], pgno: usize, dbi_index: u8, conflicts: &mut u64) -> bool {
|
||||
if pgno >= owner_map.len() {
|
||||
return false;
|
||||
}
|
||||
let prev = owner_map[pgno];
|
||||
if prev == UNREFERENCED {
|
||||
owner_map[pgno] = dbi_index;
|
||||
return true;
|
||||
}
|
||||
if prev != dbi_index {
|
||||
*conflicts += 1;
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
|
||||
fn walk_tree_collect(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
root_pgno: u32,
|
||||
page_count: usize,
|
||||
dbi_index: u8,
|
||||
) -> Vec<(usize, u8)> {
|
||||
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut result: Vec<(usize, u8)> = Vec::new();
|
||||
let mut visited = vec![false; page_count];
|
||||
let mut stack: Vec<u32> = vec![root_pgno];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= page_count || visited[pgno_usize] {
|
||||
continue;
|
||||
}
|
||||
visited[pgno_usize] = true;
|
||||
result.push((pgno_usize, dbi_index));
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() {
|
||||
continue;
|
||||
}
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
stack.push(u32_le(buf, node_abs));
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
|
||||
if data_off + 48 > buf.len() { continue; }
|
||||
let sub = TreeDescriptor::parse(buf, data_off);
|
||||
if sub.root != P_INVALID {
|
||||
stack.push(sub.root);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if flags & P_LARGE != 0 {
|
||||
let ov_count = page_overflow_count(buf, page_off) as usize;
|
||||
for op in (pgno_usize + 1)..(pgno_usize + ov_count) {
|
||||
if op < page_count && !visited[op] {
|
||||
visited[op] = true;
|
||||
result.push((op, dbi_index));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result
|
||||
}
|
||||
|
||||
fn discover_named_dbis(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
main_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_main: u8,
|
||||
dbi_start: u8,
|
||||
) -> Vec<DBIInfo> {
|
||||
if main_root == P_INVALID || main_root as usize >= owner_map.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let mut named: Vec<DBIInfo> = Vec::new();
|
||||
let mut next_index = dbi_start;
|
||||
let mut stack: Vec<u32> = vec![main_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_main, conflicts) { continue; }
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + 4 > buf.len() { break; }
|
||||
let child_pgno = u32_le(buf, node_abs);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_TREE != 0 {
|
||||
let key_off = node_abs + NODESIZE;
|
||||
if key_off + ksize > buf.len() || data_off + 48 > buf.len() { continue; }
|
||||
let name_bytes = &buf[key_off..key_off + ksize];
|
||||
let name = String::from_utf8_lossy(
|
||||
name_bytes.split(|&b| b == 0).next().unwrap_or(name_bytes)
|
||||
).to_string();
|
||||
let tree = TreeDescriptor::parse(buf, data_off);
|
||||
named.push(DBIInfo { name, dbi_index: next_index, tree });
|
||||
next_index = next_index.saturating_add(1);
|
||||
} else if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
mark(owner_map, op, dbi_main, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
named
|
||||
}
|
||||
|
||||
fn mark_free_pages(
|
||||
buf: &[u8],
|
||||
ps: usize,
|
||||
free_root: u32,
|
||||
owner_map: &mut [u8],
|
||||
conflicts: &mut u64,
|
||||
dbi_free: u8,
|
||||
) -> u64 {
|
||||
if free_root == P_INVALID || free_root as usize >= owner_map.len() {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let mut marked: u64 = 0;
|
||||
let mut stack: Vec<u32> = vec![free_root];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
let pgno_usize = pgno as usize;
|
||||
if pgno_usize >= owner_map.len() { continue; }
|
||||
if !mark(owner_map, pgno_usize, dbi_free, conflicts) { continue; }
|
||||
marked += 1;
|
||||
|
||||
let page_off = pgno_usize * ps;
|
||||
if page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let flags = page_flags(buf, page_off);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let child_pgno = u32_le(buf, page_off + PAGEHDRSZ + node_rel);
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf, page_off);
|
||||
for i in 0..nkeys {
|
||||
let idx_off = page_off + PAGEHDRSZ + i * 2;
|
||||
if idx_off + 2 > buf.len() { break; }
|
||||
let node_rel = u16_le(buf, idx_off) as usize;
|
||||
let node_abs = page_off + PAGEHDRSZ + node_rel;
|
||||
if node_abs + NODESIZE > buf.len() { break; }
|
||||
|
||||
let mn_flags = buf[node_abs + 4];
|
||||
let ksize = u16_le(buf, node_abs + 6) as usize;
|
||||
let dsize = u32_le(buf, node_abs) as usize;
|
||||
let data_off = node_abs + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > buf.len() { continue; }
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= owner_map.len() { continue; }
|
||||
let ov_page_off = ov_pgno * ps;
|
||||
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
|
||||
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if mark(owner_map, op, dbi_free, conflicts) { marked += 1; }
|
||||
}
|
||||
let ov_data_off = ov_pgno * ps + PAGEHDRSZ;
|
||||
if dsize >= 4 && ov_data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, ov_data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = ov_data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dsize >= 4 && data_off + 4 <= buf.len() {
|
||||
let pnl_count = u32_le(buf, data_off) as usize;
|
||||
for j in 0..pnl_count {
|
||||
let fp_off = data_off + 4 + j * 4;
|
||||
if fp_off + 4 > buf.len() { break; }
|
||||
let fp = u32_le(buf, fp_off) as usize;
|
||||
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
|
||||
owner_map[fp] = dbi_free;
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
marked
|
||||
}
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
pub struct Command {
|
||||
#[arg(short, long, default_value = "owner_map.bin")]
|
||||
output: PathBuf,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
pub fn execute<N: NodeTypesWithDB>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
_tool: &DbTool<N>,
|
||||
) -> eyre::Result<()> {
|
||||
let db_path = data_dir.db().join("mdbx.dat");
|
||||
eyre::ensure!(db_path.exists(), "mdbx.dat not found at {:?}", db_path);
|
||||
|
||||
let t0 = Instant::now();
|
||||
|
||||
let file = fs::File::open(&db_path)
|
||||
.wrap_err_with(|| format!("Failed to open {}", db_path.display()))?;
|
||||
let mmap = unsafe { Mmap::map(&file)? };
|
||||
let buf: &[u8] = &mmap;
|
||||
|
||||
if buf.len() < PAGEHDRSZ + 0xC0 {
|
||||
eyre::bail!("File too small for MDBX meta page");
|
||||
}
|
||||
let flags0 = u16_le(buf, 0x0A);
|
||||
if flags0 & P_META == 0 {
|
||||
eyre::bail!("Page 0 missing P_META flag");
|
||||
}
|
||||
let magic = u64_le(buf, PAGEHDRSZ);
|
||||
if (magic >> 8) != MDBX_MAGIC {
|
||||
eyre::bail!("MDBX magic mismatch");
|
||||
}
|
||||
|
||||
let candidates = [4096usize, 8192, 16384, 32768, 65536];
|
||||
let mut ps = 4096usize;
|
||||
let geo_now_raw = u32_le(buf, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
|
||||
for &candidate in &candidates {
|
||||
let mapped = geo_now_raw * candidate;
|
||||
if mapped >= buf.len() / 2 && mapped <= buf.len() * 4 {
|
||||
ps = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let mut best_txnid: u64 = 0;
|
||||
let mut best_meta_base: usize = 0;
|
||||
for pgno in 0..3usize {
|
||||
let page_base = pgno * ps;
|
||||
let meta_base = page_base + PAGEHDRSZ;
|
||||
if meta_base + 0xC0 > buf.len() { continue; }
|
||||
let pflags = u16_le(buf, page_base + 0x0A);
|
||||
if pflags & P_META == 0 { continue; }
|
||||
let m = u64_le(buf, meta_base);
|
||||
if (m >> 8) != MDBX_MAGIC { continue; }
|
||||
let txnid_a = u64_le(buf, meta_base + META_TXNID_A);
|
||||
let txnid_b = u64_le(buf, meta_base + META_TXNID_B);
|
||||
if txnid_a == txnid_b && txnid_a > best_txnid {
|
||||
best_txnid = txnid_a;
|
||||
best_meta_base = meta_base;
|
||||
}
|
||||
}
|
||||
eyre::ensure!(best_txnid > 0, "No valid meta page found");
|
||||
|
||||
let page_count = u32_le(buf, best_meta_base + META_GEO + GEO_NOW) as usize;
|
||||
let free_tree = TreeDescriptor::parse(buf, best_meta_base + META_FREE_DB);
|
||||
let main_tree = TreeDescriptor::parse(buf, best_meta_base + META_MAIN_DB);
|
||||
|
||||
println!("MDBX owner map generator (parallel)");
|
||||
println!(" page_size: {ps}");
|
||||
println!(" page_count: {page_count}");
|
||||
println!(" txnid: {best_txnid}");
|
||||
println!(" FreeDB root: {} ({} items)", free_tree.root, free_tree.items);
|
||||
println!(" MainDB root: {} ({} items)", main_tree.root, main_tree.items);
|
||||
println!();
|
||||
|
||||
let mut owner_map = vec![UNREFERENCED; page_count];
|
||||
let mut conflicts: u64 = 0;
|
||||
|
||||
for pgno in 0..std::cmp::min(3, page_count) {
|
||||
owner_map[pgno] = DBI_META;
|
||||
}
|
||||
|
||||
println!("Discovering named DBIs via MainDB walk...");
|
||||
let discovered = discover_named_dbis(
|
||||
buf, ps, main_tree.root, &mut owner_map, &mut conflicts, 1, 2,
|
||||
);
|
||||
|
||||
let mut name_to_dbi: HashMap<&str, u8> = HashMap::new();
|
||||
for (idx, table) in Tables::ALL.iter().enumerate() {
|
||||
name_to_dbi.insert(table.name(), (idx + 2) as u8);
|
||||
}
|
||||
|
||||
let mut named_dbis: Vec<DBIInfo> = Vec::new();
|
||||
let mut remap: HashMap<u8, u8> = HashMap::new();
|
||||
for dbi in &discovered {
|
||||
if let Some(&real_dbi) = name_to_dbi.get(dbi.name.as_str()) {
|
||||
remap.insert(dbi.dbi_index, real_dbi);
|
||||
named_dbis.push(DBIInfo {
|
||||
name: dbi.name.clone(),
|
||||
dbi_index: real_dbi,
|
||||
tree: dbi.tree.clone(),
|
||||
});
|
||||
} else {
|
||||
named_dbis.push(dbi.clone());
|
||||
}
|
||||
}
|
||||
|
||||
for byte in owner_map.iter_mut() {
|
||||
if let Some(&new_dbi) = remap.get(byte) {
|
||||
*byte = new_dbi;
|
||||
}
|
||||
}
|
||||
|
||||
println!("Found {} named DBI(s)", named_dbis.len());
|
||||
|
||||
println!("Walking FreeDB / GC...");
|
||||
let free_marked = mark_free_pages(buf, ps, free_tree.root, &mut owner_map, &mut conflicts, 0);
|
||||
println!("FreeDB: {} pages marked", free_marked);
|
||||
|
||||
println!("Walking {} named DBIs in parallel...", named_dbis.len());
|
||||
let t_par = Instant::now();
|
||||
|
||||
let results: Vec<(String, u8, Vec<(usize, u8)>)> = named_dbis
|
||||
.par_iter()
|
||||
.filter(|dbi| dbi.tree.root != P_INVALID)
|
||||
.map(|dbi| {
|
||||
let pages = walk_tree_collect(buf, ps, dbi.tree.root, page_count, dbi.dbi_index);
|
||||
(dbi.name.clone(), dbi.dbi_index, pages)
|
||||
})
|
||||
.collect();
|
||||
|
||||
for (name, dbi_index, pages) in &results {
|
||||
let count = pages.len();
|
||||
for &(pgno, idx) in pages {
|
||||
mark(&mut owner_map, pgno, idx, &mut conflicts);
|
||||
}
|
||||
println!(" [{:2}] {:30} {:>10} pages", dbi_index, name, count);
|
||||
}
|
||||
|
||||
println!("Parallel walk done in {:.2}s", t_par.elapsed().as_secs_f64());
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
|
||||
let mut counts: HashMap<u8, u64> = HashMap::new();
|
||||
let mut unreferenced: u64 = 0;
|
||||
for &b in &owner_map {
|
||||
if b == UNREFERENCED {
|
||||
unreferenced += 1;
|
||||
} else {
|
||||
*counts.entry(b).or_default() += 1;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
println!("Walk complete in {:.2}s", elapsed.as_secs_f64());
|
||||
println!(" Total pages: {page_count}");
|
||||
println!(" Unreferenced: {unreferenced}");
|
||||
println!(" Conflicts: {conflicts}");
|
||||
println!();
|
||||
|
||||
if let Some(&c) = counts.get(&DBI_META) {
|
||||
println!(" [0xFE] {:30} {:>10} pages", "<meta>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&0) {
|
||||
println!(" [ 0] {:30} {:>10} pages", "<free/GC>", c);
|
||||
}
|
||||
if let Some(&c) = counts.get(&1) {
|
||||
println!(" [ 1] {:30} {:>10} pages", "<main>", c);
|
||||
}
|
||||
let mut sorted_dbis = named_dbis.clone();
|
||||
sorted_dbis.sort_by_key(|d| d.dbi_index);
|
||||
for dbi in &sorted_dbis {
|
||||
let walked = counts.get(&dbi.dbi_index).copied().unwrap_or(0);
|
||||
let expected = dbi.tree.total_pages();
|
||||
let mismatch = if expected > 0 && walked != expected {
|
||||
format!(" !! MISMATCH (tree_t says {})", expected)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(" [{:3}] {:30} {:>10} pages{}", dbi.dbi_index, dbi.name, walked, mismatch);
|
||||
}
|
||||
println!(" [0xFF] {:30} {:>10} pages", "<unreferenced>", unreferenced);
|
||||
|
||||
let mut out = fs::File::create(&self.output)?;
|
||||
out.write_all(&owner_map)?;
|
||||
println!();
|
||||
println!("Written {} bytes to {}", owner_map.len(), self.output.display());
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,7 @@ mod settings;
|
||||
mod state;
|
||||
mod static_file_header;
|
||||
mod stats;
|
||||
pub mod gen_ownermap;
|
||||
/// DB List TUI
|
||||
mod tui;
|
||||
|
||||
@@ -71,6 +72,8 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Generate a page owner map for MDBX page visualization
|
||||
GenOwnermap(gen_ownermap::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -214,6 +217,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::GenOwnermap(command) => {
|
||||
db_exec!(self.env, tool, N, AccessRights::RO, {
|
||||
command.execute(data_dir, &tool)?;
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -73,6 +73,7 @@ reth-prune-types = { workspace = true, optional = true }
|
||||
reth-stages = { workspace = true, optional = true }
|
||||
reth-static-file = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
@@ -150,6 +151,7 @@ rocksdb = [
|
||||
"reth-e2e-test-utils/rocksdb",
|
||||
]
|
||||
edge = ["rocksdb"]
|
||||
pageviz = ["dep:reth-mdbx-viz"]
|
||||
|
||||
[[test]]
|
||||
name = "e2e_testsuite"
|
||||
|
||||
@@ -2677,8 +2677,23 @@ where
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(block_num_hash.number, true, 0, 0, 0);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
let pageviz_start = std::time::Instant::now();
|
||||
|
||||
let executed = execute(&mut self.payload_validator, input, ctx)?;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
reth_mdbx_viz::pageviz_emit_block_marker(
|
||||
block_num_hash.number,
|
||||
false,
|
||||
executed.recovered_block().senders().len() as u16,
|
||||
pageviz_start.elapsed().as_millis() as u32,
|
||||
executed.recovered_block().gas_used(),
|
||||
);
|
||||
|
||||
// if the parent is the canonical head, we can insert the block as the pending block
|
||||
if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash()
|
||||
{
|
||||
|
||||
@@ -129,3 +129,4 @@ op = [
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-engine-tree/pageviz"]
|
||||
|
||||
@@ -27,6 +27,8 @@ alloy-primitives.workspace = true
|
||||
# mdbx
|
||||
reth-libmdbx = { workspace = true, optional = true, features = ["return-borrowed", "read-tx-timeouts"] }
|
||||
eyre = { workspace = true, optional = true }
|
||||
reth-mdbx-viz = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, optional = true, features = ["full"] }
|
||||
|
||||
# metrics
|
||||
reth-metrics = { workspace = true, optional = true }
|
||||
@@ -97,6 +99,7 @@ op = [
|
||||
"reth-db-api/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
pageviz = ["reth-libmdbx/pageviz", "dep:reth-mdbx-viz", "dep:tokio", "mdbx"]
|
||||
disable-lock = []
|
||||
|
||||
[[bench]]
|
||||
|
||||
@@ -522,6 +522,81 @@ impl DatabaseEnv {
|
||||
self
|
||||
}
|
||||
|
||||
/// Starts the real-time page access visualization server.
|
||||
///
|
||||
/// This enables the C-level page access instrumentation hooks,
|
||||
/// starts a background drainer thread that coalesces events,
|
||||
/// and launches an HTTP+WebSocket server on the given port.
|
||||
///
|
||||
/// The server serves an interactive HTML visualization at `http://localhost:{port}/`
|
||||
/// showing page accesses in real-time with fading highlights:
|
||||
/// - Blue = read, Red = write, Yellow = free
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub fn start_pageviz(&self, port: u16, db_path: std::path::PathBuf) {
|
||||
use reth_libmdbx::pageviz::PageVizDrainer;
|
||||
|
||||
let max_dbi = self.dbis.values().copied().max().unwrap_or(0) as usize;
|
||||
let mut dbi_names = vec![String::new(); max_dbi + 1];
|
||||
dbi_names[0] = "FREE_DBI".to_string();
|
||||
if dbi_names.len() > 1 {
|
||||
dbi_names[1] = "MAIN_DBI".to_string();
|
||||
}
|
||||
for (name, &dbi) in self.dbis.iter() {
|
||||
dbi_names[dbi as usize] = name.to_string();
|
||||
}
|
||||
|
||||
let mut name_to_dbi = std::collections::HashMap::new();
|
||||
for (&name, &dbi) in self.dbis.iter() {
|
||||
name_to_dbi.insert(name, dbi as u8);
|
||||
}
|
||||
|
||||
let mdbx_dat = db_path.join("mdbx.dat");
|
||||
let walk = match reth_mdbx_viz::walker::walk_mdbx(&mdbx_dat, &name_to_dbi) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
reth_tracing::tracing::error!("pageviz: failed to walk B-tree: {e}");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let page_count = walk.page_count;
|
||||
let page_size = walk.page_size;
|
||||
|
||||
let mut drainer = PageVizDrainer::new();
|
||||
drainer.enable();
|
||||
let rx = drainer.start(60);
|
||||
|
||||
let config = reth_mdbx_viz::VizConfig {
|
||||
port,
|
||||
page_count: page_count as u64,
|
||||
page_size: page_size as u32,
|
||||
dbi_names,
|
||||
owner_map: walk.owner_map,
|
||||
tree_info: walk.tree_info,
|
||||
db_path: mdbx_dat,
|
||||
};
|
||||
|
||||
// Spawn the viz server on a background tokio runtime
|
||||
std::thread::Builder::new()
|
||||
.name("mdbx-viz-server".into())
|
||||
.spawn(move || {
|
||||
let rt = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("failed to build tokio runtime for pageviz");
|
||||
rt.block_on(async {
|
||||
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
|
||||
reth_tracing::tracing::error!("pageviz server error: {e}");
|
||||
}
|
||||
});
|
||||
// Keep drainer alive while server runs
|
||||
drop(drainer);
|
||||
})
|
||||
.expect("failed to spawn pageviz server thread");
|
||||
|
||||
reth_tracing::tracing::info!("pageviz server started on port {port}");
|
||||
}
|
||||
|
||||
/// Creates all the tables defined in [`Tables`], if necessary.
|
||||
///
|
||||
/// This keeps tracks of the created table handles and stores them for better efficiency.
|
||||
|
||||
@@ -45,11 +45,22 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
|
||||
path: P,
|
||||
args: DatabaseArguments,
|
||||
) -> eyre::Result<DatabaseEnv> {
|
||||
let db_path = path.as_ref().to_path_buf();
|
||||
let client_version = args.client_version().clone();
|
||||
let mut db = create_db(path, args)?;
|
||||
db.create_and_track_tables_for::<TS>()?;
|
||||
db.record_client_version(client_version)?;
|
||||
drop_orphan_tables(&db);
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
{
|
||||
let port = std::env::var("RETH_PAGEVIZ_PORT")
|
||||
.ok()
|
||||
.and_then(|s| s.parse().ok())
|
||||
.unwrap_or(3141u16);
|
||||
db.start_pageviz(port, db_path);
|
||||
}
|
||||
|
||||
Ok(db)
|
||||
}
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@ dashmap = { workspace = true, features = ["inline"], optional = true }
|
||||
default = []
|
||||
return-borrowed = []
|
||||
read-tx-timeouts = ["dep:dashmap"]
|
||||
pageviz = ["reth-mdbx-sys/pageviz"]
|
||||
|
||||
[dev-dependencies]
|
||||
criterion.workspace = true
|
||||
|
||||
@@ -11,3 +11,7 @@ repository.workspace = true
|
||||
[build-dependencies]
|
||||
cc.workspace = true
|
||||
bindgen = { workspace = true, features = ["runtime"] }
|
||||
|
||||
[features]
|
||||
default = []
|
||||
pageviz = []
|
||||
|
||||
@@ -30,6 +30,11 @@ fn main() {
|
||||
#[cfg(not(debug_assertions))]
|
||||
cc.define("MDBX_DEBUG", "0").define("NDEBUG", None);
|
||||
|
||||
// Enable page visualization instrumentation
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.define("MDBX_PAGEVIZ", "1");
|
||||
}
|
||||
|
||||
// Propagate `-C target-cpu=native`
|
||||
let rustflags = env::var("CARGO_ENCODED_RUSTFLAGS").unwrap();
|
||||
if rustflags.contains("target-cpu=native") &&
|
||||
@@ -38,7 +43,11 @@ fn main() {
|
||||
cc.flag("-march=native");
|
||||
}
|
||||
|
||||
cc.file(mdbx.join("mdbx.c")).compile("libmdbx.a");
|
||||
cc.file(mdbx.join("mdbx.c"));
|
||||
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
|
||||
cc.file(mdbx.join("mdbx_pageviz.c"));
|
||||
}
|
||||
cc.compile("libmdbx.a");
|
||||
}
|
||||
|
||||
fn generate_bindings(mdbx: &Path, out_file: &Path) {
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
#include MDBX_CONFIG_H
|
||||
#endif
|
||||
|
||||
|
||||
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
|
||||
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
|
||||
#undef NDEBUG
|
||||
@@ -46,6 +47,7 @@
|
||||
#define _DARWIN_C_SOURCE
|
||||
#endif /* _DARWIN_C_SOURCE */
|
||||
|
||||
#include "mdbx_pageviz.h"
|
||||
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
|
||||
#define __USE_MINGW_ANSI_STDIO 1
|
||||
#endif /* MinGW */
|
||||
@@ -6193,6 +6195,7 @@ MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const p
|
||||
static inline int page_touch(MDBX_cursor *mc) {
|
||||
page_t *const mp = mc->pg[mc->top];
|
||||
MDBX_txn *txn = mc->txn;
|
||||
MDBX_PAGEVIZ_WRITE(mc, mp->pgno);
|
||||
|
||||
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
|
||||
@@ -19926,6 +19929,9 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t
|
||||
eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
|
||||
|
||||
if (rc == MDBX_SUCCESS) {
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
eASSERT(env, limit_bytes == env->dxb_mmap.limit);
|
||||
eASSERT(env, size_bytes <= env->dxb_mmap.filesize);
|
||||
if (mode == explicit_resize)
|
||||
@@ -20339,6 +20345,10 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bit
|
||||
if (unlikely(err != MDBX_SUCCESS))
|
||||
return err;
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
|
||||
#endif
|
||||
|
||||
#if defined(MADV_DONTDUMP)
|
||||
err =
|
||||
madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
|
||||
@@ -23090,6 +23100,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
return err;
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -23104,6 +23115,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
|
||||
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
|
||||
tASSERT(txn, lp->flags == P_LOOSE);
|
||||
loose[++count] = lp->pgno;
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
|
||||
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
|
||||
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
|
||||
}
|
||||
@@ -31626,6 +31638,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
|
||||
if (unlikely(r.err != MDBX_SUCCESS))
|
||||
goto bailout;
|
||||
#endif /* MDBX_DISABLE_VALIDATION */
|
||||
MDBX_PAGEVIZ_READ(mc, pgno);
|
||||
return r;
|
||||
}
|
||||
|
||||
@@ -31842,6 +31855,7 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
|
||||
return ret;
|
||||
|
||||
DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = (uint16_t)flags;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -31870,6 +31884,7 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
|
||||
return ret;
|
||||
|
||||
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
|
||||
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
|
||||
ret.page->flags = P_LARGE;
|
||||
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
|
||||
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
|
||||
@@ -32015,6 +32030,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
|
||||
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
|
||||
tASSERT(txn, mp->pgno != pgno);
|
||||
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
|
||||
MDBX_PAGEVIZ_FREE(mc, mp->pgno);
|
||||
/* Update the parent page, if any, to point to the new page */
|
||||
if (likely(mc->top)) {
|
||||
page_t *parent = mc->pg[mc->top - 1];
|
||||
@@ -32211,6 +32227,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
|
||||
unsigned pageflags /* maybe unknown/zero */) {
|
||||
int rc;
|
||||
MDBX_txn *const txn = mc->txn;
|
||||
MDBX_PAGEVIZ_FREE(mc, pgno);
|
||||
tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));
|
||||
|
||||
/* During deleting entire subtrees, it is reasonable and possible to avoid
|
||||
|
||||
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.c
vendored
Normal file
119
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.c
vendored
Normal file
@@ -0,0 +1,119 @@
|
||||
#include "mdbx_pageviz.h"
|
||||
|
||||
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
|
||||
|
||||
/* ── Global state ─────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_global = NULL;
|
||||
|
||||
/* ── Thread-local ring index ──────────────────────────────────────────── */
|
||||
|
||||
MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring = MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
|
||||
/* ── Lifecycle ────────────────────────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void) {
|
||||
mdbx_pageviz_state_t *state = calloc(1, sizeof(mdbx_pageviz_state_t));
|
||||
if (state)
|
||||
mdbx_pageviz_global = state;
|
||||
return state;
|
||||
}
|
||||
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state) {
|
||||
if (!state)
|
||||
return;
|
||||
if (mdbx_pageviz_global == state)
|
||||
mdbx_pageviz_global = NULL;
|
||||
free(state);
|
||||
}
|
||||
|
||||
/* ── Enable / Disable ─────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 1, memory_order_release);
|
||||
}
|
||||
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state) {
|
||||
atomic_store_explicit(&state->enabled, 0, memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Drain ────────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count) {
|
||||
if (!state || ring_idx >= MDBX_PAGEVIZ_MAX_RINGS || !out_buf || max_count == 0)
|
||||
return 0;
|
||||
|
||||
mdbx_pageviz_ring_t *ring = &state->rings[ring_idx];
|
||||
uint32_t head = atomic_load_explicit(&ring->published_head,
|
||||
memory_order_acquire);
|
||||
uint32_t tail = atomic_load_explicit(&ring->consumer_tail,
|
||||
memory_order_relaxed);
|
||||
|
||||
if (head == tail)
|
||||
return 0;
|
||||
|
||||
uint32_t avail = head - tail;
|
||||
uint32_t count = avail < max_count ? avail : max_count;
|
||||
for (uint32_t i = 0; i < count; i++) {
|
||||
uint32_t slot = (tail + i) & MDBX_PAGEVIZ_RING_MASK;
|
||||
out_buf[i] = ring->events[slot];
|
||||
}
|
||||
|
||||
atomic_store_explicit(&ring->consumer_tail, tail + count,
|
||||
memory_order_release);
|
||||
return count;
|
||||
}
|
||||
|
||||
/* ── Queries ──────────────────────────────────────────────────────────── */
|
||||
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state) {
|
||||
return atomic_load_explicit(&state->ring_count, memory_order_relaxed);
|
||||
}
|
||||
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx) {
|
||||
return atomic_load_explicit(&state->rings[ring_idx].dropped,
|
||||
memory_order_relaxed);
|
||||
}
|
||||
|
||||
/* ── Mapping info ────────────────────────────────────────────────────── */
|
||||
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return;
|
||||
state->mapping.mdbx_page_size = mdbx_page_size;
|
||||
state->mapping.sys_page_size = (uint32_t)sysconf(_SC_PAGESIZE);
|
||||
state->mapping.len = len;
|
||||
/* base written last so consumers see consistent len+page_size first */
|
||||
__atomic_store_n((void *volatile *)&state->mapping.base, base, __ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (!state)
|
||||
return 0;
|
||||
void *b = __atomic_load_n((void *volatile *)&state->mapping.base, __ATOMIC_ACQUIRE);
|
||||
if (!b)
|
||||
return 0;
|
||||
*out_base = b;
|
||||
*out_len = state->mapping.len;
|
||||
*out_mdbx_ps = state->mapping.mdbx_page_size;
|
||||
*out_sys_ps = state->mapping.sys_page_size;
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* ── Block marker emit (non-inline wrapper for Rust FFI) ──────────────── */
|
||||
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded) {
|
||||
mdbx_pageviz_emit(op, ((uint32_t)gas_encoded << 24) | ((uint32_t)duration_encoded << 16) | (uint32_t)tx_count, block_number);
|
||||
}
|
||||
|
||||
#else /* !MDBX_PAGEVIZ */
|
||||
|
||||
typedef int mdbx_pageviz_empty_tu_;
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
195
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.h
vendored
Normal file
195
crates/storage/libmdbx-rs/mdbx-sys/libmdbx/mdbx_pageviz.h
vendored
Normal file
@@ -0,0 +1,195 @@
|
||||
/* mdbx_pageviz.h — Standalone page-access visualization ring buffer for MDBX.
|
||||
*
|
||||
* Self-contained: no dependencies on MDBX internals.
|
||||
* Gate everything behind MDBX_PAGEVIZ; when not defined, all macros expand to
|
||||
* nothing (zero overhead).
|
||||
*
|
||||
* Compatible with C11 and C17. */
|
||||
|
||||
#ifndef MDBX_PAGEVIZ_H
|
||||
#define MDBX_PAGEVIZ_H
|
||||
|
||||
/* ── Disabled stub macros ─────────────────────────────────────────────── */
|
||||
|
||||
#if !defined(MDBX_PAGEVIZ) || !(MDBX_PAGEVIZ)
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) ((void)0)
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) ((void)0)
|
||||
|
||||
#else /* MDBX_PAGEVIZ enabled ───────────────────────────────────────────── */
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <unistd.h>
|
||||
|
||||
/* ── Tunables ─────────────────────────────────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_CAPACITY 65536u /* must be power-of-2 */
|
||||
#define MDBX_PAGEVIZ_MAX_RINGS 128u
|
||||
#define MDBX_PAGEVIZ_PUBLISH_INTERVAL 32u
|
||||
|
||||
/* ── Event encoding ───────────────────────────────────────────────────── */
|
||||
|
||||
/* Layout of a 64-bit event word:
|
||||
* bits 63..56 op (1=READ, 2=WRITE, 3=FREE)
|
||||
* bits 55..32 dbi (lower 24 bits of a uint32_t)
|
||||
* bits 31..0 pgno (uint32_t page number) */
|
||||
|
||||
#define MDBX_PAGEVIZ_OP_READ 1
|
||||
#define MDBX_PAGEVIZ_OP_WRITE 2
|
||||
#define MDBX_PAGEVIZ_OP_FREE 3
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_START 4
|
||||
#define MDBX_PAGEVIZ_OP_BLOCK_END 5
|
||||
|
||||
#define MDBX_PAGEVIZ_ENCODE(op, dbi, pgno) \
|
||||
(((uint64_t)(op) << 56) | ((uint64_t)((dbi) & 0x00FFFFFFu) << 32) | \
|
||||
(uint64_t)(uint32_t)(pgno))
|
||||
|
||||
#define MDBX_PAGEVIZ_DECODE_OP(ev) ((uint8_t)((ev) >> 56))
|
||||
#define MDBX_PAGEVIZ_DECODE_DBI(ev) ((uint32_t)(((ev) >> 32) & 0x00FFFFFFu))
|
||||
#define MDBX_PAGEVIZ_DECODE_PGNO(ev) ((uint32_t)(ev))
|
||||
|
||||
/* ── Ring-buffer mask (power-of-2 capacity) ───────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_RING_MASK (MDBX_PAGEVIZ_RING_CAPACITY - 1u)
|
||||
|
||||
/* ── Platform TLS ─────────────────────────────────────────────────────── */
|
||||
|
||||
#if defined(_MSC_VER)
|
||||
# define MDBX_PAGEVIZ_TLS __declspec(thread)
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
# define MDBX_PAGEVIZ_TLS __thread
|
||||
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
|
||||
# define MDBX_PAGEVIZ_TLS _Thread_local
|
||||
#else
|
||||
# error "No thread-local storage support detected"
|
||||
#endif
|
||||
|
||||
/* ── Structures ───────────────────────────────────────────────────────── */
|
||||
|
||||
typedef struct mdbx_pageviz_ring {
|
||||
uint64_t events[MDBX_PAGEVIZ_RING_CAPACITY];
|
||||
_Atomic uint32_t published_head; /* consumer reads (acquire) */
|
||||
uint32_t local_head; /* producer-only, no atomic needed */
|
||||
_Atomic uint32_t consumer_tail; /* consumer updates after draining */
|
||||
_Atomic uint64_t dropped; /* unused, kept for ABI compat */
|
||||
uint32_t _pad[10]; /* pad to separate cache lines */
|
||||
} mdbx_pageviz_ring_t;
|
||||
|
||||
typedef struct mdbx_pageviz_mapping {
|
||||
volatile void *base;
|
||||
volatile size_t len;
|
||||
volatile uint32_t mdbx_page_size;
|
||||
volatile uint32_t sys_page_size;
|
||||
} mdbx_pageviz_mapping_t;
|
||||
|
||||
typedef struct mdbx_pageviz_state {
|
||||
mdbx_pageviz_ring_t rings[MDBX_PAGEVIZ_MAX_RINGS];
|
||||
_Atomic uint32_t ring_count; /* how many rings registered */
|
||||
_Atomic uint32_t enabled; /* runtime enable/disable */
|
||||
mdbx_pageviz_mapping_t mapping;
|
||||
} mdbx_pageviz_state_t;
|
||||
|
||||
/* ── Global state (defined in .c) ─────────────────────────────────────── */
|
||||
|
||||
extern mdbx_pageviz_state_t *mdbx_pageviz_global;
|
||||
|
||||
/* ── Thread-local ring index (defined in .c) ──────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_TLS_UNREGISTERED UINT32_MAX
|
||||
|
||||
extern MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring;
|
||||
|
||||
/* ── Inline helpers (private) ─────────────────────────────────────────── */
|
||||
|
||||
static inline uint32_t mdbx_pageviz_register_ring_(mdbx_pageviz_state_t *s) {
|
||||
uint32_t idx =
|
||||
atomic_fetch_add_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
if (idx >= MDBX_PAGEVIZ_MAX_RINGS) {
|
||||
atomic_fetch_sub_explicit(&s->ring_count, 1, memory_order_relaxed);
|
||||
return MDBX_PAGEVIZ_TLS_UNREGISTERED;
|
||||
}
|
||||
mdbx_pageviz_ring_t *r = &s->rings[idx];
|
||||
atomic_store_explicit(&r->published_head, 0, memory_order_relaxed);
|
||||
r->local_head = 0;
|
||||
atomic_store_explicit(&r->consumer_tail, 0, memory_order_relaxed);
|
||||
atomic_store_explicit(&r->dropped, 0, memory_order_relaxed);
|
||||
return idx;
|
||||
}
|
||||
|
||||
/* ── Hot-path emit (inlined) ──────────────────────────────────────────── */
|
||||
|
||||
static inline void mdbx_pageviz_emit(uint8_t op, uint32_t dbi,
|
||||
uint32_t pgno) {
|
||||
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
|
||||
if (__builtin_expect(
|
||||
state == NULL ||
|
||||
!atomic_load_explicit(&state->enabled, memory_order_relaxed),
|
||||
1))
|
||||
return;
|
||||
|
||||
uint32_t idx = mdbx_pageviz_tls_ring;
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0)) {
|
||||
idx = mdbx_pageviz_register_ring_(state);
|
||||
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0))
|
||||
return;
|
||||
mdbx_pageviz_tls_ring = idx;
|
||||
}
|
||||
|
||||
mdbx_pageviz_ring_t *r = &state->rings[idx];
|
||||
|
||||
/* Block (spin) until the consumer drains enough space. */
|
||||
while (__builtin_expect(
|
||||
(uint32_t)(r->local_head -
|
||||
atomic_load_explicit(&r->consumer_tail, memory_order_acquire))
|
||||
>= MDBX_PAGEVIZ_RING_CAPACITY, 0)) {
|
||||
/* Flush our writes so the consumer can make progress. */
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
|
||||
__builtin_ia32_pause();
|
||||
#elif defined(__aarch64__) || defined(_M_ARM64)
|
||||
__asm__ volatile("yield");
|
||||
#endif
|
||||
}
|
||||
|
||||
r->events[r->local_head & MDBX_PAGEVIZ_RING_MASK] =
|
||||
MDBX_PAGEVIZ_ENCODE(op, dbi, pgno);
|
||||
r->local_head++;
|
||||
|
||||
if ((r->local_head & (MDBX_PAGEVIZ_PUBLISH_INTERVAL - 1u)) == 0)
|
||||
atomic_store_explicit(&r->published_head, r->local_head,
|
||||
memory_order_release);
|
||||
}
|
||||
|
||||
/* ── Macro hooks (inserted into mdbx.c) ──────────────────────────────── */
|
||||
|
||||
#define MDBX_PAGEVIZ_READ(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_READ, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_WRITE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_WRITE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
#define MDBX_PAGEVIZ_FREE(mc, pgno) \
|
||||
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, (uint32_t)cursor_dbi(mc), (pgno))
|
||||
|
||||
/* ── Public API (defined in .c) ───────────────────────────────────────── */
|
||||
|
||||
mdbx_pageviz_state_t *mdbx_pageviz_create(void);
|
||||
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state);
|
||||
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state);
|
||||
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
|
||||
uint64_t *out_buf, uint32_t max_count);
|
||||
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state);
|
||||
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx);
|
||||
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size);
|
||||
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
|
||||
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps);
|
||||
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
|
||||
uint16_t tx_count, uint8_t duration_encoded,
|
||||
uint8_t gas_encoded);
|
||||
|
||||
#endif /* MDBX_PAGEVIZ */
|
||||
#endif /* MDBX_PAGEVIZ_H */
|
||||
@@ -36,6 +36,9 @@ mod flags;
|
||||
mod transaction;
|
||||
mod txn_manager;
|
||||
|
||||
#[cfg(feature = "pageviz")]
|
||||
pub mod pageviz;
|
||||
|
||||
#[cfg(test)]
|
||||
mod test_utils {
|
||||
use super::*;
|
||||
|
||||
226
crates/storage/libmdbx-rs/src/pageviz.rs
Normal file
226
crates/storage/libmdbx-rs/src/pageviz.rs
Normal file
@@ -0,0 +1,226 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc, Arc,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
#[repr(u8)]
|
||||
pub enum PageOp {
|
||||
Read = 1,
|
||||
Write = 2,
|
||||
Free = 3,
|
||||
BlockStart = 4,
|
||||
BlockEnd = 5,
|
||||
}
|
||||
|
||||
impl PageOp {
|
||||
fn from_u8(v: u8) -> Option<Self> {
|
||||
match v {
|
||||
1 => Some(Self::Read),
|
||||
2 => Some(Self::Write),
|
||||
3 => Some(Self::Free),
|
||||
4 => Some(Self::BlockStart),
|
||||
5 => Some(Self::BlockEnd),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_block_marker(self) -> bool {
|
||||
matches!(self, Self::BlockStart | Self::BlockEnd)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct PageEvent {
|
||||
pub pgno: u32,
|
||||
pub dbi: u32,
|
||||
pub op: PageOp,
|
||||
}
|
||||
|
||||
impl PageEvent {
|
||||
fn decode(raw: u64) -> Option<Self> {
|
||||
let op_byte = (raw >> 56) as u8;
|
||||
let op = PageOp::from_u8(op_byte)?;
|
||||
let dbi = ((raw >> 32) & 0x00FF_FFFF) as u32;
|
||||
let pgno = raw as u32;
|
||||
Some(Self { pgno, dbi, op })
|
||||
}
|
||||
}
|
||||
|
||||
unsafe extern "C" {
|
||||
fn mdbx_pageviz_create() -> *mut std::ffi::c_void;
|
||||
fn mdbx_pageviz_destroy(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_enable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_disable(state: *mut std::ffi::c_void);
|
||||
fn mdbx_pageviz_drain(
|
||||
state: *mut std::ffi::c_void,
|
||||
ring_idx: u32,
|
||||
out_buf: *mut u64,
|
||||
max_count: u32,
|
||||
) -> u32;
|
||||
fn mdbx_pageviz_ring_count(state: *mut std::ffi::c_void) -> u32;
|
||||
fn mdbx_pageviz_dropped(state: *mut std::ffi::c_void, ring_idx: u32) -> u64;
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizStats {
|
||||
pub ring_count: u32,
|
||||
pub total_dropped: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct StatePtr(*mut std::ffi::c_void);
|
||||
|
||||
// SAFETY: The C state is thread-safe (uses atomics internally).
|
||||
unsafe impl Send for StatePtr {}
|
||||
unsafe impl Sync for StatePtr {}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct PageVizDrainer {
|
||||
state: StatePtr,
|
||||
running: Arc<AtomicBool>,
|
||||
handle: Option<std::thread::JoinHandle<()>>,
|
||||
}
|
||||
|
||||
const DRAIN_BUF_LEN: usize = 4096;
|
||||
const OVERLOAD_THRESHOLD: usize = 200_000;
|
||||
const READ_SAMPLE_RATE: usize = 8;
|
||||
|
||||
impl PageVizDrainer {
|
||||
pub fn new() -> Self {
|
||||
let state = StatePtr(unsafe { mdbx_pageviz_create() });
|
||||
Self { state, running: Arc::new(AtomicBool::new(false)), handle: None }
|
||||
}
|
||||
|
||||
pub fn enable(&self) {
|
||||
unsafe { mdbx_pageviz_enable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn disable(&self) {
|
||||
unsafe { mdbx_pageviz_disable(self.state.0) }
|
||||
}
|
||||
|
||||
pub fn start(&mut self, tick_hz: u32) -> mpsc::Receiver<Vec<PageEvent>> {
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let running = self.running.clone();
|
||||
running.store(true, Ordering::SeqCst);
|
||||
|
||||
let state_addr = self.state.0 as usize;
|
||||
let tick_interval = Duration::from_micros(1_000_000 / u64::from(tick_hz));
|
||||
|
||||
let handle = std::thread::Builder::new()
|
||||
.name("mdbx-pageviz".into())
|
||||
.spawn(move || {
|
||||
let state = state_addr as *mut std::ffi::c_void;
|
||||
let mut buf = [0u64; DRAIN_BUF_LEN];
|
||||
let mut coalesced: HashMap<u32, PageEvent> = HashMap::new();
|
||||
let mut markers: Vec<PageEvent> = Vec::new();
|
||||
let mut read_counter: usize = 0;
|
||||
|
||||
while running.load(Ordering::Relaxed) {
|
||||
let tick_start = Instant::now();
|
||||
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(state) };
|
||||
|
||||
for ring_idx in 0..ring_count {
|
||||
let dropped = unsafe { mdbx_pageviz_dropped(state, ring_idx) };
|
||||
if dropped > 0 {
|
||||
tracing::warn!(
|
||||
target: "libmdbx::pageviz",
|
||||
ring_idx,
|
||||
dropped,
|
||||
"pageviz ring dropped events"
|
||||
);
|
||||
}
|
||||
|
||||
loop {
|
||||
let count = unsafe {
|
||||
mdbx_pageviz_drain(
|
||||
state,
|
||||
ring_idx,
|
||||
buf.as_mut_ptr(),
|
||||
DRAIN_BUF_LEN as u32,
|
||||
)
|
||||
};
|
||||
if count == 0 {
|
||||
break;
|
||||
}
|
||||
for &raw in &buf[..count as usize] {
|
||||
if let Some(evt) = PageEvent::decode(raw) {
|
||||
if evt.op.is_block_marker() {
|
||||
markers.push(evt);
|
||||
} else {
|
||||
coalesced.insert(evt.pgno, evt);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !coalesced.is_empty() || !markers.is_empty() {
|
||||
let mut events: Vec<PageEvent> = if coalesced.len() > OVERLOAD_THRESHOLD {
|
||||
coalesced
|
||||
.drain()
|
||||
.filter(|(_, evt)| {
|
||||
if evt.op == PageOp::Read {
|
||||
read_counter += 1;
|
||||
read_counter % READ_SAMPLE_RATE == 0
|
||||
} else {
|
||||
true
|
||||
}
|
||||
})
|
||||
.map(|(_, evt)| evt)
|
||||
.collect()
|
||||
} else {
|
||||
coalesced.drain().map(|(_, evt)| evt).collect()
|
||||
};
|
||||
|
||||
events.extend(markers.drain(..));
|
||||
|
||||
if tx.send(events).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let elapsed = tick_start.elapsed();
|
||||
if let Some(remaining) = tick_interval.checked_sub(elapsed) {
|
||||
std::thread::sleep(remaining);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn mdbx-pageviz thread");
|
||||
|
||||
self.handle = Some(handle);
|
||||
rx
|
||||
}
|
||||
|
||||
pub fn stop(&mut self) {
|
||||
self.running.store(false, Ordering::SeqCst);
|
||||
if let Some(handle) = self.handle.take() {
|
||||
let _ = handle.join();
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> PageVizStats {
|
||||
let ring_count = unsafe { mdbx_pageviz_ring_count(self.state.0) };
|
||||
let mut total_dropped = 0u64;
|
||||
for ring_idx in 0..ring_count {
|
||||
total_dropped += unsafe { mdbx_pageviz_dropped(self.state.0, ring_idx) };
|
||||
}
|
||||
PageVizStats { ring_count, total_dropped }
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for PageVizDrainer {
|
||||
fn drop(&mut self) {
|
||||
self.stop();
|
||||
unsafe {
|
||||
mdbx_pageviz_disable(self.state.0);
|
||||
mdbx_pageviz_destroy(self.state.0);
|
||||
}
|
||||
}
|
||||
}
|
||||
1381
crates/storage/mdbx-viz/Cargo.lock
generated
Normal file
1381
crates/storage/mdbx-viz/Cargo.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
23
crates/storage/mdbx-viz/Cargo.toml
Normal file
23
crates/storage/mdbx-viz/Cargo.toml
Normal file
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "reth-mdbx-viz"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license = "Apache-2.0"
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Real-time MDBX page access visualization server"
|
||||
|
||||
[dependencies]
|
||||
reth-libmdbx = { workspace = true, features = ["pageviz"] }
|
||||
reth-mdbx-sys = { workspace = true, features = ["pageviz"] }
|
||||
axum = { version = "0.7", features = ["ws"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
clap = { version = "4", features = ["derive"] }
|
||||
tracing.workspace = true
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
rayon.workspace = true
|
||||
eyre.workspace = true
|
||||
libc = "0.2"
|
||||
952
crates/storage/mdbx-viz/src/index.html
Normal file
952
crates/storage/mdbx-viz/src/index.html
Normal file
@@ -0,0 +1,952 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>MDBX Live Viz</title>
|
||||
<style>
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
html,body{background:#0d1117;color:#c9d1d9;font-family:'SF Mono','Fira Code','Cascadia Code',monospace;height:100%;overflow:hidden}
|
||||
body{display:flex;flex-direction:row;height:100%}
|
||||
#main-panel{flex:1;display:flex;flex-direction:column;padding:16px;overflow:hidden;min-width:0}
|
||||
#side-panel{flex:0 0 540px;border-left:1px solid #30363d;overflow-y:auto;padding:16px;background:#161b22}
|
||||
h1{color:#f0f6fc;font-size:18px;margin-bottom:2px}
|
||||
.subtitle{color:#8b949e;font-size:12px;margin-bottom:10px}
|
||||
.sub-row{display:flex;align-items:baseline;gap:12px;margin-bottom:6px;flex-wrap:wrap}
|
||||
.sub-row .subtitle{margin-bottom:0;flex-shrink:0}
|
||||
.sub-row .viz-controls{margin-bottom:0;margin-left:auto}
|
||||
.controls{margin-bottom:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap}
|
||||
.controls button{background:#21262d;border:1px solid #30363d;color:#c9d1d9;padding:3px 10px;border-radius:4px;cursor:pointer;font-family:inherit;font-size:12px}
|
||||
.controls button:hover{background:#30363d}
|
||||
.zoom-info{font-size:11px;color:#8b949e}
|
||||
.status-bar{display:flex;gap:12px;align-items:center;margin-bottom:6px;font-size:11px}
|
||||
.dot{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:4px;vertical-align:middle}
|
||||
.dot.connected{background:#4ade80}.dot.disconnected{background:#f87171}.dot.connecting{background:#facc15}
|
||||
.stat-label{color:#8b949e}.stat-value{color:#c9d1d9;font-weight:600}
|
||||
#canvas-wrap{border:1px solid #30363d;border-radius:6px;overflow:hidden;background:#000;position:relative;cursor:crosshair;flex:1;min-height:0}
|
||||
canvas{display:block;image-rendering:pixelated;width:100%;height:100%}
|
||||
#tooltip{position:fixed;background:#1c2128;border:1px solid #30363d;border-radius:6px;padding:8px 12px;font-size:12px;pointer-events:none;display:none;z-index:100;box-shadow:0 4px 12px rgba(0,0,0,.4);max-width:360px}
|
||||
#tooltip .tt-row{display:flex;align-items:center;gap:6px}
|
||||
#tooltip .tt-swatch{width:10px;height:10px;border-radius:2px;flex-shrink:0;border:1px solid rgba(255,255,255,0.2)}
|
||||
#tooltip .tt-name{color:#f0f6fc;font-weight:600}
|
||||
#tooltip .tt-detail{color:#8b949e;margin-top:2px}
|
||||
#side-panel h2{color:#f0f6fc;font-size:14px;margin-bottom:8px}
|
||||
.viz-controls{margin-bottom:12px;font-size:11px}
|
||||
.viz-controls label{display:inline-flex;align-items:center;gap:4px;margin-right:10px;cursor:pointer;color:#8b949e}
|
||||
.viz-controls label:hover{color:#c9d1d9}
|
||||
.viz-controls input[type=checkbox]{accent-color:#58a6ff}
|
||||
.viz-controls input[type=range]{width:100px;accent-color:#58a6ff}
|
||||
.fade-val{color:#c9d1d9;min-width:35px;display:inline-block}
|
||||
table{width:100%;border-collapse:collapse;font-size:11px;background:#0d1117;border:1px solid #30363d;border-radius:6px;overflow:hidden;table-layout:fixed}
|
||||
td,th{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
th{text-align:left;padding:6px 8px;background:#1c2128;color:#8b949e;font-weight:500;border-bottom:1px solid #30363d;font-size:10px;text-transform:uppercase;position:sticky;top:0}
|
||||
td{padding:5px 8px;border-bottom:1px solid #21262d}
|
||||
tr.tbl-row{cursor:pointer;transition:background 0.1s}
|
||||
tr.tbl-row:hover td{background:#1c2128}
|
||||
tr.tbl-row.active td{background:#1c2128;outline:1px solid #58a6ff}
|
||||
.pct-bar{display:inline-block;height:6px;border-radius:3px;margin-right:6px;vertical-align:middle}
|
||||
.swatch-cell{width:12px;height:12px;border-radius:2px;flex-shrink:0}
|
||||
#loading{display:flex;flex-direction:column;align-items:center;justify-content:center;height:100%;color:#8b949e;font-size:14px;gap:12px}
|
||||
#load-bar-wrap{width:280px;height:6px;background:#21262d;border-radius:3px;overflow:hidden;display:none}
|
||||
#load-bar{height:100%;width:0;background:#58a6ff;border-radius:3px;transition:width 0.15s}
|
||||
#flame-wrap{border:1px solid #30363d;border-radius:0 0 6px 6px;margin-top:-1px;background:#0d1117}
|
||||
#flame-wrap canvas{display:block}
|
||||
.spark-row{display:flex;align-items:center;gap:6px;margin-bottom:3px;font-size:10px}
|
||||
.spark-label{width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
|
||||
.spark-cv{border:1px solid #21262d;border-radius:2px;background:#0d1117}
|
||||
.spark-val{color:#c9d1d9;min-width:30px;text-align:right}
|
||||
.tree-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:4px;font-size:10px;cursor:pointer}
|
||||
.tree-card:hover{border-color:#58a6ff}
|
||||
.tree-card .tree-name{color:#f0f6fc;font-weight:600;font-size:11px}
|
||||
.tree-card .tree-stat{color:#8b949e;margin-top:2px}
|
||||
.tree-card .tree-bar{display:flex;gap:1px;margin-top:3px;height:6px;border-radius:2px;overflow:hidden}
|
||||
.tree-card .tree-seg{height:100%}
|
||||
.cache-stat{display:flex;justify-content:space-between;align-items:center;font-size:10px;padding:2px 0}
|
||||
.cache-bar{height:4px;border-radius:2px;background:#21262d;flex:1;margin:0 6px;position:relative;overflow:hidden}
|
||||
.cache-fill{height:100%;border-radius:2px;background:#4ade80;position:absolute;left:0;top:0}
|
||||
.cache-pct{color:#4ade80;font-weight:600;min-width:36px;text-align:right}
|
||||
.cache-label{color:#8b949e;min-width:80px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
.fault-cnt{color:#d946ef;font-size:10px;margin-left:4px}
|
||||
.block-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:3px;font-size:10px;cursor:pointer;transition:border-color 0.15s}
|
||||
.block-card:hover{border-color:#58a6ff}
|
||||
.block-card.replaying{border-color:#f59e0b;background:#1e1b13}
|
||||
.block-card.recording{border-color:#4ade80;animation:pulse-border 1s infinite}
|
||||
.block-card .block-num{color:#f0f6fc;font-weight:600;font-size:11px}
|
||||
.block-card .block-stat{color:#8b949e;margin-top:1px}
|
||||
@keyframes pulse-border{0%,100%{border-color:#4ade80}50%{border-color:#166534}}
|
||||
#block-live{cursor:pointer;font-size:10px;margin-left:6px;user-select:none}
|
||||
.utbl-row{display:flex;align-items:center;gap:6px;padding:3px 4px;font-size:10px;cursor:pointer}
|
||||
.utbl-row:hover{background:#1c2128}
|
||||
.utbl-row.active{background:#1c2128;outline:1px solid #58a6ff}
|
||||
.utbl-sw{width:8px;height:8px;border-radius:2px;flex-shrink:0}
|
||||
.utbl-name{width:110px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
|
||||
.utbl-spark{flex-shrink:0}
|
||||
.utbl-cache{flex:1;min-width:0}
|
||||
.utbl-cache-bar-wrap{width:100%;height:6px;background:#21262d;border-radius:3px;overflow:hidden}
|
||||
.utbl-cache-fill{height:100%;border-radius:3px;transition:width 0.3s}
|
||||
.utbl-cache-pct{text-align:center;font-size:9px;color:#4ade80;font-weight:600;margin-top:1px;line-height:1}
|
||||
.utbl-diff{width:52px;text-align:right;font-size:9px}
|
||||
.utbl-faults{width:36px;text-align:right;color:#d946ef;font-size:9px}
|
||||
#block-panel::-webkit-scrollbar,#side-panel::-webkit-scrollbar{width:6px}
|
||||
#block-panel::-webkit-scrollbar-track,#side-panel::-webkit-scrollbar-track{background:#161b22}
|
||||
#block-panel::-webkit-scrollbar-thumb,#side-panel::-webkit-scrollbar-thumb{background:#30363d;border-radius:3px}
|
||||
#block-panel::-webkit-scrollbar-thumb:hover,#side-panel::-webkit-scrollbar-thumb:hover{background:#484f58}
|
||||
#block-panel,#side-panel{scrollbar-width:thin;scrollbar-color:#30363d #161b22}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="main-panel">
|
||||
<h1>MDBX Live Viz</h1>
|
||||
<div class="sub-row">
|
||||
<div class="subtitle" id="subtitle">Loading...</div>
|
||||
<div class="viz-controls">
|
||||
<label><input type="checkbox" id="chk-read" checked> <span style="color:#3c82f6">■</span> Reads</label>
|
||||
<label><input type="checkbox" id="chk-write" checked> <span style="color:#ef4444">■</span> Writes</label>
|
||||
<label><input type="checkbox" id="chk-free" checked> <span style="color:#eab308">■</span> Frees</label>
|
||||
<label><input type="checkbox" id="chk-fixw"> Fixed Width</label>
|
||||
<label><input type="checkbox" id="chk-trails" checked> Trails</label>
|
||||
<label><input type="checkbox" id="chk-heatmap"> Heatmap</label>
|
||||
<label><input type="checkbox" id="chk-flame"> Timeline</label>
|
||||
<label><input type="checkbox" id="chk-cache"> Page Cache</label>
|
||||
<label>Fade: <input type="range" id="fade-slider" min="200" max="8000" value="2000" step="100"> <span class="fade-val" id="fade-val">2.0s</span></label>
|
||||
<label>Trail W: <input type="range" id="trail-width" min="0.5" max="6" value="1.0" step="0.5"> <span class="fade-val" id="trail-w-val">1.0</span></label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="status-bar">
|
||||
<span><span class="dot connecting" id="ws-dot"></span><span id="ws-status">Connecting</span></span>
|
||||
<span class="stat-label">Events/s: <span class="stat-value" id="evt-rate">0</span></span>
|
||||
<span class="stat-label">Total: <span class="stat-value" id="evt-total">0</span></span>
|
||||
<span class="stat-label">Active: <span class="stat-value" id="active-count">0</span></span>
|
||||
<span class="stat-label">FPS: <span class="stat-value" id="fps-counter">-</span></span>
|
||||
</div>
|
||||
<div class="status-bar">
|
||||
<span class="stat-label" style="color:#3c82f6">R: <span class="stat-value" id="cnt-read">0</span></span>
|
||||
<span class="stat-label" style="color:#ef4444">W: <span class="stat-value" id="cnt-write">0</span></span>
|
||||
<span class="stat-label" style="color:#eab308">F: <span class="stat-value" id="cnt-free">0</span></span>
|
||||
<span class="stat-label">Latest: <span class="stat-value" id="evt-latest">-</span></span>
|
||||
<span id="lag-warn" style="display:none;color:#f87171;font-weight:600">⚠ Lagged <span id="lag-count">0</span> msgs</span>
|
||||
</div>
|
||||
<div class="controls">
|
||||
<button id="btn-zin">Zoom +</button>
|
||||
<button id="btn-zout">Zoom −</button>
|
||||
<button id="btn-reset">Reset</button>
|
||||
<button id="btn-clear">Clear Highlights</button>
|
||||
<span class="zoom-info" id="zoom-info"></span>
|
||||
</div>
|
||||
<div id="canvas-wrap">
|
||||
<div id="loading"><span id="load-text">Loading page map…</span><div id="load-bar-wrap"><div id="load-bar"></div></div></div>
|
||||
<canvas id="map" style="display:none"></canvas>
|
||||
</div>
|
||||
<div id="flame-wrap" style="height:0;overflow:hidden;transition:height 0.2s">
|
||||
<canvas id="flame"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div id="side-panel">
|
||||
<h2>Blocks <span id="block-live" style="color:#4ade80" onclick="window._goLive&&window._goLive()">● LIVE</span></h2>
|
||||
<div id="block-panel" style="max-height:172px;overflow-y:auto"></div>
|
||||
<h2 style="margin-top:12px">Tables <span id="cache-total" style="color:#8b949e;font-weight:400;font-size:11px"></span> <span id="cache-ago" style="color:#8b949e;font-weight:400;font-size:10px"></span></h2>
|
||||
<div id="unified-tables"></div>
|
||||
<h2 style="margin-top:12px">Table Details</h2>
|
||||
<table><thead><tr><th></th><th>Table</th><th>Pages</th><th>Size</th><th>%</th></tr></thead><tbody id="tbl-body"></tbody></table>
|
||||
<h2 style="margin-top:12px">B-Tree Structure</h2>
|
||||
<div id="tree-info"></div>
|
||||
</div>
|
||||
<div id="tooltip">
|
||||
<div class="tt-row"><div class="tt-swatch" id="tt-swatch"></div><div class="tt-name" id="tt-name"></div></div>
|
||||
<div class="tt-detail" id="tt-detail"></div>
|
||||
</div>
|
||||
<script>
|
||||
(function(){
|
||||
"use strict";
|
||||
function hsl2r(h,s,l){h/=360;s/=100;l/=100;let r,g,b;if(!s){r=g=b=l}else{const q=l<.5?l*(1+s):l+s-l*s,p=2*l-q;const f=(p,q,t)=>{if(t<0)t+=1;if(t>1)t-=1;if(t<1/6)return p+(q-p)*6*t;if(t<.5)return q;if(t<2/3)return p+(q-p)*(2/3-t)*6;return p};r=f(p,q,h+1/3);g=f(p,q,h);b=f(p,q,h-1/3)}return[Math.round(r*255),Math.round(g*255),Math.round(b*255)]}
|
||||
const DBI_HUES=new Float32Array(256);
|
||||
for(let i=2;i<254;i++)DBI_HUES[i]=((i-2)*137.508)%360;
|
||||
const PAL=new Array(256),PAL_DESAT=new Array(256),PAL_DARK=new Array(256),PAL_DD=new Array(256);
|
||||
PAL[0]=hsl2r(220,15,82);PAL_DESAT[0]=hsl2r(220,5,72);PAL_DARK[0]=hsl2r(220,15,28);PAL_DD[0]=hsl2r(220,5,22);
|
||||
PAL[1]=hsl2r(220,30,24);PAL_DESAT[1]=hsl2r(220,8,22);PAL_DARK[1]=hsl2r(220,30,12);PAL_DD[1]=hsl2r(220,8,12);
|
||||
for(let i=2;i<254;i++){const h=DBI_HUES[i];PAL[i]=hsl2r(h,70,55);PAL_DESAT[i]=hsl2r(h,15,50);PAL_DARK[i]=hsl2r(h,70,22);PAL_DD[i]=hsl2r(h,15,20)}
|
||||
PAL[0xFE]=[20,20,23];PAL_DESAT[0xFE]=[20,20,23];PAL_DARK[0xFE]=[12,12,14];PAL_DD[0xFE]=[12,12,14];
|
||||
PAL[0xFF]=[28,28,32];PAL_DESAT[0xFF]=[28,28,32];PAL_DARK[0xFF]=[16,16,18];PAL_DD[0xFF]=[16,16,18];
|
||||
const FLASH_TINT={1:[140,180,255],2:[255,140,140],3:[255,220,100]};
|
||||
let NP=0,PS=4096,GC=0,GR=0,DBIS=[],DBI_IDX={},oMap=null,lOp=null,lTs=null,FADE=2000;
|
||||
let cntR=0,cntW=0,cntF=0;
|
||||
let sR=true,sW=true,sF=true,fW=false,showTrails=true,trailW=1.0;
|
||||
const TRAIL_CAP=4096;let trailBuf=new Array(TRAIL_CAP);let trailStart=0,trailN=0;
|
||||
const TRC={1:"60,130,246",2:"255,80,80",3:"234,179,8"};
|
||||
let activePgs=0;
|
||||
const activeSet=new Set();
|
||||
let oMapDirty=false;
|
||||
let unrefCount=0;
|
||||
let outR=0,outG=0,outB=0;
|
||||
let cacheTotalCached=0,cacheCachedPer=null,cachePrevPer=null,cachePrevTotal=0,cacheStale=true;
|
||||
let hmMode=false,hmCounts=null;
|
||||
let cacheMode=false,resMap=null,faultCounts={},faultWindow=[],faultCurrent={};
|
||||
let cacheLastUpdate=0;
|
||||
let zbCv=null,zbCx=null,zbDirty=true,zbW=0,zbH=0,zbPX=0,zbPY=0,zbSc=0,zbFlt=false;
|
||||
const BASE_LUT=new Uint32Array(256*32*2);
|
||||
function buildLUT(){
|
||||
for(let dbi=0;dbi<256;dbi++){for(let hq=0;hq<32;hq++){const ht=hq/31;for(let c=0;c<2;c++){
|
||||
let cf,cd;
|
||||
if(c){cf=PAL[dbi];cd=PAL_DESAT[dbi]}else{cf=PAL_DARK[dbi];cd=PAL_DD[dbi]}
|
||||
if(!cf)cf=PAL[255];if(!cd)cd=PAL_DESAT[255];
|
||||
const r=Math.round(cd[0]+(cf[0]-cd[0])*ht);
|
||||
const g=Math.round(cd[1]+(cf[1]-cd[1])*ht);
|
||||
const b=Math.round(cd[2]+(cf[2]-cd[2])*ht);
|
||||
BASE_LUT[(dbi*32+hq)*2+c]=(255<<24)|(b<<16)|(g<<8)|r;
|
||||
}}}
|
||||
}
|
||||
buildLUT();
|
||||
function heatQ(pg){if(!hmMode||!hmCounts)return 31;const cnt=hmCounts[pg];return cnt>0?Math.min(31,Math.floor(Math.log2(cnt+1)*31/16)):0}
|
||||
function rollingFaults(dbi){let s=0;for(const w of faultWindow)s+=(w[dbi]||0);return s}
|
||||
function markZbDirty(){zbDirty=true}
|
||||
let blockRecording=null,blockHistory=[],replayMode=false,replayBlockRef=null,replayTimer=null,liveSnapshot=null;
|
||||
const BLOCK_HISTORY_CAP=15;
|
||||
let blockPanelDirty=false;
|
||||
let showFlame=false;const FLAME_SECS=120;let flameData={},flameHead=0;
|
||||
const SPARK_SECS=60;let sparkData={},sparkAccum={};
|
||||
const cv=document.getElementById("map"),cx=cv.getContext("2d"),wr=document.getElementById("canvas-wrap");
|
||||
const ldEl=document.getElementById("loading"),zmEl=document.getElementById("zoom-info");
|
||||
const tt=document.getElementById("tooltip"),ttSw=document.getElementById("tt-swatch");
|
||||
const ttNm=document.getElementById("tt-name"),ttDt=document.getElementById("tt-detail");
|
||||
const tb=document.getElementById("tbl-body");
|
||||
let W=0,H=0,iD=null,oC=null,oCx=null,pX=0,pY=0,sc=1,mS=.01;const xS=128;
|
||||
const sel=new Set();let tEv=0,rEv=0,lRT=performance.now();
|
||||
function cl(v,a,b){return Math.max(a,Math.min(b,v))}
|
||||
function growTo(newNP){
|
||||
if(newNP<=NP)return;
|
||||
const oldNP=NP;
|
||||
const nOm=new Uint8Array(newNP);nOm.set(oMap);nOm.fill(0xFF,oldNP);oMap=nOm;
|
||||
const nOp=new Uint8Array(newNP);nOp.set(lOp);lOp=nOp;
|
||||
const nTs=new Float64Array(newNP);nTs.set(lTs);lTs=nTs;
|
||||
if(hmCounts){const nHm=new Uint32Array(newNP);nHm.set(hmCounts);hmCounts=nHm}
|
||||
if(resMap){const nRm=new Uint8Array(newNP);nRm.set(resMap);resMap=nRm}
|
||||
unrefCount+=newNP-oldNP;
|
||||
NP=newNP;GR=Math.ceil(NP/GC);
|
||||
oMapDirty=true;
|
||||
iD=null;oC=null;markZbDirty();
|
||||
}
|
||||
function oX(){return 0}
|
||||
function oY(){return Math.max(0,(H-GR*sc)/2)}
|
||||
function cP(){const vw=W/sc,vh=H/sc;pX=cl(pX,0,Math.max(0,GC-vw));pY=cl(pY,0,Math.max(0,GR-vh))}
|
||||
function s2w(sx,sy){return{wx:pX+(sx-oX())/sc,wy:pY+(sy-oY())/sc}}
|
||||
function w2p(wx,wy){const px=Math.floor(wx),py=Math.floor(wy);if(px<0||py<0||px>=GC||py>=GR)return-1;const i=py*GC+px;return i<NP?i:-1}
|
||||
|
||||
function trailPush(from,to,op,ts){
|
||||
const idx=(trailStart+trailN)%TRAIL_CAP;
|
||||
if(!trailBuf[idx])trailBuf[idx]={from,to,op,ts};
|
||||
else{trailBuf[idx].from=from;trailBuf[idx].to=to;trailBuf[idx].op=op;trailBuf[idx].ts=ts}
|
||||
if(trailN<TRAIL_CAP)trailN++;
|
||||
else trailStart=(trailStart+1)%TRAIL_CAP;
|
||||
}
|
||||
function trailExpire(now){
|
||||
while(trailN>0){
|
||||
const t=trailBuf[trailStart];
|
||||
if(now-t.ts>=FADE){trailStart=(trailStart+1)%TRAIL_CAP;trailN--}
|
||||
else break;
|
||||
}
|
||||
}
|
||||
function pCBase(pg,flt){
|
||||
if(pg<0||pg>=NP)return false;
|
||||
const ow=oMap[pg];
|
||||
if(flt&&!sel.has(ow)){outR=25;outG=25;outB=28;return true}
|
||||
const hq=heatQ(pg);
|
||||
const cached=(cacheMode&&resMap&&pg<resMap.length)?resMap[pg]?1:0:1;
|
||||
const rgba=BASE_LUT[(ow*32+hq)*2+cached];
|
||||
outR=rgba&0xFF;outG=(rgba>>8)&0xFF;outB=(rgba>>16)&0xFF;
|
||||
return true;
|
||||
}
|
||||
function pC(pg,flt,now){
|
||||
if(!pCBase(pg,flt))return false;
|
||||
const op=lOp[pg];
|
||||
if(op&&((op===1&&sR)||(op===2&&sW)||(op===3&&sF))){
|
||||
const age=now-lTs[pg];
|
||||
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);activeSet.delete(pg)}
|
||||
else{const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
|
||||
outR=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
|
||||
outG=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
|
||||
outB=Math.min(255,Math.round(outB+(fl[2]-outB)*fi))}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
function pg2s(pg){const col=pg%GC,row=Math.floor(pg/GC);return{sx:oX()+(col-pX+.5)*sc,sy:oY()+(row-pY+.5)*sc}}
|
||||
function drawTrails(now){
|
||||
if(!showTrails||trailN===0)return;
|
||||
cx.lineCap="round";cx.lineWidth=trailW;
|
||||
const paths={1:[],2:[],3:[]};
|
||||
for(let j=0;j<trailN;j++){
|
||||
const idx=(trailStart+j)%TRAIL_CAP;
|
||||
const t=trailBuf[idx];
|
||||
const age=now-t.ts;if(age>=FADE)continue;
|
||||
if((t.op===1&&!sR)||(t.op===2&&!sW)||(t.op===3&&!sF))continue;
|
||||
const a=1-Math.pow(age/FADE,2);
|
||||
const f=pg2s(t.from),e=pg2s(t.to);
|
||||
(paths[t.op]||(paths[t.op]=[])).push({fx:f.sx,fy:f.sy,ex:e.sx,ey:e.sy,a});
|
||||
}
|
||||
for(const op of [1,2,3]){
|
||||
const segs=paths[op];if(!segs||!segs.length)continue;
|
||||
const groups={};
|
||||
for(const s of segs){
|
||||
const ak=Math.round(s.a*10);
|
||||
if(!groups[ak])groups[ak]=[];
|
||||
groups[ak].push(s);
|
||||
}
|
||||
const c=TRC[op]||TRC[1];
|
||||
for(const [ak,gs] of Object.entries(groups)){
|
||||
cx.strokeStyle=`rgba(${c},${(ak/10*0.6).toFixed(2)})`;
|
||||
cx.beginPath();
|
||||
for(const s of gs){cx.moveTo(s.fx,s.fy);cx.lineTo(s.ex,s.ey)}
|
||||
cx.stroke();
|
||||
}
|
||||
}
|
||||
}
|
||||
function rebuildZoomBase(flt){
|
||||
if(!zbCv){zbCv=document.createElement("canvas");zbCx=zbCv.getContext("2d")}
|
||||
zbCv.width=W;zbCv.height=H;zbW=W;zbH=H;zbPX=pX;zbPY=pY;zbSc=sc;zbFlt=flt;
|
||||
const zd=zbCx.createImageData(W,H),d=zd.data,ox=oX(),oy=oY(),ips=1/sc;
|
||||
for(let sy=0;sy<H;sy++){
|
||||
const wy0=Math.max(0,Math.floor(pY+(sy-oy)*ips)),wy1=Math.min(GR-1,Math.floor(pY+(sy-oy+1)*ips));
|
||||
for(let sx=0;sx<W;sx++){
|
||||
const wx0=Math.max(0,Math.floor(pX+(sx-ox)*ips)),wx1=Math.min(GC-1,Math.floor(pX+(sx-ox+1)*ips));
|
||||
const o=(sy*W+sx)<<2;
|
||||
if(wx1<0||wy1<0||wx0>=GC||wy0>=GR){d[o]=0;d[o+1]=0;d[o+2]=0;d[o+3]=255;continue}
|
||||
const cpx=Math.floor((wx0+wx1)/2),cpy=Math.floor((wy0+wy1)/2);
|
||||
const ci=cpy*GC+cpx;
|
||||
if(ci>=0&&ci<NP&&pCBase(ci,flt)){d[o]=outR;d[o+1]=outG;d[o+2]=outB}
|
||||
d[o+3]=255;
|
||||
}
|
||||
}
|
||||
zbCx.putImageData(zd,0,0);
|
||||
}
|
||||
function drawActiveOverlay(now,flt){
|
||||
if(activeSet.size===0)return;
|
||||
const ox=oX(),oy=oY(),ips=1/sc;
|
||||
const expired=[];
|
||||
for(const pg of activeSet){
|
||||
const op=lOp[pg];
|
||||
if(!op){expired.push(pg);continue}
|
||||
if((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF))continue;
|
||||
const age=now-lTs[pg];
|
||||
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);expired.push(pg);continue}
|
||||
pCBase(pg,flt);
|
||||
const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
|
||||
const r=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
|
||||
const g=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
|
||||
const b=Math.min(255,Math.round(outB+(fl[2]-outB)*fi));
|
||||
const col=pg%GC,row=Math.floor(pg/GC);
|
||||
const sx=ox+(col-pX)*sc,sy=oy+(row-pY)*sc;
|
||||
if(sx<-sc||sy<-sc||sx>W||sy>H)continue;
|
||||
const psz=Math.max(1,Math.ceil(sc));
|
||||
cx.fillStyle=`rgb(${r},${g},${b})`;
|
||||
cx.fillRect(Math.floor(sx),Math.floor(sy),psz,psz);
|
||||
}
|
||||
for(const pg of expired)activeSet.delete(pg);
|
||||
}
|
||||
function render(){
|
||||
if(!oMap||W<1||H<1)return;
|
||||
const flt=sel.size>0,now=performance.now();
|
||||
if(sc>=1){
|
||||
const vw=W/sc,vh=H/sc,x0=Math.floor(pX),y0=Math.floor(pY);
|
||||
const x1=Math.min(GC,Math.ceil(pX+vw)+1),y1=Math.min(GR,Math.ceil(pY+vh)+1);
|
||||
const vW=Math.max(1,x1-x0),vH=Math.max(1,y1-y0);
|
||||
if(!oC){oC=document.createElement("canvas");oCx=oC.getContext("2d")}
|
||||
if(oC.width!==vW||oC.height!==vH){oC.width=vW;oC.height=vH}
|
||||
const od=oCx.createImageData(vW,vH),d=od.data;
|
||||
for(let y=0;y<vH;y++){const gy=y0+y;for(let x=0;x<vW;x++){
|
||||
const i=gy*GC+(x0+x),o=(y*vW+x)<<2;
|
||||
const ok=(i>=0&&i<NP)?pC(i,flt,now):false;
|
||||
if(ok){d[o]=outR;d[o+1]=outG;d[o+2]=outB}else{d[o]=0;d[o+1]=0;d[o+2]=0}d[o+3]=255;
|
||||
}}
|
||||
oCx.putImageData(od,0,0);cx.imageSmoothingEnabled=false;cx.clearRect(0,0,W,H);
|
||||
cx.drawImage(oC,oX()-(pX-x0)*sc,oY()-(pY-y0)*sc,vW*sc,vH*sc);
|
||||
drawTrails(now);
|
||||
zmEl.textContent=sc.toFixed(sc>=10?0:1)+" px/page";
|
||||
}else{
|
||||
if(zbDirty||!zbCv||zbW!==W||zbH!==H||zbPX!==pX||zbPY!==pY||zbSc!==sc||zbFlt!==flt){
|
||||
rebuildZoomBase(flt);zbDirty=false;
|
||||
}
|
||||
cx.clearRect(0,0,W,H);
|
||||
cx.drawImage(zbCv,0,0);
|
||||
drawActiveOverlay(now,flt);
|
||||
drawTrails(now);
|
||||
const pp=1/sc;zmEl.textContent="1 px = "+pp.toFixed(pp>=10?0:1)+" pages";
|
||||
}
|
||||
}
|
||||
let anim=false;
|
||||
let fpsFrames=0,fpsLast=performance.now();
|
||||
function rLoop(){
|
||||
render();let act=0;const now=performance.now();
|
||||
act+=activePgs;
|
||||
if(showTrails){trailExpire(now);act+=trailN}
|
||||
document.getElementById("active-count").textContent=act.toLocaleString();
|
||||
fpsFrames++;
|
||||
const fpsNow=performance.now();
|
||||
if(fpsNow-fpsLast>=500){
|
||||
document.getElementById("fps-counter").textContent=Math.round(fpsFrames/((fpsNow-fpsLast)/1000));
|
||||
fpsFrames=0;fpsLast=fpsNow;
|
||||
}
|
||||
if(act>0&&!(replayMode&&!replayTimer))requestAnimationFrame(rLoop);else{anim=false;document.getElementById("fps-counter").textContent="-"}
|
||||
}
|
||||
function kick(){if(!anim){anim=true;requestAnimationFrame(rLoop)}}
|
||||
let schedPending=false;
|
||||
function sched(){if(!schedPending){schedPending=true;requestAnimationFrame(()=>{schedPending=false;render()})}}
|
||||
function szCv(){const w=wr.clientWidth,h=wr.clientHeight;if(w<1||h<1)return;W=w;H=h;cv.width=W;cv.height=H;iD=null}
|
||||
function fmtSz(mb){if(mb>=1024)return(mb/1024).toFixed(1)+" GB";if(mb>=1)return mb.toFixed(1)+" MB";return(mb*1024).toFixed(0)+" KB"}
|
||||
function bldTbl(){
|
||||
tb.innerHTML="";const mx=Math.max(...DBIS.map(d=>d.pct),.1);
|
||||
DBIS.forEach(d=>{const tr=document.createElement("tr");tr.className="tbl-row";tr.dataset.dbiIndex=d.dbi_index;
|
||||
const c=PAL[d.dbi_index]||PAL[255];
|
||||
tr.innerHTML=`<td><div class="swatch-cell" style="background:rgb(${c[0]},${c[1]},${c[2]})"></div></td><td>${d.name}</td><td>${d.pages.toLocaleString()}</td><td>${fmtSz(d.size_mb)}</td><td><span class="pct-bar" style="width:${d.pct/mx*60}px;background:rgb(${c[0]},${c[1]},${c[2]})"></span>${d.pct}%</td>`;
|
||||
tr.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
tb.appendChild(tr);
|
||||
});
|
||||
}
|
||||
function uAR(){document.querySelectorAll(".tbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")});document.querySelectorAll(".utbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")})}
|
||||
function reflow(){if(!fW)return;GC=Math.max(1,Math.floor(W/sc));GR=Math.ceil(NP/GC);pX=0;cP()}
|
||||
function boot(){szCv();const aspect=W>0&&H>0?W/H:1;GC=Math.max(1,Math.ceil(Math.sqrt(NP*aspect)));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);sc=mS;pX=0;pY=0;render()}
|
||||
function zA(ns,sx,sy){ns=cl(ns,mS,xS);if(Math.abs(ns-sc)<1e-6)return;const b=s2w(sx,sy);sc=ns;if(fW){reflow();pY=b.wy-(sy-oY())/sc;cP()}else{pX=b.wx-(sx-oX())/sc;pY=b.wy-(sy-oY())/sc;cP()}sched()}
|
||||
document.getElementById("btn-zin").addEventListener("click",()=>zA(sc*1.3,W>>1,H>>1));
|
||||
document.getElementById("btn-zout").addEventListener("click",()=>zA(sc/1.3,W>>1,H>>1));
|
||||
document.getElementById("btn-reset").addEventListener("click",()=>{sc=mS;pX=0;pY=0;sel.clear();uAR();markZbDirty();render()});
|
||||
document.getElementById("btn-clear").addEventListener("click",()=>{if(lOp)lOp.fill(0);if(hmCounts)hmCounts.fill(0);activePgs=0;activeSet.clear();markZbDirty();sched()});
|
||||
wr.addEventListener("wheel",e=>{e.preventDefault();const r=cv.getBoundingClientRect();zA(sc*Math.exp(-(e.deltaMode===1?e.deltaY*16:e.deltaY)*.002),(e.clientX-r.left)/r.width*W,(e.clientY-r.top)/r.height*H)},{passive:false});
|
||||
let drag=false,lCX=0,lCY=0;
|
||||
cv.addEventListener("pointerdown",e=>{drag=true;lCX=e.clientX;lCY=e.clientY;cv.setPointerCapture(e.pointerId);cv.style.cursor="grabbing"});
|
||||
cv.addEventListener("pointermove",e=>{
|
||||
if(drag){if(!fW)pX-=(e.clientX-lCX)/sc;pY-=(e.clientY-lCY)/sc;lCX=e.clientX;lCY=e.clientY;cP();sched();tt.style.display="none";return}
|
||||
const r=cv.getBoundingClientRect(),sx=(e.clientX-r.left)/r.width*W,sy=(e.clientY-r.top)/r.height*H;
|
||||
const w=s2w(sx,sy),pg=w2p(w.wx,w.wy);if(pg<0){tt.style.display="none";return}
|
||||
const ow=oMap[pg],inf=DBI_IDX[ow],c=PAL[ow]||PAL[255];
|
||||
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
let ex="";const op=lOp[pg];if(op===1)ex=" [READ]";else if(op===2)ex=" [WRITE]";else if(op===3)ex=" [FREE]";
|
||||
ttNm.textContent=`Page ${pg.toLocaleString()} \u2014 ${inf?inf.name:"idx "+ow}${ex}`;
|
||||
let dtxt=inf?`${inf.pages.toLocaleString()} pages, ${inf.pct}%`:"";
|
||||
if(hmCounts&&pg<hmCounts.length&&hmCounts[pg]>0)dtxt+=(dtxt?" \u00b7 ":"")+hmCounts[pg].toLocaleString()+" hits";
|
||||
if(resMap&&pg<resMap.length){
|
||||
const cached=resMap[pg];
|
||||
dtxt+=(dtxt?" \u00b7 ":"")+(cached?'<span style="color:#28c840;font-weight:600">CACHED</span>':'<span style="color:#f04040;font-weight:600">NOT CACHED</span>');
|
||||
}
|
||||
ttDt.innerHTML=dtxt;
|
||||
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
|
||||
});
|
||||
cv.addEventListener("pointerup",()=>{drag=false;cv.style.cursor="crosshair"});
|
||||
cv.addEventListener("pointercancel",()=>{drag=false;cv.style.cursor="crosshair"});
|
||||
cv.addEventListener("mouseleave",()=>{tt.style.display="none"});
|
||||
document.getElementById("chk-read").addEventListener("change",e=>{sR=e.target.checked;sched()});
|
||||
document.getElementById("chk-write").addEventListener("change",e=>{sW=e.target.checked;sched()});
|
||||
document.getElementById("chk-free").addEventListener("change",e=>{sF=e.target.checked;sched()});
|
||||
document.getElementById("fade-slider").addEventListener("input",e=>{FADE=parseInt(e.target.value);document.getElementById("fade-val").textContent=(FADE/1000).toFixed(1)+"s"});
|
||||
document.getElementById("chk-trails").addEventListener("change",e=>{showTrails=e.target.checked;if(!showTrails){trailStart=0;trailN=0}sched()});
|
||||
document.getElementById("trail-width").addEventListener("input",e=>{trailW=parseFloat(e.target.value);document.getElementById("trail-w-val").textContent=trailW.toFixed(1);sched()});
|
||||
document.getElementById("chk-heatmap").addEventListener("change",e=>{hmMode=e.target.checked;markZbDirty();sched()});
|
||||
document.getElementById("chk-cache").addEventListener("change",e=>{cacheMode=e.target.checked;markZbDirty();sched()});
|
||||
document.getElementById("chk-flame").addEventListener("change",e=>{
|
||||
showFlame=e.target.checked;const fw=document.getElementById("flame-wrap");
|
||||
const at=DBIS.filter(d=>d.pages>0);fw.style.height=showFlame?Math.min(at.length*8+24,200)+"px":"0";
|
||||
if(showFlame){szFlame();drawFlame()}
|
||||
});
|
||||
document.getElementById("chk-fixw").addEventListener("change",e=>{fW=e.target.checked;if(fW){reflow()}else{GC=Math.ceil(Math.sqrt(NP));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);pX=0;cP()}sched()});
|
||||
window.addEventListener("resize",()=>{if(!oMap)return;szCv();reflow();mS=Math.min(W/GC,H/GR,1);if(sc<mS)sc=mS;cP();render();if(showFlame){szFlame();drawFlame()}});
|
||||
const wD=document.getElementById("ws-dot"),wS=document.getElementById("ws-status");
|
||||
function connectWS(){
|
||||
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws`);
|
||||
ws.binaryType="arraybuffer";
|
||||
ws.onopen=()=>{wD.className="dot connected";wS.textContent="Connected"};
|
||||
ws.onclose=()=>{wD.className="dot disconnected";wS.textContent="Reconnecting...";setTimeout(connectWS,1000)};
|
||||
ws.onerror=()=>ws.close();
|
||||
let lagTimer=0;
|
||||
ws.onmessage=msg=>{
|
||||
if(typeof msg.data==="string"){try{const j=JSON.parse(msg.data);if(j.lagged){const el=document.getElementById("lag-warn"),lc=document.getElementById("lag-count");lc.textContent=j.lagged.toLocaleString();el.style.display="inline";clearTimeout(lagTimer);lagTimer=setTimeout(()=>{el.style.display="none"},5000)}}catch(e){}return}
|
||||
if(!oMap)return;const v=new DataView(msg.data);const now=performance.now();let cnt=0;
|
||||
let lastPg=-1,lastDbi=0,lastOp=0,prevPg=-1;
|
||||
{let scan=0,maxPg=NP;while(scan+8<=msg.data.byteLength){const pg=v.getUint32(scan,true),op=v.getUint8(scan+6);scan+=8;if(op>=4)continue;if(pg>=maxPg)maxPg=pg+1}if(maxPg>NP)growTo(maxPg)}
|
||||
let off=0;
|
||||
while(off+8<=msg.data.byteLength){
|
||||
const pg=v.getUint32(off,true),dbi=v.getUint16(off+4,true),op=v.getUint8(off+6),rsv=v.getUint8(off+7);off+=8;
|
||||
if(op===4){if(!replayMode){blockRecording={number:pg,events:[],before:{},faults:0,startTime:performance.now()};blockPanelDirty=true}continue}
|
||||
if(op===5){
|
||||
if(!replayMode&&blockRecording&&blockRecording.number===pg){
|
||||
const txCount=dbi&0xFF,gasEnc=(dbi>>8)&0xFF;
|
||||
blockRecording.txCount=txCount;
|
||||
blockRecording.duration=rsv>0?Math.pow(2,rsv/20.0):0;
|
||||
blockRecording.gasUsed=gasEnc>0?Math.pow(2,gasEnc/10.0):0;
|
||||
let r=0,w=0,f=0;
|
||||
for(const e of blockRecording.events){if(e.op===1)r++;else if(e.op===2)w++;else if(e.op===3)f++}
|
||||
blockRecording.reads=r;blockRecording.writes=w;blockRecording.frees=f;
|
||||
blockRecording.uniquePages=Object.keys(blockRecording.before).length;
|
||||
blockRecording.panelSnapshot={
|
||||
dbiPages:DBIS.map(d=>d.pages),
|
||||
cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,
|
||||
cachePrevPer:cachePrevPer?cachePrevPer.slice():null,
|
||||
cacheTotalCached,cachePrevTotal,unrefCount,
|
||||
faultWindow:faultWindow.map(w=>({...w})),
|
||||
faultCurrent:{...faultCurrent}
|
||||
};
|
||||
blockHistory.push(blockRecording);
|
||||
if(blockHistory.length>BLOCK_HISTORY_CAP)blockHistory.shift();
|
||||
blockRecording=null;
|
||||
blockPanelDirty=true;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if(blockRecording){
|
||||
if(!(pg in blockRecording.before))blockRecording.before[pg]={om:oMap[pg],rm:resMap?resMap[pg]:0};
|
||||
blockRecording.events.push({pg,dbi,op});
|
||||
}
|
||||
if(pg>=NP)continue;cnt++;
|
||||
if(op===1)cntR++;else if(op===2)cntW++;else if(op===3)cntF++;
|
||||
if(cacheMode&&resMap&&op===1&&pg<resMap.length&&!resMap[pg]){faultCounts[dbi]=(faultCounts[dbi]||0)+1;faultCurrent[dbi]=(faultCurrent[dbi]||0)+1;if(blockRecording){blockRecording.faults=(blockRecording.faults||0)+1}}
|
||||
if(resMap&&pg<resMap.length&&op!==3)resMap[pg]=1;
|
||||
if(hmCounts)hmCounts[pg]++;
|
||||
if(sparkAccum[dbi]!==undefined)sparkAccum[dbi]++;
|
||||
if(op===2&&dbi>0&&dbi<0xFE){const old=oMap[pg];if(old!==dbi){oMap[pg]=dbi;oMapDirty=true;
|
||||
const di=DBI_IDX[dbi];if(di)di.pages++;
|
||||
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
|
||||
if(old===0xFF)unrefCount--;
|
||||
}}
|
||||
if(op===3){const old=oMap[pg];if(old!==1){oMap[pg]=1;oMapDirty=true;
|
||||
const di=DBI_IDX[1];if(di)di.pages++;
|
||||
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
|
||||
if(old===0xFF)unrefCount--;
|
||||
}}
|
||||
if(replayMode)continue;
|
||||
const prev=lOp[pg],prevLive=prev&&(now-lTs[pg])<FADE;
|
||||
if(!prevLive||op>=prev){
|
||||
if(prev===0)activePgs++;
|
||||
lOp[pg]=op;lTs[pg]=now;
|
||||
activeSet.add(pg);
|
||||
}
|
||||
if(showTrails&&prevPg>=0&&prevPg!==pg&&!((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF)))trailPush(prevPg,pg,op,now);
|
||||
prevPg=pg;
|
||||
if(op>=2){lastPg=pg;lastDbi=dbi;lastOp=op}else if(lastPg<0){lastPg=pg;lastDbi=dbi;lastOp=op}
|
||||
}
|
||||
tEv+=cnt;rEv+=cnt;document.getElementById("evt-total").textContent=tEv.toLocaleString();
|
||||
document.getElementById("cnt-read").textContent=cntR.toLocaleString();
|
||||
document.getElementById("cnt-write").textContent=cntW.toLocaleString();
|
||||
document.getElementById("cnt-free").textContent=cntF.toLocaleString();
|
||||
if(lastPg>=0){const inf=DBI_IDX[lastDbi];const nm=inf?inf.name:"dbi"+lastDbi;const opN=lastOp===1?"R":lastOp===2?"W":"F";document.getElementById("evt-latest").textContent=opN+" pg"+lastPg.toLocaleString()+" "+nm}
|
||||
const el=now-lRT;if(el>=1000){document.getElementById("evt-rate").textContent=Math.round(rEv/(el/1000)).toLocaleString();rEv=0;lRT=now}
|
||||
if(cnt>0&&!replayMode)kick();
|
||||
};
|
||||
}
|
||||
function szFlame(){
|
||||
const fw=document.getElementById("flame-wrap"),fc=document.getElementById("flame");
|
||||
if(!fw||!fc)return;fc.width=fw.clientWidth||300;fc.height=fw.clientHeight||100;
|
||||
}
|
||||
function drawFlame(){
|
||||
const fc=document.getElementById("flame");if(!fc)return;
|
||||
const ctx=fc.getContext("2d"),cw=fc.width,ch=fc.height;ctx.clearRect(0,0,cw,ch);
|
||||
const at=DBIS.filter(d=>d.pages>0);if(!at.length)return;
|
||||
const lw=42,rw=cw-lw,rowH=Math.max(4,Math.floor((ch-14)/Math.max(1,at.length)));
|
||||
ctx.font="7px monospace";
|
||||
for(let r=0;r<at.length;r++){
|
||||
const d=at[r],fd=flameData[d.dbi_index];if(!fd)continue;
|
||||
const c=PAL[d.dbi_index]||PAL[255],y=r*rowH;
|
||||
let mx=1;for(let s=0;s<FLAME_SECS;s++){const v=fd[s];if(v>mx)mx=v}
|
||||
for(let s=0;s<FLAME_SECS;s++){
|
||||
const idx=((flameHead-FLAME_SECS+s)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
|
||||
const ops=fd[idx];if(ops<=0)continue;
|
||||
const x=lw+(s/FLAME_SECS)*rw,w=Math.ceil(rw/FLAME_SECS);
|
||||
const a=Math.min(1,Math.log2(ops+1)/Math.log2(mx+1));
|
||||
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},${a.toFixed(2)})`;
|
||||
ctx.fillRect(x,y,w,rowH-1);
|
||||
}
|
||||
ctx.fillStyle="#8b949e";ctx.fillText(d.name.slice(0,6),2,y+rowH-2);
|
||||
}
|
||||
const ay=at.length*rowH+2;ctx.fillStyle="#30363d";ctx.fillRect(lw,ay,rw,1);
|
||||
ctx.fillStyle="#8b949e";ctx.font="7px monospace";
|
||||
for(let s=30;s<FLAME_SECS;s+=30){
|
||||
const x=lw+((FLAME_SECS-s)/FLAME_SECS)*rw;
|
||||
ctx.fillText("-"+s+"s",x,ay+10);
|
||||
}
|
||||
}
|
||||
function buildUnifiedPanel(){
|
||||
const el=document.getElementById("unified-tables");if(!el)return;el.innerHTML="";
|
||||
const hdr=document.createElement("div");hdr.className="utbl-row";hdr.style.cssText="cursor:default;border-bottom:1px solid #30363d;margin-bottom:2px;padding-bottom:4px";
|
||||
hdr.innerHTML='<div class="utbl-sw" style="visibility:hidden"></div><div class="utbl-name" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Table</div><div class="utbl-spark" style="width:80px;text-align:center;color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Activity</div><div class="utbl-cache" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px;text-align:center">Cache</div><div class="utbl-diff" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Diff</div><div class="utbl-faults" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px" title="Page faults over last 3 residency updates">Fault<span style="font-size:7px;text-transform:none"> (3)</span></div>';
|
||||
el.appendChild(hdr);
|
||||
DBIS.forEach(d=>{
|
||||
if(d.pages<=0&&d.dbi_index>2)return;
|
||||
const c=PAL[d.dbi_index]||PAL[255];
|
||||
const row=document.createElement("div");row.className="utbl-row";
|
||||
row.dataset.dbiIndex=d.dbi_index;
|
||||
const sw=document.createElement("div");sw.className="utbl-sw";
|
||||
sw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
const nm=document.createElement("div");nm.className="utbl-name";nm.textContent=d.name;
|
||||
const cv=document.createElement("canvas");cv.className="utbl-spark";cv.width=80;cv.height=16;
|
||||
cv.style.cssText="width:80px;height:16px;border:1px solid #21262d;border-radius:2px;background:#0d1117";
|
||||
const cp=document.createElement("div");cp.className="utbl-cache";
|
||||
const cpBar=document.createElement("div");cpBar.className="utbl-cache-bar-wrap";
|
||||
const cpFill=document.createElement("div");cpFill.className="utbl-cache-fill";cpFill.style.width="0%";cpFill.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
cpBar.appendChild(cpFill);
|
||||
const cpPct=document.createElement("div");cpPct.className="utbl-cache-pct";
|
||||
cp.append(cpBar,cpPct);
|
||||
const cd=document.createElement("div");cd.className="utbl-diff";
|
||||
const ft=document.createElement("div");ft.className="utbl-faults";
|
||||
row.append(sw,nm,cv,cp,cd,ft);
|
||||
row.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
el.appendChild(row);
|
||||
d._sparkCv=cv;d._sparkCtx=cv.getContext("2d");
|
||||
d._cacheEl=cp;d._cacheFill=cpFill;d._cachePct=cpPct;d._diffEl=cd;d._faultEl=ft;
|
||||
});
|
||||
}
|
||||
function drawSparkline(d){
|
||||
const cv=d._sparkCv,ctx=d._sparkCtx;if(!cv||!ctx)return;
|
||||
const sd=sparkData[d.dbi_index];if(!sd)return;
|
||||
const w=cv.width,h=cv.height,c=PAL[d.dbi_index]||PAL[255];
|
||||
ctx.clearRect(0,0,w,h);
|
||||
let mx=1;for(let i=0;i<SPARK_SECS;i++){const v=sd.ring[i];if(v>mx)mx=v}
|
||||
const latest=sd.ring[((sd.head-1)%SPARK_SECS+SPARK_SECS)%SPARK_SECS];
|
||||
ctx.beginPath();ctx.moveTo(0,h);
|
||||
for(let i=0;i<SPARK_SECS;i++){
|
||||
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
|
||||
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
|
||||
if(i===0)ctx.lineTo(x,y);else ctx.lineTo(x,y);
|
||||
}
|
||||
ctx.lineTo(w,h);ctx.closePath();
|
||||
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},0.3)`;ctx.fill();
|
||||
ctx.beginPath();
|
||||
for(let i=0;i<SPARK_SECS;i++){
|
||||
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
|
||||
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
|
||||
if(i===0)ctx.moveTo(x,y);else ctx.lineTo(x,y);
|
||||
}
|
||||
ctx.strokeStyle=`rgba(${c[0]},${c[1]},${c[2]},0.8)`;ctx.lineWidth=1;ctx.stroke();
|
||||
if(d._sparkVal)d._sparkVal.textContent=latest.toLocaleString();
|
||||
}
|
||||
function buildTreeInfo(info){
|
||||
const el=document.getElementById("tree-info");if(!el)return;el.innerHTML="";
|
||||
info.forEach(ti=>{
|
||||
if(ti.height===0&&ti.branch_pages===0&&ti.leaf_pages===0&&ti.large_pages===0)return;
|
||||
const card=document.createElement("div");card.className="tree-card";
|
||||
const total=ti.branch_pages+ti.leaf_pages+ti.large_pages;
|
||||
const ipl=ti.leaf_pages>0?Math.round(ti.items/ti.leaf_pages):0;
|
||||
let html=`<div class="tree-name">${ti.name} <span style="color:#8b949e;font-weight:400">depth ${ti.height}</span></div>`;
|
||||
html+=`<div class="tree-stat">B:${ti.branch_pages.toLocaleString()} L:${ti.leaf_pages.toLocaleString()} O:${ti.large_pages.toLocaleString()} items:${ti.items.toLocaleString()}</div>`;
|
||||
if(total>0){
|
||||
const bp=Math.max(1,Math.round(ti.branch_pages/total*100)),lp=Math.max(1,Math.round(ti.leaf_pages/total*100)),op=Math.max(1,100-bp-lp);
|
||||
html+=`<div class="tree-bar"><div class="tree-seg" style="width:${bp}%;background:#3c82f6"></div><div class="tree-seg" style="width:${lp}%;background:#4ade80"></div><div class="tree-seg" style="width:${op}%;background:#f58231"></div></div>`;
|
||||
}
|
||||
if(ipl>0)html+=`<div class="tree-stat">~${ipl.toLocaleString()} items/leaf</div>`;
|
||||
card.innerHTML=html;
|
||||
card.addEventListener("click",()=>{const i=ti.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
|
||||
el.appendChild(card);
|
||||
});
|
||||
}
|
||||
function fmtGas(g){if(g>=1e6)return(g/1e6).toFixed(1)+"M";if(g>=1e3)return(g/1e3).toFixed(0)+"K";return g.toFixed(0)}
|
||||
function updateBlockPanel(){
|
||||
blockPanelDirty=false;
|
||||
const el=document.getElementById("block-panel");if(!el)return;
|
||||
el.innerHTML="";
|
||||
if(blockRecording){
|
||||
const c=document.createElement("div");c.className="block-card recording";
|
||||
const elapsed=performance.now()-blockRecording.startTime;
|
||||
const elStr=elapsed<1000?elapsed.toFixed(0)+"ms":(elapsed/1000).toFixed(1)+"s";
|
||||
c.innerHTML=`<div class="block-num">⏱ #${blockRecording.number.toLocaleString()} recording\u2026</div><div class="block-stat">${blockRecording.events.length.toLocaleString()} events ${elStr}</div>`;
|
||||
el.appendChild(c);
|
||||
}
|
||||
for(let i=blockHistory.length-1;i>=0;i--){
|
||||
const b=blockHistory[i],isReplaying=replayMode&&replayBlockRef===b;
|
||||
const c=document.createElement("div");c.className="block-card"+(isReplaying?" replaying":"");
|
||||
const total=b.reads+b.writes+b.frees;
|
||||
const dur=b.duration>0?(b.duration<1000?b.duration.toFixed(0)+"ms":(b.duration/1000).toFixed(1)+"s"):"";
|
||||
const gas=b.gasUsed>0?fmtGas(b.gasUsed)+"gas":"";
|
||||
c.innerHTML=`<div class="block-num">▶ #${b.number.toLocaleString()} ${b.txCount}tx${dur?" "+dur:""}${gas?" "+gas:""} ${total.toLocaleString()} ops</div><div class="block-stat"><span style="color:#3c82f6">R:${b.reads.toLocaleString()}</span> <span style="color:#ef4444">W:${b.writes.toLocaleString()}</span> <span style="color:#eab308">F:${b.frees.toLocaleString()}</span>${b.faults?` <span style="color:#d946ef">${b.faults}f</span>`:""} ${(b.uniquePages||0).toLocaleString()} pages</div>`;
|
||||
c.addEventListener("click",()=>startReplay(b));
|
||||
el.appendChild(c);
|
||||
}
|
||||
}
|
||||
function snapPanelState(){
|
||||
return{dbiPages:DBIS.map(d=>d.pages),cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,cachePrevPer:cachePrevPer?cachePrevPer.slice():null,cacheTotalCached,cachePrevTotal,unrefCount,faultWindow:faultWindow.map(w=>({...w})),faultCurrent:{...faultCurrent}};
|
||||
}
|
||||
function applyPanelSnap(snap){
|
||||
DBIS.forEach((d,i)=>{d.pages=snap.dbiPages[i]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
if(snap.cacheCachedPer&&cacheCachedPer){for(let i=0;i<cacheCachedPer.length;i++)cacheCachedPer[i]=snap.cacheCachedPer[i]||0}
|
||||
cachePrevPer=snap.cachePrevPer?snap.cachePrevPer.slice():null;
|
||||
cacheTotalCached=snap.cacheTotalCached;cachePrevTotal=snap.cachePrevTotal;
|
||||
unrefCount=snap.unrefCount;
|
||||
faultWindow=snap.faultWindow.map(w=>({...w}));faultCurrent={...snap.faultCurrent};
|
||||
cacheStale=true;bldTbl();updateCacheStats();
|
||||
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
}
|
||||
function startReplay(block){
|
||||
if(replayMode&&replayBlockRef===block&&replayTimer)return;
|
||||
if(!liveSnapshot)liveSnapshot=snapPanelState();
|
||||
replayMode=true;replayBlockRef=block;
|
||||
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
|
||||
lOp.fill(0);activeSet.clear();activePgs=0;
|
||||
if(hmCounts)hmCounts.fill(0);
|
||||
trailStart=0;trailN=0;
|
||||
for(const pgStr in block.before){
|
||||
const pg=parseInt(pgStr,10);if(pg>=NP)continue;
|
||||
const snap=block.before[pg];
|
||||
oMap[pg]=snap.om;
|
||||
if(resMap&&pg<resMap.length)resMap[pg]=snap.rm;
|
||||
}
|
||||
if(block.panelSnapshot)applyPanelSnap(block.panelSnapshot);
|
||||
markZbDirty();sched();
|
||||
document.getElementById("block-live").style.color="#8b949e";
|
||||
updateBlockPanel();
|
||||
const events=block.events;
|
||||
const batchSize=Math.max(1,Math.ceil(events.length/60));
|
||||
let idx=0;
|
||||
function step(){
|
||||
if(!replayMode){replayTimer=null;return}
|
||||
const end=Math.min(idx+batchSize,events.length);
|
||||
const now=performance.now();
|
||||
for(let i=idx;i<end;i++){
|
||||
const e=events[i];
|
||||
if(e.pg>=NP)continue;
|
||||
if(hmCounts)hmCounts[e.pg]++;
|
||||
if(cacheMode&&resMap&&e.op===1&&e.pg<resMap.length&&!resMap[e.pg]){faultCounts[e.dbi]=(faultCounts[e.dbi]||0)+1;faultCurrent[e.dbi]=(faultCurrent[e.dbi]||0)+1}
|
||||
if(resMap&&e.pg<resMap.length&&e.op!==3)resMap[e.pg]=1;
|
||||
lOp[e.pg]=e.op;lTs[e.pg]=now;
|
||||
if(!activeSet.has(e.pg)){activeSet.add(e.pg);activePgs++}
|
||||
if(e.op===2&&e.dbi>0&&e.dbi<0xFE)oMap[e.pg]=e.dbi;
|
||||
if(showTrails&&i>0&&!((e.op===1&&!sR)||(e.op===2&&!sW)||(e.op===3&&!sF))){const p=events[i-1];if(p.pg!==e.pg)trailPush(p.pg,e.pg,e.op,now)}
|
||||
}
|
||||
idx=end;
|
||||
markZbDirty();kick();
|
||||
if(idx<events.length){replayTimer=requestAnimationFrame(step)}
|
||||
else{
|
||||
replayTimer=null;
|
||||
const pin=performance.now()+FADE*10;
|
||||
for(const pg of activeSet)lTs[pg]=pin;
|
||||
markZbDirty();sched();
|
||||
updateBlockPanel();
|
||||
}
|
||||
}
|
||||
replayTimer=requestAnimationFrame(step);
|
||||
}
|
||||
function goLive(){
|
||||
replayMode=false;replayBlockRef=null;
|
||||
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
|
||||
lOp.fill(0);activeSet.clear();activePgs=0;
|
||||
trailStart=0;trailN=0;
|
||||
if(liveSnapshot){applyPanelSnap(liveSnapshot);liveSnapshot=null}
|
||||
oMapDirty=true;cacheStale=true;
|
||||
markZbDirty();sched();
|
||||
document.getElementById("block-live").style.color="#4ade80";
|
||||
updateBlockPanel();updateCacheStats();
|
||||
}
|
||||
window._goLive=goLive;
|
||||
function connectResidencyWS(){
|
||||
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws_residency`);
|
||||
ws.binaryType="arraybuffer";
|
||||
ws.onclose=()=>setTimeout(connectResidencyWS,3000);
|
||||
ws.onerror=()=>ws.close();
|
||||
ws.onmessage=msg=>{
|
||||
if(!msg.data||msg.data.byteLength<1)return;
|
||||
if(replayMode)return;
|
||||
const v=new DataView(msg.data);
|
||||
const msgType=v.getUint8(0);
|
||||
if(msgType===0){
|
||||
const src=new Uint8Array(msg.data,1);
|
||||
const newLen=src.length;
|
||||
if(!resMap||resMap.length!==newLen){
|
||||
const oldMap=resMap;const oldLen=oldMap?oldMap.length:0;
|
||||
resMap=new Uint8Array(newLen);
|
||||
if(oldMap){
|
||||
const shared=Math.min(oldLen,newLen,NP);
|
||||
for(let i=0;i<shared;i++){
|
||||
const ns=src[i],os=oldMap[i];
|
||||
if(ns!==os){
|
||||
const dbi=oMap?oMap[i]:0;
|
||||
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[i]=ns;
|
||||
}
|
||||
for(let i=shared;i<newLen;i++){
|
||||
resMap[i]=src[i];
|
||||
if(src[i]){cacheTotalCached++;if(cacheCachedPer&&oMap){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
|
||||
}
|
||||
}else{
|
||||
resMap.set(src);
|
||||
cacheTotalCached=0;if(cacheCachedPer)cacheCachedPer.fill(0);
|
||||
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
|
||||
for(let i=0;i<lim;i++){
|
||||
if(src[i]){cacheTotalCached++;if(cacheCachedPer){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
|
||||
}
|
||||
}
|
||||
}else{
|
||||
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
|
||||
for(let i=0;i<lim;i++){
|
||||
const ns=src[i],os=resMap[i];
|
||||
if(ns!==os){
|
||||
const dbi=oMap?oMap[i]:0;
|
||||
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[i]=ns;
|
||||
}
|
||||
}
|
||||
cacheStale=true;
|
||||
cacheLastUpdate=performance.now();
|
||||
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
|
||||
if(cacheMode)markZbDirty();
|
||||
updateCacheStats();
|
||||
if(cacheMode)sched();
|
||||
}else if(msgType===1){
|
||||
if(!resMap)return;
|
||||
const spanCount=v.getUint32(1,true);
|
||||
let off=5;
|
||||
for(let s=0;s<spanCount;s++){
|
||||
const start=v.getUint32(off,true);off+=4;
|
||||
const len=v.getUint32(off,true);off+=4;
|
||||
const state=v.getUint8(off);off+=1;
|
||||
for(let p=start;p<start+len&&p<resMap.length;p++){
|
||||
const oldState=resMap[p];
|
||||
if(oldState!==state){
|
||||
const dbi=oMap?oMap[p]:0;
|
||||
if(state&&!oldState){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
|
||||
else if(!state&&oldState){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
|
||||
}
|
||||
resMap[p]=state;
|
||||
}
|
||||
}
|
||||
cacheStale=true;
|
||||
cacheLastUpdate=performance.now();
|
||||
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
|
||||
if(cacheMode)markZbDirty();
|
||||
updateCacheStats();
|
||||
if(cacheMode)sched();
|
||||
}
|
||||
};
|
||||
}
|
||||
let cacheStatsPending=false;
|
||||
function updateCacheStats(){
|
||||
if(!cacheStale||!cacheCachedPer)return;
|
||||
if(cacheStatsPending)return;
|
||||
cacheStatsPending=true;
|
||||
requestAnimationFrame(()=>{
|
||||
cacheStatsPending=false;
|
||||
if(!cacheStale||!cacheCachedPer)return;
|
||||
cacheStale=false;
|
||||
const limit=Math.min(resMap?resMap.length:0,NP);
|
||||
const totalPct=limit>0?(cacheTotalCached/limit*100).toFixed(1):"0";
|
||||
const prevTotalPct=limit>0?(cachePrevTotal/limit*100):0;
|
||||
const totalDiff=limit>0?(cacheTotalCached/limit*100-prevTotalPct):0;
|
||||
const tdS=cachePrevPer?(totalDiff>=0?"+":"")+totalDiff.toFixed(2)+"%":"";
|
||||
const tdC=totalDiff>0?"#4ade80":totalDiff<0?"#f87171":"#8b949e";
|
||||
document.getElementById("cache-total").innerHTML=`${cacheTotalCached.toLocaleString()}/${limit.toLocaleString()} (${totalPct}%) ${cachePrevPer?`<span style="color:${tdC}">${tdS}</span>`:""}`;
|
||||
DBIS.forEach(d=>{
|
||||
if(!d._cacheEl)return;
|
||||
const total=d.pages;if(total<=0){if(d._cachePct)d._cachePct.textContent="-";if(d._cacheFill)d._cacheFill.style.width="0%";d._diffEl.textContent="";d._faultEl.textContent="";return}
|
||||
const cached=cacheCachedPer[d.dbi_index]||0;
|
||||
const pct=total>0?(cached/total*100):0;
|
||||
if(d._cacheFill)d._cacheFill.style.width=pct.toFixed(1)+"%";
|
||||
if(d._cachePct){d._cachePct.textContent=pct.toFixed(1)+"%";d._cachePct.style.color=pct>50?"#4ade80":"#f87171"}
|
||||
const prevCached=cachePrevPer?cachePrevPer[d.dbi_index]||0:0;
|
||||
const prevPct=total>0?(prevCached/total*100):0;
|
||||
const diff=cachePrevPer?(pct-prevPct):0;
|
||||
if(cachePrevPer){
|
||||
d._diffEl.textContent=(diff>=0?"+":"")+diff.toFixed(2)+"%";
|
||||
d._diffEl.style.color=diff>0?"#4ade80":diff<0?"#f87171":"#8b949e";
|
||||
}
|
||||
const fc=rollingFaults(d.dbi_index);
|
||||
d._faultEl.textContent=fc>0?fc+"f":"";
|
||||
});
|
||||
cachePrevPer=cacheCachedPer.slice();
|
||||
cachePrevTotal=cacheTotalCached;
|
||||
});
|
||||
}
|
||||
async function init(){
|
||||
try{
|
||||
const r=await fetch("/api/info"),info=await r.json();NP=info.page_count;PS=info.page_size;
|
||||
DBIS=info.dbi_names.map((n,i)=>({name:n,dbi_index:i,pages:0,pct:0,size_mb:0}));
|
||||
DBI_IDX={};DBIS.forEach(d=>{DBI_IDX[d.dbi_index]=d});
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 loading owner map\u2026`;
|
||||
}catch(e){document.getElementById("subtitle").textContent="Failed: "+e.message}
|
||||
try{
|
||||
const lt=document.getElementById("load-text"),lbw=document.getElementById("load-bar-wrap"),lb=document.getElementById("load-bar");
|
||||
lt.textContent="Downloading owner map\u2026";lbw.style.display="block";
|
||||
const r=await fetch("/api/owner_map");
|
||||
if(r.ok){
|
||||
const total=parseInt(r.headers.get("content-length")||"0",10)||NP;
|
||||
const reader=r.body.getReader();
|
||||
const buf=new Uint8Array(total||NP);let pos=0;
|
||||
for(;;){const{done,value}=await reader.read();if(done)break;buf.set(value,pos);pos+=value.length;
|
||||
const pct=total>0?Math.min(100,pos/total*100):0;
|
||||
lb.style.width=pct.toFixed(1)+"%";
|
||||
lt.textContent=`Downloading owner map\u2026 ${(pos/(1024*1024)).toFixed(1)}/${(total/(1024*1024)).toFixed(1)} MB`;
|
||||
}
|
||||
oMap=buf.length===pos?buf:buf.slice(0,pos);
|
||||
lt.textContent="Processing owner map\u2026";lb.style.width="100%";
|
||||
await new Promise(r=>setTimeout(r,0));
|
||||
const ct={};for(let i=0;i<NP;i++){const o=oMap[i];ct[o]=(ct[o]||0)+1}
|
||||
DBIS.forEach(d=>{d.pages=ct[d.dbi_index]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
}else{oMap=new Uint8Array(NP);oMap.fill(0xFF)}
|
||||
}catch(e){oMap=new Uint8Array(NP);oMap.fill(0xFF)}
|
||||
unrefCount=oMap.reduce((n,b)=>n+(b===0xFF?1:0),0);const ref=NP-unrefCount;
|
||||
const gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${ref.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
lOp=new Uint8Array(NP);lTs=new Float64Array(NP);hmCounts=new Uint32Array(NP);
|
||||
cacheCachedPer=new Array(DBIS.length).fill(0);
|
||||
DBIS.forEach(d=>{sparkData[d.dbi_index]={ring:new Array(SPARK_SECS).fill(0),head:0};sparkAccum[d.dbi_index]=0;flameData[d.dbi_index]=new Array(FLAME_SECS).fill(0)});
|
||||
bldTbl();buildUnifiedPanel();
|
||||
let treeInfo=[];
|
||||
try{const r=await fetch("/api/tree_info");if(r.ok)treeInfo=await r.json()}catch(e){}
|
||||
if(treeInfo.length>0)buildTreeInfo(treeInfo);
|
||||
ldEl.style.display="none";cv.style.display="block";boot();connectWS();connectResidencyWS();
|
||||
const flCv=document.getElementById("flame");
|
||||
flCv.addEventListener("mousemove",e=>{
|
||||
if(!showFlame)return;
|
||||
const at=DBIS.filter(d=>d.pages>0);if(!at.length){tt.style.display="none";return}
|
||||
const rect=flCv.getBoundingClientRect();
|
||||
const mx=e.clientX-rect.left,my=e.clientY-rect.top;
|
||||
const lw=42,rw=flCv.width-lw;
|
||||
const rowH=Math.max(4,Math.floor((flCv.height-14)/Math.max(1,at.length)));
|
||||
const ri=Math.floor(my/rowH);
|
||||
if(ri<0||ri>=at.length||mx<lw){tt.style.display="none";return}
|
||||
const d=at[ri],c=PAL[d.dbi_index]||PAL[255];
|
||||
const secIdx=Math.floor((mx-lw)/rw*FLAME_SECS);
|
||||
const idx=((flameHead-FLAME_SECS+secIdx)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
|
||||
const fd=flameData[d.dbi_index];
|
||||
const ops=fd?fd[idx]:0;
|
||||
const ago=FLAME_SECS-secIdx;
|
||||
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
|
||||
ttNm.textContent=d.name;
|
||||
ttDt.innerHTML=`${ops.toLocaleString()} ops/s \u00b7 ${ago}s ago`;
|
||||
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
|
||||
});
|
||||
flCv.addEventListener("mouseleave",()=>{tt.style.display="none"});
|
||||
setInterval(()=>{
|
||||
DBIS.forEach(d=>{
|
||||
const v=sparkAccum[d.dbi_index]||0;
|
||||
const sd=sparkData[d.dbi_index];if(sd){sd.ring[sd.head%SPARK_SECS]=v;sd.head++}
|
||||
const fd=flameData[d.dbi_index];if(fd)fd[flameHead%FLAME_SECS]=v;
|
||||
sparkAccum[d.dbi_index]=0;
|
||||
});
|
||||
flameHead++;
|
||||
DBIS.forEach(d=>drawSparkline(d));
|
||||
if(showFlame)drawFlame();
|
||||
},1000);
|
||||
setInterval(()=>{if(!oMap||!oMapDirty||replayMode)return;oMapDirty=false;
|
||||
DBIS.forEach(d=>{d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
|
||||
bldTbl();uAR();markZbDirty();sched();
|
||||
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
|
||||
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
|
||||
},2000);
|
||||
setInterval(()=>{
|
||||
if(!cacheLastUpdate)return;
|
||||
const ago=Math.round((performance.now()-cacheLastUpdate)/1000);
|
||||
let txt;
|
||||
if(ago<60)txt=ago+"s ago";
|
||||
else if(ago<3600)txt=Math.floor(ago/60)+"m "+ago%60+"s ago";
|
||||
else txt=Math.floor(ago/3600)+"h ago";
|
||||
document.getElementById("cache-ago").textContent=txt;
|
||||
},1000);
|
||||
setInterval(()=>{if(blockPanelDirty)updateBlockPanel()},250);
|
||||
setInterval(()=>{
|
||||
if(activeSet.size<5000)return;
|
||||
const now=performance.now();
|
||||
for(const pg of activeSet){
|
||||
if(!lOp[pg]||now-lTs[pg]>=FADE){activeSet.delete(pg);if(lOp[pg]){lOp[pg]=0;activePgs=Math.max(0,activePgs-1)}}
|
||||
}
|
||||
},5000);
|
||||
}
|
||||
init();
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
577
crates/storage/mdbx-viz/src/lib.rs
Normal file
577
crates/storage/mdbx-viz/src/lib.rs
Normal file
@@ -0,0 +1,577 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
mpsc, Arc, RwLock,
|
||||
};
|
||||
|
||||
use axum::{
|
||||
extract::{
|
||||
ws::{Message, WebSocket, WebSocketUpgrade},
|
||||
State,
|
||||
},
|
||||
http::{header, StatusCode},
|
||||
response::{Html, IntoResponse},
|
||||
routing::get,
|
||||
Router,
|
||||
};
|
||||
use reth_libmdbx::pageviz::{PageEvent, PageOp};
|
||||
use serde::Serialize;
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
pub mod walker;
|
||||
|
||||
unsafe extern "C" {
|
||||
fn mdbx_pageviz_emit_block_marker(op: u8, block_number: u32, tx_count: u16, duration_encoded: u8, gas_encoded: u8);
|
||||
}
|
||||
|
||||
pub fn pageviz_emit_block_marker(block_number: u64, start: bool, tx_count: u16, duration_ms: u32, gas_used: u64) {
|
||||
let dur_enc = if duration_ms == 0 { 0u8 } else { ((duration_ms as f64).log2() * 20.0).round().min(255.0) as u8 };
|
||||
let gas_enc = if gas_used == 0 { 0u8 } else { ((gas_used as f64).log2() * 10.0).round().min(255.0) as u8 };
|
||||
unsafe {
|
||||
let op = if start { 4u8 } else { 5u8 };
|
||||
mdbx_pageviz_emit_block_marker(op, block_number as u32, tx_count, dur_enc, gas_enc);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Serialize)]
|
||||
struct ResidencyStats {
|
||||
total_pages: u64,
|
||||
cached_pages: u64,
|
||||
pct: f64,
|
||||
per_table: Vec<TableResidency>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default, Serialize)]
|
||||
struct TableResidency {
|
||||
dbi: usize,
|
||||
name: String,
|
||||
total: u64,
|
||||
cached: u64,
|
||||
pct: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct AppState {
|
||||
info: VizInfo,
|
||||
owner_map: Arc<RwLock<Vec<u8>>>,
|
||||
tree_info: Arc<Vec<walker::TreeInfo>>,
|
||||
subscribers: Arc<AtomicUsize>,
|
||||
event_tx: broadcast::Sender<Vec<u8>>,
|
||||
residency: Arc<RwLock<Vec<u8>>>,
|
||||
residency_stats: Arc<RwLock<ResidencyStats>>,
|
||||
residency_tx: broadcast::Sender<Vec<u8>>,
|
||||
residency_subscribers: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub struct VizInfo {
|
||||
pub page_count: u64,
|
||||
pub page_size: u32,
|
||||
pub dbi_names: Vec<String>,
|
||||
}
|
||||
|
||||
pub struct VizConfig {
|
||||
pub port: u16,
|
||||
pub page_count: u64,
|
||||
pub page_size: u32,
|
||||
pub dbi_names: Vec<String>,
|
||||
pub owner_map: Vec<u8>,
|
||||
pub tree_info: Vec<walker::TreeInfo>,
|
||||
pub db_path: PathBuf,
|
||||
}
|
||||
|
||||
const WIRE_EVENT_SIZE: usize = 8;
|
||||
|
||||
fn encode_events(events: &[PageEvent]) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(events.len() * WIRE_EVENT_SIZE);
|
||||
for ev in events {
|
||||
buf.extend_from_slice(&ev.pgno.to_le_bytes());
|
||||
if ev.op.is_block_marker() {
|
||||
let tx_lo = (ev.dbi & 0xFF) as u8;
|
||||
let gas = ((ev.dbi >> 24) & 0xFF) as u8;
|
||||
buf.extend_from_slice(&u16::to_le_bytes((gas as u16) << 8 | tx_lo as u16));
|
||||
buf.push(ev.op as u8);
|
||||
buf.push(((ev.dbi >> 16) & 0xFF) as u8);
|
||||
} else {
|
||||
buf.extend_from_slice(&(ev.dbi as u16).to_le_bytes());
|
||||
buf.push(ev.op as u8);
|
||||
buf.push(0u8);
|
||||
}
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
struct ReadOnlyMapping {
|
||||
base: *mut libc::c_void,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl ReadOnlyMapping {
|
||||
fn new(path: &std::path::Path) -> Option<Self> {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
let file = std::fs::File::open(path).ok()?;
|
||||
let len = file.metadata().ok()?.len() as usize;
|
||||
if len == 0 {
|
||||
return None;
|
||||
}
|
||||
let base = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if base == libc::MAP_FAILED {
|
||||
return None;
|
||||
}
|
||||
Some(Self { base, len })
|
||||
}
|
||||
|
||||
fn remap(&mut self, path: &std::path::Path) -> bool {
|
||||
use std::os::unix::io::AsRawFd;
|
||||
let file = match std::fs::File::open(path) {
|
||||
Ok(f) => f,
|
||||
Err(_) => return false,
|
||||
};
|
||||
let new_len = match file.metadata() {
|
||||
Ok(m) => m.len() as usize,
|
||||
Err(_) => return false,
|
||||
};
|
||||
if new_len == 0 {
|
||||
return false;
|
||||
}
|
||||
unsafe {
|
||||
libc::munmap(self.base, self.len);
|
||||
}
|
||||
let base = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
new_len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if base == libc::MAP_FAILED {
|
||||
return false;
|
||||
}
|
||||
self.base = base;
|
||||
self.len = new_len;
|
||||
true
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReadOnlyMapping {
|
||||
fn drop(&mut self) {
|
||||
unsafe {
|
||||
libc::munmap(self.base, self.len);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for ReadOnlyMapping {}
|
||||
|
||||
pub async fn start_viz_server(
|
||||
config: VizConfig,
|
||||
rx: mpsc::Receiver<Vec<PageEvent>>,
|
||||
) -> std::io::Result<()> {
|
||||
let (event_tx, _) = broadcast::channel::<Vec<u8>>(256);
|
||||
let (residency_tx, _) = broadcast::channel::<Vec<u8>>(64);
|
||||
|
||||
let owner_map = Arc::new(RwLock::new(config.owner_map));
|
||||
let subscribers = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let state = AppState {
|
||||
info: VizInfo {
|
||||
page_count: config.page_count,
|
||||
page_size: config.page_size,
|
||||
dbi_names: config.dbi_names,
|
||||
},
|
||||
owner_map: owner_map.clone(),
|
||||
tree_info: Arc::new(config.tree_info),
|
||||
subscribers: subscribers.clone(),
|
||||
event_tx: event_tx.clone(),
|
||||
residency: Arc::new(RwLock::new(Vec::new())),
|
||||
residency_stats: Arc::new(RwLock::new(ResidencyStats::default())),
|
||||
residency_tx: residency_tx.clone(),
|
||||
residency_subscribers: Arc::new(AtomicUsize::new(0)),
|
||||
};
|
||||
|
||||
let bridge_tx = event_tx.clone();
|
||||
std::thread::Builder::new()
|
||||
.name("viz-bridge".into())
|
||||
.spawn(move || {
|
||||
while let Ok(events) = rx.recv() {
|
||||
if events.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
{
|
||||
let mut map = owner_map.write().unwrap();
|
||||
for ev in &events {
|
||||
let pg = ev.pgno as usize;
|
||||
if ev.op == PageOp::Write && ev.dbi > 0 && ev.dbi < 0xFE && pg < map.len()
|
||||
{
|
||||
map[pg] = ev.dbi as u8;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if subscribers.load(Ordering::Relaxed) > 0 {
|
||||
let encoded = encode_events(&events);
|
||||
let _ = bridge_tx.send(encoded);
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn viz-bridge thread");
|
||||
|
||||
let res_map = state.residency.clone();
|
||||
let res_stats = state.residency_stats.clone();
|
||||
let res_tx = state.residency_tx.clone();
|
||||
let res_subs = state.residency_subscribers.clone();
|
||||
let own_map = state.owner_map.clone();
|
||||
let info_clone = state.info.clone();
|
||||
let db_path = config.db_path;
|
||||
let mdbx_ps = config.page_size;
|
||||
let sys_ps = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u32;
|
||||
|
||||
std::thread::Builder::new()
|
||||
.name("residency-poller".into())
|
||||
.spawn(move || {
|
||||
let mut mapping: Option<ReadOnlyMapping> = None;
|
||||
let mut prev: Vec<u8> = Vec::new();
|
||||
|
||||
if db_path.as_os_str().is_empty() {
|
||||
tracing::warn!("residency-poller: no db_path configured, residency polling disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
std::thread::sleep(std::time::Duration::from_secs(3));
|
||||
|
||||
if res_subs.load(Ordering::Relaxed) == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if mapping.is_none() {
|
||||
match ReadOnlyMapping::new(&db_path) {
|
||||
Some(m) => {
|
||||
tracing::info!(
|
||||
"residency-poller: mapped {} ({} bytes, mdbx_ps={}, sys_ps={})",
|
||||
db_path.display(),
|
||||
m.len,
|
||||
mdbx_ps,
|
||||
sys_ps,
|
||||
);
|
||||
mapping = Some(m);
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(
|
||||
"residency-poller: failed to mmap {}",
|
||||
db_path.display()
|
||||
);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let file_len = match std::fs::metadata(&db_path) {
|
||||
Ok(m) => m.len() as usize,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let m = mapping.as_mut().unwrap();
|
||||
if file_len > m.len {
|
||||
tracing::info!(
|
||||
"residency-poller: file grew {} -> {}, remapping",
|
||||
m.len,
|
||||
file_len,
|
||||
);
|
||||
if !m.remap(&db_path) {
|
||||
tracing::warn!("residency-poller: remap failed");
|
||||
mapping = None;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let m = mapping.as_ref().unwrap();
|
||||
let base = m.base;
|
||||
let len = m.len;
|
||||
|
||||
if mdbx_ps == 0 || sys_ps == 0 || len == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let sys_pages = (len + sys_ps as usize - 1) / sys_ps as usize;
|
||||
let mdbx_page_count = len / mdbx_ps as usize;
|
||||
|
||||
let mut mincore_vec = vec![0u8; sys_pages];
|
||||
let ret = unsafe { libc::mincore(base, len, mincore_vec.as_mut_ptr()) };
|
||||
if ret != 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut cur = vec![0u8; mdbx_page_count];
|
||||
if mdbx_ps == sys_ps {
|
||||
for i in 0..mdbx_page_count.min(sys_pages) {
|
||||
cur[i] = mincore_vec[i] & 1;
|
||||
}
|
||||
} else if mdbx_ps > sys_ps {
|
||||
let ratio = mdbx_ps as usize / sys_ps as usize;
|
||||
for i in 0..mdbx_page_count {
|
||||
let base_idx = i * ratio;
|
||||
let mut all_resident = true;
|
||||
for j in 0..ratio {
|
||||
if base_idx + j >= sys_pages
|
||||
|| (mincore_vec[base_idx + j] & 1) == 0
|
||||
{
|
||||
all_resident = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
cur[i] = if all_resident { 1 } else { 0 };
|
||||
}
|
||||
} else {
|
||||
let ratio = sys_ps as usize / mdbx_ps as usize;
|
||||
for i in 0..mdbx_page_count {
|
||||
let sys_idx = i / ratio;
|
||||
cur[i] = if sys_idx < sys_pages {
|
||||
mincore_vec[sys_idx] & 1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut rm = res_map.write().unwrap();
|
||||
rm.clear();
|
||||
rm.extend_from_slice(&cur);
|
||||
}
|
||||
|
||||
{
|
||||
let omap = own_map.read().unwrap();
|
||||
let max_dbi = info_clone.dbi_names.len();
|
||||
let mut total_per = vec![0u64; max_dbi];
|
||||
let mut cached_per = vec![0u64; max_dbi];
|
||||
let mut total_cached = 0u64;
|
||||
let limit = mdbx_page_count.min(omap.len());
|
||||
|
||||
for i in 0..limit {
|
||||
let dbi = omap[i] as usize;
|
||||
if dbi < max_dbi {
|
||||
total_per[dbi] += 1;
|
||||
if cur[i] == 1 {
|
||||
cached_per[dbi] += 1;
|
||||
total_cached += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let mut stats = res_stats.write().unwrap();
|
||||
stats.total_pages = mdbx_page_count as u64;
|
||||
stats.cached_pages = total_cached;
|
||||
stats.pct = if mdbx_page_count > 0 {
|
||||
(total_cached as f64 / mdbx_page_count as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
stats.per_table = (0..max_dbi)
|
||||
.filter(|&d| total_per[d] > 0)
|
||||
.map(|d| TableResidency {
|
||||
dbi: d,
|
||||
name: info_clone.dbi_names.get(d).cloned().unwrap_or_default(),
|
||||
total: total_per[d],
|
||||
cached: cached_per[d],
|
||||
pct: if total_per[d] > 0 {
|
||||
(cached_per[d] as f64 / total_per[d] as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
},
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
|
||||
if prev.len() != cur.len() {
|
||||
let mut buf = Vec::with_capacity(1 + cur.len());
|
||||
buf.push(0u8);
|
||||
buf.extend_from_slice(&cur);
|
||||
let _ = res_tx.send(buf);
|
||||
tracing::info!(
|
||||
"residency-poller: broadcast full snapshot ({} pages)",
|
||||
cur.len(),
|
||||
);
|
||||
} else {
|
||||
let mut spans: Vec<u8> = Vec::new();
|
||||
let mut span_count: u32 = 0;
|
||||
spans.extend_from_slice(&[1u8]);
|
||||
spans.extend_from_slice(&[0u8; 4]);
|
||||
|
||||
let mut i = 0;
|
||||
while i < cur.len() {
|
||||
if cur[i] != prev[i] {
|
||||
let state_val = cur[i];
|
||||
let start = i as u32;
|
||||
let mut end = i + 1;
|
||||
while end < cur.len()
|
||||
&& cur[end] != prev[end]
|
||||
&& cur[end] == state_val
|
||||
{
|
||||
end += 1;
|
||||
}
|
||||
let run_len = (end - i) as u32;
|
||||
spans.extend_from_slice(&start.to_le_bytes());
|
||||
spans.extend_from_slice(&run_len.to_le_bytes());
|
||||
spans.push(state_val);
|
||||
span_count += 1;
|
||||
i = end;
|
||||
} else {
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if span_count > 0 {
|
||||
spans[1..5].copy_from_slice(&span_count.to_le_bytes());
|
||||
let _ = res_tx.send(spans);
|
||||
}
|
||||
}
|
||||
|
||||
prev = cur;
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn residency-poller thread");
|
||||
|
||||
let app = Router::new()
|
||||
.route("/", get(index_handler))
|
||||
.route("/ws", get(ws_handler))
|
||||
.route("/api/info", get(info_handler))
|
||||
.route("/api/owner_map", get(owner_map_handler))
|
||||
.route("/api/tree_info", get(tree_info_handler))
|
||||
.route("/api/residency", get(residency_handler))
|
||||
.route("/api/residency_stats", get(residency_stats_handler))
|
||||
.route("/ws_residency", get(ws_residency_handler))
|
||||
.with_state(state);
|
||||
|
||||
let addr = format!("0.0.0.0:{}", config.port);
|
||||
tracing::info!("mdbx-viz server listening on {addr}");
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&addr).await?;
|
||||
axum::serve(listener, app).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn index_handler() -> Html<&'static str> {
|
||||
Html(include_str!("index.html"))
|
||||
}
|
||||
|
||||
async fn info_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
axum::Json(state.info)
|
||||
}
|
||||
|
||||
async fn owner_map_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let map = state.owner_map.read().unwrap();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/octet-stream")],
|
||||
map.clone(),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn tree_info_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
axum::Json(state.tree_info.as_ref().clone())
|
||||
}
|
||||
|
||||
async fn ws_handler(
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<AppState>,
|
||||
) -> impl IntoResponse {
|
||||
ws.on_upgrade(move |socket| handle_ws(socket, state))
|
||||
}
|
||||
|
||||
async fn handle_ws(mut socket: WebSocket, state: AppState) {
|
||||
state.subscribers.fetch_add(1, Ordering::Relaxed);
|
||||
let mut rx = state.event_tx.subscribe();
|
||||
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(data) => {
|
||||
if socket.send(Message::Binary(data.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(n)) => {
|
||||
tracing::warn!("ws client lagged by {n} messages");
|
||||
let msg = format!("{{\"lagged\":{n}}}");
|
||||
if socket.send(Message::Text(msg)).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
|
||||
state.subscribers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
async fn residency_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let map = state.residency.read().unwrap();
|
||||
(
|
||||
StatusCode::OK,
|
||||
[(header::CONTENT_TYPE, "application/octet-stream")],
|
||||
map.clone(),
|
||||
)
|
||||
.into_response()
|
||||
}
|
||||
|
||||
async fn residency_stats_handler(State(state): State<AppState>) -> impl IntoResponse {
|
||||
let stats = state.residency_stats.read().unwrap();
|
||||
axum::Json(stats.clone())
|
||||
}
|
||||
|
||||
async fn ws_residency_handler(
|
||||
ws: WebSocketUpgrade,
|
||||
State(state): State<AppState>,
|
||||
) -> impl IntoResponse {
|
||||
ws.on_upgrade(move |socket| handle_ws_residency(socket, state))
|
||||
}
|
||||
|
||||
async fn handle_ws_residency(mut socket: WebSocket, state: AppState) {
|
||||
state.residency_subscribers.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
{
|
||||
let msg = {
|
||||
let map = state.residency.read().unwrap();
|
||||
if !map.is_empty() {
|
||||
let mut buf = Vec::with_capacity(1 + map.len());
|
||||
buf.push(0u8);
|
||||
buf.extend_from_slice(&map);
|
||||
Some(buf)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
};
|
||||
if let Some(msg) = msg {
|
||||
let _ = socket.send(Message::Binary(msg.into())).await;
|
||||
}
|
||||
}
|
||||
|
||||
let mut rx = state.residency_tx.subscribe();
|
||||
loop {
|
||||
match rx.recv().await {
|
||||
Ok(data) => {
|
||||
if socket.send(Message::Binary(data.into())).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(broadcast::error::RecvError::Lagged(_)) => continue,
|
||||
Err(broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
|
||||
state.residency_subscribers.fetch_sub(1, Ordering::Relaxed);
|
||||
}
|
||||
93
crates/storage/mdbx-viz/src/main.rs
Normal file
93
crates/storage/mdbx-viz/src/main.rs
Normal file
@@ -0,0 +1,93 @@
|
||||
use std::sync::mpsc;
|
||||
use std::time::Duration;
|
||||
|
||||
use clap::Parser;
|
||||
use reth_libmdbx::pageviz::{PageEvent, PageOp};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "reth-mdbx-viz", about = "Real-time MDBX page visualization")]
|
||||
struct Args {
|
||||
/// Path to the .bin owner map file (generated by mdbx-viz Python tool)
|
||||
#[arg(short, long)]
|
||||
bin: Option<String>,
|
||||
|
||||
/// Page size in bytes
|
||||
#[arg(long, default_value = "4096")]
|
||||
page_size: u32,
|
||||
|
||||
/// Port to serve on
|
||||
#[arg(short, long, default_value = "3141")]
|
||||
port: u16,
|
||||
|
||||
/// Generate fake events for testing
|
||||
#[arg(long)]
|
||||
demo: bool,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "info".into()),
|
||||
)
|
||||
.init();
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
let (owner_map, page_count) = if let Some(ref bin_path) = args.bin {
|
||||
let data = std::fs::read(bin_path).expect("failed to read .bin file");
|
||||
let pc = data.len() as u64;
|
||||
tracing::info!("loaded owner map: {pc} pages from {bin_path}");
|
||||
(data, pc)
|
||||
} else {
|
||||
tracing::info!("no --bin provided, starting with empty owner map");
|
||||
(vec![0xFFu8; 1_000_000], 1_000_000)
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
|
||||
if args.demo {
|
||||
let np = page_count as u32;
|
||||
std::thread::Builder::new()
|
||||
.name("demo-events".into())
|
||||
.spawn(move || {
|
||||
let mut pgno: u32 = 0;
|
||||
loop {
|
||||
let mut batch = Vec::with_capacity(256);
|
||||
for _ in 0..256 {
|
||||
let p = pgno % np;
|
||||
batch.push(PageEvent {
|
||||
pgno: p,
|
||||
dbi: (p % 8) + 2,
|
||||
op: match p % 10 {
|
||||
0..=6 => PageOp::Read,
|
||||
7..=8 => PageOp::Write,
|
||||
_ => PageOp::Free,
|
||||
},
|
||||
});
|
||||
pgno = pgno.wrapping_add(37);
|
||||
}
|
||||
if tx.send(batch).is_err() {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(33));
|
||||
}
|
||||
})
|
||||
.expect("failed to spawn demo thread");
|
||||
}
|
||||
|
||||
let config = reth_mdbx_viz::VizConfig {
|
||||
port: args.port,
|
||||
page_count,
|
||||
page_size: args.page_size,
|
||||
dbi_names: vec![],
|
||||
tree_info: vec![],
|
||||
owner_map,
|
||||
db_path: std::path::PathBuf::new(),
|
||||
};
|
||||
|
||||
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
|
||||
tracing::error!("server error: {e}");
|
||||
}
|
||||
}
|
||||
757
crates/storage/mdbx-viz/src/walker.rs
Normal file
757
crates/storage/mdbx-viz/src/walker.rs
Normal file
@@ -0,0 +1,757 @@
|
||||
use rayon::prelude::*;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
io::{self, Read, Write},
|
||||
os::unix::io::AsRawFd,
|
||||
path::Path,
|
||||
sync::atomic::{AtomicU8, AtomicU64, Ordering},
|
||||
time::Instant,
|
||||
};
|
||||
|
||||
const PAGEHDRSZ: usize = 20;
|
||||
const NODESIZE: usize = 8;
|
||||
|
||||
const P_BRANCH: u16 = 0x01;
|
||||
const P_LEAF: u16 = 0x02;
|
||||
const P_LARGE: u16 = 0x04;
|
||||
const P_META: u16 = 0x08;
|
||||
const P_DUPFIX: u16 = 0x20;
|
||||
|
||||
const N_BIG: u8 = 0x01;
|
||||
const N_TREE: u8 = 0x02;
|
||||
const N_DUP: u8 = 0x04;
|
||||
|
||||
const P_INVALID: u32 = 0xFFFFFFFF;
|
||||
const UNREFERENCED: u8 = 0xFF;
|
||||
const DBI_META: u8 = 0xFE;
|
||||
|
||||
const META_GEO: usize = 0x14;
|
||||
const META_FREE_DB: usize = 0x28;
|
||||
const META_MAIN_DB: usize = 0x58;
|
||||
const META_TXNID_A: usize = 0x08;
|
||||
const META_TXNID_B: usize = 0xB0;
|
||||
const GEO_NOW: usize = 0x0C;
|
||||
|
||||
const TREE_ROOT: usize = 0x08;
|
||||
const TREE_BRANCH_PAGES: usize = 0x0C;
|
||||
const TREE_LEAF_PAGES: usize = 0x10;
|
||||
const TREE_LARGE_PAGES: usize = 0x14;
|
||||
const TREE_ITEMS: usize = 0x20;
|
||||
|
||||
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
|
||||
|
||||
const CACHE_MAGIC: u64 = 0x5056495A_43414348; // "PVIZC ACH"
|
||||
const CACHE_VERSION: u32 = 1;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct TreeInfo {
|
||||
pub name: String,
|
||||
pub dbi_index: u8,
|
||||
pub height: u16,
|
||||
pub branch_pages: u32,
|
||||
pub leaf_pages: u32,
|
||||
pub large_pages: u32,
|
||||
pub items: u64,
|
||||
pub root_pgno: u32,
|
||||
}
|
||||
|
||||
pub struct WalkResult {
|
||||
pub owner_map: Vec<u8>,
|
||||
pub page_count: usize,
|
||||
pub page_size: usize,
|
||||
pub tree_info: Vec<TreeInfo>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
struct TreeDescriptor {
|
||||
flags: u16,
|
||||
height: u16,
|
||||
dupfix_size: u32,
|
||||
root: u32,
|
||||
branch_pages: u32,
|
||||
leaf_pages: u32,
|
||||
large_pages: u32,
|
||||
sequence: u64,
|
||||
items: u64,
|
||||
mod_txnid: u64,
|
||||
}
|
||||
|
||||
struct DBIInfo {
|
||||
name: String,
|
||||
dbi_index: u8,
|
||||
tree: TreeDescriptor,
|
||||
}
|
||||
|
||||
struct MmapFile {
|
||||
ptr: *mut u8,
|
||||
len: usize,
|
||||
}
|
||||
|
||||
impl MmapFile {
|
||||
fn open(path: &Path) -> io::Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let len = file.metadata()?.len() as usize;
|
||||
if len == 0 {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "empty file"));
|
||||
}
|
||||
let ptr = unsafe {
|
||||
libc::mmap(
|
||||
std::ptr::null_mut(),
|
||||
len,
|
||||
libc::PROT_READ,
|
||||
libc::MAP_SHARED,
|
||||
file.as_raw_fd(),
|
||||
0,
|
||||
)
|
||||
};
|
||||
if ptr == libc::MAP_FAILED {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(Self { ptr: ptr as *mut u8, len })
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page(&self, pgno: usize, ps: usize) -> Option<&[u8]> {
|
||||
let off = pgno.checked_mul(ps)?;
|
||||
if off + ps > self.len {
|
||||
return None;
|
||||
}
|
||||
Some(unsafe { std::slice::from_raw_parts(self.ptr.add(off), ps) })
|
||||
}
|
||||
|
||||
fn evict(&self) {
|
||||
unsafe {
|
||||
libc::madvise(self.ptr as *mut libc::c_void, self.len, libc::MADV_DONTNEED);
|
||||
}
|
||||
tracing::info!(size_gb = self.len / (1024 * 1024 * 1024), "evicted walker mmap from page cache");
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for MmapFile {
|
||||
fn drop(&mut self) {
|
||||
unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.len); }
|
||||
}
|
||||
}
|
||||
|
||||
unsafe impl Send for MmapFile {}
|
||||
unsafe impl Sync for MmapFile {}
|
||||
|
||||
#[inline]
|
||||
fn u16_le(buf: &[u8], off: usize) -> u16 {
|
||||
u16::from_le_bytes([buf[off], buf[off + 1]])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn u32_le(buf: &[u8], off: usize) -> u32 {
|
||||
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn u64_le(buf: &[u8], off: usize) -> u64 {
|
||||
u64::from_le_bytes([
|
||||
buf[off], buf[off + 1], buf[off + 2], buf[off + 3],
|
||||
buf[off + 4], buf[off + 5], buf[off + 6], buf[off + 7],
|
||||
])
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_flags(buf: &[u8]) -> u16 {
|
||||
u16_le(buf, 0x0A)
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_nkeys(buf: &[u8]) -> usize {
|
||||
u16_le(buf, 0x0C) as usize / 2
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn page_overflow_count(buf: &[u8]) -> u32 {
|
||||
u32_le(buf, 0x0C)
|
||||
}
|
||||
|
||||
fn parse_tree_descriptor(buf: &[u8], off: usize) -> TreeDescriptor {
|
||||
TreeDescriptor {
|
||||
flags: u16_le(buf, off),
|
||||
height: u16_le(buf, off + 0x02),
|
||||
dupfix_size: u32_le(buf, off + 0x04),
|
||||
root: u32_le(buf, off + TREE_ROOT),
|
||||
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
|
||||
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
|
||||
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
|
||||
sequence: u64_le(buf, off + 0x18),
|
||||
items: u64_le(buf, off + TREE_ITEMS),
|
||||
mod_txnid: u64_le(buf, off + 0x28),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn claim(owner_map: &[AtomicU8], pgno: usize, dbi_index: u8, conflicts: &AtomicU64) -> bool {
|
||||
if pgno >= owner_map.len() {
|
||||
return false;
|
||||
}
|
||||
match owner_map[pgno].compare_exchange(UNREFERENCED, dbi_index, Ordering::Relaxed, Ordering::Relaxed) {
|
||||
Ok(_) => true,
|
||||
Err(existing) => {
|
||||
if existing != dbi_index {
|
||||
conflicts.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn walk_tree_claim(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
root_pgno: u32,
|
||||
page_count: usize,
|
||||
dbi_index: u8,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
) {
|
||||
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
|
||||
return;
|
||||
}
|
||||
|
||||
let mut stack: Vec<usize> = vec![root_pgno as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_index, conflicts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if op < page_count {
|
||||
claim(owner_map, op, dbi_index, conflicts);
|
||||
}
|
||||
}
|
||||
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
|
||||
if data_off + TREE_ROOT + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let sub_root = u32_le(buf, data_off + TREE_ROOT);
|
||||
if sub_root != P_INVALID {
|
||||
stack.push(sub_root as usize);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if flags & P_LARGE != 0 {
|
||||
let ov_count = page_overflow_count(buf) as usize;
|
||||
for op in (pgno + 1)..pgno + ov_count {
|
||||
if op < page_count {
|
||||
claim(owner_map, op, dbi_index, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn discover_named_dbis(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
main_root: u32,
|
||||
page_count: usize,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
dbi_main: u8,
|
||||
dbi_start: u8,
|
||||
) -> Vec<DBIInfo> {
|
||||
let mut named = Vec::new();
|
||||
if main_root == P_INVALID || main_root as usize >= page_count {
|
||||
return named;
|
||||
}
|
||||
|
||||
let mut next_index = dbi_start;
|
||||
let mut stack: Vec<usize> = vec![main_root as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_main, conflicts) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_TREE != 0 {
|
||||
let key_off = node_off + NODESIZE;
|
||||
if key_off + ksize > ps || data_off + 0x30 > ps {
|
||||
continue;
|
||||
}
|
||||
let key_bytes = &buf[key_off..key_off + ksize];
|
||||
let name = match key_bytes.iter().position(|&b| b == 0) {
|
||||
Some(nul) => &key_bytes[..nul],
|
||||
None => key_bytes,
|
||||
};
|
||||
let name = String::from_utf8_lossy(name).into_owned();
|
||||
let tree = parse_tree_descriptor(buf, data_off);
|
||||
named.push(DBIInfo { name, dbi_index: next_index, tree });
|
||||
next_index = next_index.wrapping_add(1);
|
||||
} else if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
claim(owner_map, op, dbi_main, conflicts);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
named
|
||||
}
|
||||
|
||||
fn mark_free_pages(
|
||||
mmap: &MmapFile,
|
||||
ps: usize,
|
||||
free_root: u32,
|
||||
page_count: usize,
|
||||
owner_map: &[AtomicU8],
|
||||
conflicts: &AtomicU64,
|
||||
dbi_free: u8,
|
||||
) -> u64 {
|
||||
let mut marked: u64 = 0;
|
||||
if free_root == P_INVALID || free_root as usize >= page_count {
|
||||
return marked;
|
||||
}
|
||||
|
||||
let mut stack: Vec<usize> = vec![free_root as usize];
|
||||
|
||||
while let Some(pgno) = stack.pop() {
|
||||
if pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
if !claim(owner_map, pgno, dbi_free, conflicts) {
|
||||
continue;
|
||||
}
|
||||
marked += 1;
|
||||
|
||||
let buf = match mmap.page(pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
let flags = page_flags(buf);
|
||||
|
||||
if flags & P_BRANCH != 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
|
||||
stack.push(child_pgno);
|
||||
}
|
||||
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
|
||||
let nkeys = page_nkeys(buf);
|
||||
for i in 0..nkeys {
|
||||
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
|
||||
let node_off = PAGEHDRSZ + node_rel;
|
||||
if node_off + NODESIZE > ps {
|
||||
continue;
|
||||
}
|
||||
let mn_flags = buf[node_off + 4];
|
||||
let ksize = u16_le(buf, node_off + 6) as usize;
|
||||
let dsize = u32_le(buf, node_off) as usize;
|
||||
let data_off = node_off + NODESIZE + ksize;
|
||||
|
||||
if mn_flags & N_BIG != 0 {
|
||||
if data_off + 4 > ps {
|
||||
continue;
|
||||
}
|
||||
let ov_pgno = u32_le(buf, data_off) as usize;
|
||||
if ov_pgno >= page_count {
|
||||
continue;
|
||||
}
|
||||
let ov_buf = match mmap.page(ov_pgno, ps) {
|
||||
Some(b) => b,
|
||||
None => continue,
|
||||
};
|
||||
let ov_count = page_overflow_count(ov_buf) as usize;
|
||||
for op in ov_pgno..ov_pgno + ov_count {
|
||||
if claim(owner_map, op, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
let ov_data_off = PAGEHDRSZ;
|
||||
if dsize >= 4 && ov_data_off + 4 <= ps {
|
||||
let pnl_count = u32_le(ov_buf, ov_data_off) as usize;
|
||||
let max_entries = (ps - ov_data_off - 4) / 4;
|
||||
let pnl_count = pnl_count.min(max_entries);
|
||||
for j in 0..pnl_count {
|
||||
let off = ov_data_off + 4 + j * 4;
|
||||
if off + 4 > ps { break; }
|
||||
let fp = u32_le(ov_buf, off) as usize;
|
||||
if fp < page_count {
|
||||
if claim(owner_map, fp, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if dsize >= 4 && data_off + 4 <= ps {
|
||||
let pnl_count = u32_le(buf, data_off) as usize;
|
||||
let max_entries = (ps - data_off - 4) / 4;
|
||||
let pnl_count = pnl_count.min(max_entries);
|
||||
for j in 0..pnl_count {
|
||||
let off = data_off + 4 + j * 4;
|
||||
if off + 4 > ps { break; }
|
||||
let fp = u32_le(buf, off) as usize;
|
||||
if fp < page_count {
|
||||
if claim(owner_map, fp, dbi_free, conflicts) {
|
||||
marked += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
marked
|
||||
}
|
||||
|
||||
fn cache_path(mdbx_path: &Path) -> std::path::PathBuf {
|
||||
mdbx_path.with_file_name("mdbx_pageviz_cache.bin")
|
||||
}
|
||||
|
||||
fn try_load_cache(
|
||||
path: &Path,
|
||||
file_size: u64,
|
||||
txnid: u64,
|
||||
page_size: usize,
|
||||
page_count: usize,
|
||||
) -> Option<WalkResult> {
|
||||
let cp = cache_path(path);
|
||||
let mut f = std::fs::File::open(&cp).ok()?;
|
||||
let mut hdr = [0u8; 40];
|
||||
f.read_exact(&mut hdr).ok()?;
|
||||
|
||||
let magic = u64::from_le_bytes(hdr[0..8].try_into().unwrap());
|
||||
let ver = u32::from_le_bytes(hdr[8..12].try_into().unwrap());
|
||||
let c_txnid = u64::from_le_bytes(hdr[12..20].try_into().unwrap());
|
||||
let c_pc = u64::from_le_bytes(hdr[20..28].try_into().unwrap());
|
||||
let c_ps = u32::from_le_bytes(hdr[28..32].try_into().unwrap());
|
||||
let c_fsz = u64::from_le_bytes(hdr[32..40].try_into().unwrap());
|
||||
|
||||
if magic != CACHE_MAGIC || ver != CACHE_VERSION {
|
||||
return None;
|
||||
}
|
||||
if c_txnid != txnid || c_pc as usize != page_count || c_ps as usize != page_size || c_fsz != file_size {
|
||||
tracing::info!(
|
||||
cache_txnid = c_txnid, current_txnid = txnid,
|
||||
"cache stale, will re-walk"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut ti_len_buf = [0u8; 4];
|
||||
f.read_exact(&mut ti_len_buf).ok()?;
|
||||
let ti_len = u32::from_le_bytes(ti_len_buf) as usize;
|
||||
|
||||
let mut ti_buf = vec![0u8; ti_len];
|
||||
f.read_exact(&mut ti_buf).ok()?;
|
||||
let tree_info: Vec<TreeInfo> = serde_json::from_slice(&ti_buf).ok()?;
|
||||
|
||||
let mut owner_map = vec![0u8; page_count];
|
||||
f.read_exact(&mut owner_map).ok()?;
|
||||
|
||||
tracing::info!(page_count, "loaded owner_map from cache");
|
||||
Some(WalkResult { owner_map, page_count, page_size, tree_info })
|
||||
}
|
||||
|
||||
fn save_cache(
|
||||
path: &Path,
|
||||
file_size: u64,
|
||||
txnid: u64,
|
||||
result: &WalkResult,
|
||||
) {
|
||||
let cp = cache_path(path);
|
||||
let tmp = cp.with_extension("tmp");
|
||||
|
||||
let write_inner = || -> io::Result<()> {
|
||||
let mut f = std::fs::File::create(&tmp)?;
|
||||
f.write_all(&CACHE_MAGIC.to_le_bytes())?;
|
||||
f.write_all(&CACHE_VERSION.to_le_bytes())?;
|
||||
f.write_all(&txnid.to_le_bytes())?;
|
||||
f.write_all(&(result.page_count as u64).to_le_bytes())?;
|
||||
f.write_all(&(result.page_size as u32).to_le_bytes())?;
|
||||
f.write_all(&file_size.to_le_bytes())?;
|
||||
|
||||
let ti_json = serde_json::to_vec(&result.tree_info).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
|
||||
f.write_all(&(ti_json.len() as u32).to_le_bytes())?;
|
||||
f.write_all(&ti_json)?;
|
||||
f.write_all(&result.owner_map)?;
|
||||
f.flush()?;
|
||||
Ok(())
|
||||
};
|
||||
|
||||
match write_inner() {
|
||||
Ok(()) => {
|
||||
if let Err(e) = std::fs::rename(&tmp, &cp) {
|
||||
tracing::warn!("failed to rename cache file: {e}");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
} else {
|
||||
tracing::info!(
|
||||
path = %cp.display(),
|
||||
size_mb = result.owner_map.len() / (1024 * 1024),
|
||||
"saved owner_map cache"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("failed to write cache: {e}");
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn walk_mdbx(path: &Path, name_to_dbi: &HashMap<&str, u8>) -> eyre::Result<WalkResult> {
|
||||
let t0 = Instant::now();
|
||||
|
||||
let mmap = MmapFile::open(path)?;
|
||||
let file_size = mmap.len as u64;
|
||||
|
||||
let meta0 = mmap.page(0, 4096).ok_or_else(|| eyre::eyre!("cannot read page 0"))?;
|
||||
let pflags = page_flags(meta0);
|
||||
eyre::ensure!(pflags & P_META != 0, "page 0 missing P_META flag");
|
||||
|
||||
let magic = u64_le(meta0, PAGEHDRSZ);
|
||||
eyre::ensure!((magic >> 8) == MDBX_MAGIC, "MDBX magic mismatch");
|
||||
|
||||
let candidates: &[usize] = &[4096, 8192, 16384, 32768, 65536, 1024, 2048];
|
||||
let geo_now_raw = u32_le(meta0, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
|
||||
|
||||
let mut ps = 4096usize;
|
||||
for &candidate in candidates {
|
||||
let mapped = geo_now_raw * candidate;
|
||||
if mapped >= mmap.len / 2 && mapped <= mmap.len * 4 {
|
||||
ps = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(page_size = ps, file_size_gb = file_size / (1024 * 1024 * 1024), "detected page size");
|
||||
|
||||
let meta_pages: Vec<&[u8]> = (0..3)
|
||||
.map(|i| mmap.page(i, ps).ok_or_else(|| eyre::eyre!("cannot read meta page {i}")))
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
let mut best_idx = 0usize;
|
||||
let mut best_txnid = 0u64;
|
||||
|
||||
for (i, meta) in meta_pages.iter().enumerate() {
|
||||
let m = u64_le(meta, PAGEHDRSZ);
|
||||
if (m >> 8) != MDBX_MAGIC {
|
||||
continue;
|
||||
}
|
||||
let txnid_a = u64_le(meta, PAGEHDRSZ + META_TXNID_A);
|
||||
let txnid_b = u64_le(meta, PAGEHDRSZ + META_TXNID_B);
|
||||
if txnid_a == txnid_b && txnid_a > best_txnid {
|
||||
best_txnid = txnid_a;
|
||||
best_idx = i;
|
||||
}
|
||||
}
|
||||
|
||||
eyre::ensure!(best_txnid > 0, "no valid meta page found");
|
||||
|
||||
let best = meta_pages[best_idx];
|
||||
let mb = PAGEHDRSZ;
|
||||
let geo_now = u32_le(best, mb + META_GEO + GEO_NOW) as usize;
|
||||
let page_count = geo_now;
|
||||
|
||||
tracing::info!(best_meta = best_idx, txnid = best_txnid, page_count, "selected best meta");
|
||||
|
||||
if let Some(cached) = try_load_cache(path, file_size, best_txnid, ps, page_count) {
|
||||
let elapsed = t0.elapsed();
|
||||
tracing::info!(elapsed_ms = elapsed.as_millis() as u64, "walk skipped (cache hit)");
|
||||
return Ok(cached);
|
||||
}
|
||||
|
||||
let free_tree = parse_tree_descriptor(best, mb + META_FREE_DB);
|
||||
let main_tree = parse_tree_descriptor(best, mb + META_MAIN_DB);
|
||||
|
||||
tracing::info!(
|
||||
page_count,
|
||||
free_root = free_tree.root,
|
||||
main_root = main_tree.root,
|
||||
"parsed meta, starting walk"
|
||||
);
|
||||
|
||||
let owner_map: Vec<AtomicU8> = (0..page_count).map(|_| AtomicU8::new(UNREFERENCED)).collect();
|
||||
let conflicts = AtomicU64::new(0);
|
||||
|
||||
for pgno in 0..3.min(page_count) {
|
||||
owner_map[pgno].store(DBI_META, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
let dbi_free: u8 = 1;
|
||||
let dbi_main: u8 = 2;
|
||||
let dbi_start: u8 = 3;
|
||||
|
||||
tracing::info!("discovering named DBIs via MainDB walk");
|
||||
let mut named_dbis =
|
||||
discover_named_dbis(&mmap, ps, main_tree.root, page_count, &owner_map, &conflicts, dbi_main, dbi_start);
|
||||
tracing::info!(count = named_dbis.len(), "found named DBIs");
|
||||
|
||||
for dbi in &mut named_dbis {
|
||||
if let Some(&mapped_idx) = name_to_dbi.get(dbi.name.as_str()) {
|
||||
dbi.dbi_index = mapped_idx;
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!("walking FreeDB");
|
||||
let free_marked =
|
||||
mark_free_pages(&mmap, ps, free_tree.root, page_count, &owner_map, &conflicts, dbi_free);
|
||||
tracing::info!(marked = free_marked, "FreeDB walk complete");
|
||||
|
||||
let walk_tasks: Vec<(u32, u8)> = named_dbis
|
||||
.iter()
|
||||
.filter(|d| d.tree.root != P_INVALID)
|
||||
.map(|d| (d.tree.root, d.dbi_index))
|
||||
.collect();
|
||||
|
||||
tracing::info!(count = walk_tasks.len(), "walking named DBIs in parallel");
|
||||
|
||||
walk_tasks
|
||||
.par_iter()
|
||||
.for_each(|&(root, dbi_idx)| {
|
||||
walk_tree_claim(&mmap, ps, root, page_count, dbi_idx, &owner_map, &conflicts);
|
||||
});
|
||||
|
||||
let final_conflicts = conflicts.load(Ordering::Relaxed);
|
||||
let owner_map_vec: Vec<u8> = owner_map.iter().map(|a| a.load(Ordering::Relaxed)).collect();
|
||||
let assigned = owner_map_vec.iter().filter(|&&b| b != UNREFERENCED).count();
|
||||
|
||||
let elapsed = t0.elapsed();
|
||||
tracing::info!(
|
||||
elapsed_ms = elapsed.as_millis() as u64,
|
||||
assigned,
|
||||
unreferenced = page_count - assigned,
|
||||
conflicts = final_conflicts,
|
||||
"walk complete"
|
||||
);
|
||||
|
||||
let mut tree_info = Vec::new();
|
||||
|
||||
tree_info.push(TreeInfo {
|
||||
name: "FreeDB".to_string(),
|
||||
dbi_index: dbi_free,
|
||||
height: free_tree.height,
|
||||
branch_pages: free_tree.branch_pages,
|
||||
leaf_pages: free_tree.leaf_pages,
|
||||
large_pages: free_tree.large_pages,
|
||||
items: free_tree.items,
|
||||
root_pgno: free_tree.root,
|
||||
});
|
||||
|
||||
tree_info.push(TreeInfo {
|
||||
name: "MainDB".to_string(),
|
||||
dbi_index: dbi_main,
|
||||
height: main_tree.height,
|
||||
branch_pages: main_tree.branch_pages,
|
||||
leaf_pages: main_tree.leaf_pages,
|
||||
large_pages: main_tree.large_pages,
|
||||
items: main_tree.items,
|
||||
root_pgno: main_tree.root,
|
||||
});
|
||||
|
||||
for dbi in &named_dbis {
|
||||
tree_info.push(TreeInfo {
|
||||
name: dbi.name.clone(),
|
||||
dbi_index: dbi.dbi_index,
|
||||
height: dbi.tree.height,
|
||||
branch_pages: dbi.tree.branch_pages,
|
||||
leaf_pages: dbi.tree.leaf_pages,
|
||||
large_pages: dbi.tree.large_pages,
|
||||
items: dbi.tree.items,
|
||||
root_pgno: dbi.tree.root,
|
||||
});
|
||||
}
|
||||
|
||||
let result = WalkResult { owner_map: owner_map_vec, page_count, page_size: ps, tree_info };
|
||||
|
||||
save_cache(path, file_size, best_txnid, &result);
|
||||
|
||||
mmap.evict();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
Reference in New Issue
Block a user