Compare commits

...

12 Commits
t4 ... pageviz

Author SHA1 Message Date
Ubuntu
784b4e9dd2 more 2026-02-13 02:03:19 +00:00
Ubuntu
2badbe3f3f ha 2026-02-13 01:08:22 +00:00
Ubuntu
f2cada2660 bump 2026-02-13 00:48:13 +00:00
Ubuntu
a2006c5f86 cas 2026-02-13 00:08:42 +00:00
Ubuntu
a95cf8fe8e Merge remote-tracking branch 'origin/main' into pageviz 2026-02-12 22:41:41 +00:00
Ubuntu
852551b790 moar 2026-02-12 22:41:08 +00:00
Ubuntu
483eaae67c color 2026-02-12 22:08:34 +00:00
Ubuntu
f76a748d92 color 2026-02-12 21:36:11 +00:00
Ubuntu
fb1039cf7a hello 2026-02-12 20:16:41 +00:00
Ubuntu
ffa681795f go 2026-02-12 19:26:04 +00:00
Ubuntu
cd7f43bdf0 pgviz2 2026-02-12 18:39:58 +00:00
Ubuntu
3de7589755 pagebiz 2026-02-12 15:09:49 +00:00
26 changed files with 5170 additions and 1 deletions

128
Cargo.lock generated
View File

@@ -1520,6 +1520,64 @@ version = "1.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
[[package]]
name = "axum"
version = "0.7.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
dependencies = [
"async-trait",
"axum-core",
"base64 0.22.1",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-util",
"itoa",
"matchit",
"memchr",
"mime",
"percent-encoding",
"pin-project-lite",
"rustversion",
"serde",
"serde_json",
"serde_path_to_error",
"serde_urlencoded",
"sha1",
"sync_wrapper",
"tokio",
"tokio-tungstenite 0.24.0",
"tower",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "axum-core"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"http-body",
"http-body-util",
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
name = "backon"
version = "1.6.0"
@@ -5793,6 +5851,12 @@ dependencies = [
"regex-automata",
]
[[package]]
name = "matchit"
version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
[[package]]
name = "memchr"
version = "2.8.0"
@@ -7762,11 +7826,13 @@ dependencies = [
"humantime",
"itertools 0.14.0",
"lz4",
"memmap2",
"metrics",
"parking_lot",
"proptest",
"proptest-arbitrary-interop",
"ratatui",
"rayon",
"reqwest",
"reth-chainspec",
"reth-cli",
@@ -7979,6 +8045,7 @@ dependencies = [
"reth-db-api",
"reth-fs-util",
"reth-libmdbx",
"reth-mdbx-viz",
"reth-metrics",
"reth-nippy-jar",
"reth-primitives-traits",
@@ -7993,6 +8060,7 @@ dependencies = [
"sysinfo",
"tempfile",
"thiserror 2.0.18",
"tokio",
"tracing",
]
@@ -8393,6 +8461,7 @@ dependencies = [
"reth-evm-ethereum",
"reth-execution-types",
"reth-exex-types",
"reth-mdbx-viz",
"reth-metrics",
"reth-network-p2p",
"reth-node-ethereum",
@@ -9029,6 +9098,24 @@ dependencies = [
"cc",
]
[[package]]
name = "reth-mdbx-viz"
version = "1.11.0"
dependencies = [
"axum",
"clap",
"eyre",
"libc",
"rayon",
"reth-libmdbx",
"reth-mdbx-sys",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber 0.3.22",
]
[[package]]
name = "reth-metrics"
version = "1.11.0"
@@ -11475,6 +11562,17 @@ dependencies = [
"zmij",
]
[[package]]
name = "serde_path_to_error"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457"
dependencies = [
"itoa",
"serde",
"serde_core",
]
[[package]]
name = "serde_qs"
version = "0.8.5"
@@ -12267,6 +12365,18 @@ dependencies = [
"tokio-util",
]
[[package]]
name = "tokio-tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "edc5f74e248dc973e0dbb7b74c7e0d6fcc301c694ff50049504004ef4d0cdcd9"
dependencies = [
"futures-util",
"log",
"tokio",
"tungstenite 0.24.0",
]
[[package]]
name = "tokio-tungstenite"
version = "0.26.2"
@@ -12706,6 +12816,24 @@ version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
[[package]]
name = "tungstenite"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "18e5b8366ee7a95b16d32197d0b2604b43a0be89dc5fac9f8e96ccafbaedda8a"
dependencies = [
"byteorder",
"bytes",
"data-encoding",
"http",
"httparse",
"log",
"rand 0.8.5",
"sha1",
"thiserror 1.0.69",
"utf-8",
]
[[package]]
name = "tungstenite"
version = "0.26.2"

View File

@@ -110,6 +110,7 @@ members = [
"crates/storage/errors/",
"crates/storage/libmdbx-rs/",
"crates/storage/libmdbx-rs/mdbx-sys/",
"crates/storage/mdbx-viz/",
"crates/storage/nippy-jar/",
"crates/storage/provider/",
"crates/storage/storage-api/",
@@ -377,6 +378,7 @@ reth-invalid-block-hooks = { path = "crates/engine/invalid-block-hooks" }
reth-ipc = { path = "crates/rpc/ipc" }
reth-libmdbx = { path = "crates/storage/libmdbx-rs" }
reth-mdbx-sys = { path = "crates/storage/libmdbx-rs/mdbx-sys" }
reth-mdbx-viz = { path = "crates/storage/mdbx-viz" }
reth-metrics = { path = "crates/metrics" }
reth-net-banlist = { path = "crates/net/banlist" }
reth-net-nat = { path = "crates/net/nat" }

View File

@@ -190,6 +190,7 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
pageviz = ["reth-db/pageviz", "reth-node-builder/pageviz"]
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
edge = ["rocksdb"]

View File

@@ -87,6 +87,8 @@ tokio-stream.workspace = true
reqwest.workspace = true
url.workspace = true
metrics.workspace = true
memmap2.workspace = true
rayon.workspace = true
# io
fdlimit.workspace = true

View File

@@ -0,0 +1,564 @@
use clap::Parser;
use eyre::Context;
use memmap2::Mmap;
use rayon::prelude::*;
use reth_db_api::tables::Tables;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use std::{
collections::HashMap,
fs,
io::Write,
path::PathBuf,
time::Instant,
};
const PAGEHDRSZ: usize = 20;
const NODESIZE: usize = 8;
const P_BRANCH: u16 = 0x01;
const P_LEAF: u16 = 0x02;
const P_LARGE: u16 = 0x04;
const P_META: u16 = 0x08;
const P_DUPFIX: u16 = 0x20;
const N_BIG: u8 = 0x01;
const N_TREE: u8 = 0x02;
const N_DUP: u8 = 0x04;
const P_INVALID: u32 = 0xFFFFFFFF;
const UNREFERENCED: u8 = 0xFF;
const DBI_META: u8 = 0xFE;
const META_GEO: usize = 0x14;
const META_FREE_DB: usize = 0x28;
const META_MAIN_DB: usize = 0x58;
const META_TXNID_A: usize = 0x08;
const META_TXNID_B: usize = 0xB0;
const GEO_NOW: usize = 0x0C;
const TREE_ROOT: usize = 0x08;
const TREE_BRANCH_PAGES: usize = 0x0C;
const TREE_LEAF_PAGES: usize = 0x10;
const TREE_LARGE_PAGES: usize = 0x14;
const TREE_ITEMS: usize = 0x20;
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
fn u16_le(buf: &[u8], off: usize) -> u16 {
u16::from_le_bytes([buf[off], buf[off + 1]])
}
fn u32_le(buf: &[u8], off: usize) -> u32 {
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
}
fn u64_le(buf: &[u8], off: usize) -> u64 {
u64::from_le_bytes(buf[off..off + 8].try_into().unwrap())
}
#[derive(Debug, Clone)]
struct TreeDescriptor {
root: u32,
branch_pages: u32,
leaf_pages: u32,
large_pages: u32,
items: u64,
}
impl TreeDescriptor {
fn parse(buf: &[u8], off: usize) -> Self {
Self {
root: u32_le(buf, off + TREE_ROOT),
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
items: u64_le(buf, off + TREE_ITEMS),
}
}
fn total_pages(&self) -> u64 {
self.branch_pages as u64 + self.leaf_pages as u64 + self.large_pages as u64
}
}
#[derive(Debug, Clone)]
struct DBIInfo {
name: String,
dbi_index: u8,
tree: TreeDescriptor,
}
fn page_flags(buf: &[u8], page_off: usize) -> u16 {
u16_le(buf, page_off + 0x0A)
}
fn page_nkeys(buf: &[u8], page_off: usize) -> usize {
u16_le(buf, page_off + 0x0C) as usize / 2
}
fn page_overflow_count(buf: &[u8], page_off: usize) -> u32 {
u32_le(buf, page_off + 0x0C)
}
fn mark(owner_map: &mut [u8], pgno: usize, dbi_index: u8, conflicts: &mut u64) -> bool {
if pgno >= owner_map.len() {
return false;
}
let prev = owner_map[pgno];
if prev == UNREFERENCED {
owner_map[pgno] = dbi_index;
return true;
}
if prev != dbi_index {
*conflicts += 1;
}
false
}
fn walk_tree_collect(
buf: &[u8],
ps: usize,
root_pgno: u32,
page_count: usize,
dbi_index: u8,
) -> Vec<(usize, u8)> {
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
return Vec::new();
}
let mut result: Vec<(usize, u8)> = Vec::new();
let mut visited = vec![false; page_count];
let mut stack: Vec<u32> = vec![root_pgno];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= page_count || visited[pgno_usize] {
continue;
}
visited[pgno_usize] = true;
result.push((pgno_usize, dbi_index));
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() {
continue;
}
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + 4 > buf.len() { break; }
stack.push(u32_le(buf, node_abs));
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= page_count { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if op < page_count && !visited[op] {
visited[op] = true;
result.push((op, dbi_index));
}
}
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
if data_off + 48 > buf.len() { continue; }
let sub = TreeDescriptor::parse(buf, data_off);
if sub.root != P_INVALID {
stack.push(sub.root);
}
}
}
} else if flags & P_LARGE != 0 {
let ov_count = page_overflow_count(buf, page_off) as usize;
for op in (pgno_usize + 1)..(pgno_usize + ov_count) {
if op < page_count && !visited[op] {
visited[op] = true;
result.push((op, dbi_index));
}
}
}
}
result
}
fn discover_named_dbis(
buf: &[u8],
ps: usize,
main_root: u32,
owner_map: &mut [u8],
conflicts: &mut u64,
dbi_main: u8,
dbi_start: u8,
) -> Vec<DBIInfo> {
if main_root == P_INVALID || main_root as usize >= owner_map.len() {
return Vec::new();
}
let mut named: Vec<DBIInfo> = Vec::new();
let mut next_index = dbi_start;
let mut stack: Vec<u32> = vec![main_root];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= owner_map.len() { continue; }
if !mark(owner_map, pgno_usize, dbi_main, conflicts) { continue; }
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() { continue; }
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + 4 > buf.len() { break; }
let child_pgno = u32_le(buf, node_abs);
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_TREE != 0 {
let key_off = node_abs + NODESIZE;
if key_off + ksize > buf.len() || data_off + 48 > buf.len() { continue; }
let name_bytes = &buf[key_off..key_off + ksize];
let name = String::from_utf8_lossy(
name_bytes.split(|&b| b == 0).next().unwrap_or(name_bytes)
).to_string();
let tree = TreeDescriptor::parse(buf, data_off);
named.push(DBIInfo { name, dbi_index: next_index, tree });
next_index = next_index.saturating_add(1);
} else if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= owner_map.len() { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
mark(owner_map, op, dbi_main, conflicts);
}
}
}
}
}
named
}
fn mark_free_pages(
buf: &[u8],
ps: usize,
free_root: u32,
owner_map: &mut [u8],
conflicts: &mut u64,
dbi_free: u8,
) -> u64 {
if free_root == P_INVALID || free_root as usize >= owner_map.len() {
return 0;
}
let mut marked: u64 = 0;
let mut stack: Vec<u32> = vec![free_root];
while let Some(pgno) = stack.pop() {
let pgno_usize = pgno as usize;
if pgno_usize >= owner_map.len() { continue; }
if !mark(owner_map, pgno_usize, dbi_free, conflicts) { continue; }
marked += 1;
let page_off = pgno_usize * ps;
if page_off + PAGEHDRSZ > buf.len() { continue; }
let flags = page_flags(buf, page_off);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let child_pgno = u32_le(buf, page_off + PAGEHDRSZ + node_rel);
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf, page_off);
for i in 0..nkeys {
let idx_off = page_off + PAGEHDRSZ + i * 2;
if idx_off + 2 > buf.len() { break; }
let node_rel = u16_le(buf, idx_off) as usize;
let node_abs = page_off + PAGEHDRSZ + node_rel;
if node_abs + NODESIZE > buf.len() { break; }
let mn_flags = buf[node_abs + 4];
let ksize = u16_le(buf, node_abs + 6) as usize;
let dsize = u32_le(buf, node_abs) as usize;
let data_off = node_abs + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > buf.len() { continue; }
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= owner_map.len() { continue; }
let ov_page_off = ov_pgno * ps;
if ov_page_off + PAGEHDRSZ > buf.len() { continue; }
let ov_count = page_overflow_count(buf, ov_page_off) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if mark(owner_map, op, dbi_free, conflicts) { marked += 1; }
}
let ov_data_off = ov_pgno * ps + PAGEHDRSZ;
if dsize >= 4 && ov_data_off + 4 <= buf.len() {
let pnl_count = u32_le(buf, ov_data_off) as usize;
for j in 0..pnl_count {
let fp_off = ov_data_off + 4 + j * 4;
if fp_off + 4 > buf.len() { break; }
let fp = u32_le(buf, fp_off) as usize;
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
owner_map[fp] = dbi_free;
marked += 1;
}
}
}
} else {
if dsize >= 4 && data_off + 4 <= buf.len() {
let pnl_count = u32_le(buf, data_off) as usize;
for j in 0..pnl_count {
let fp_off = data_off + 4 + j * 4;
if fp_off + 4 > buf.len() { break; }
let fp = u32_le(buf, fp_off) as usize;
if fp < owner_map.len() && owner_map[fp] == UNREFERENCED {
owner_map[fp] = dbi_free;
marked += 1;
}
}
}
}
}
}
}
marked
}
#[derive(Parser, Debug)]
pub struct Command {
#[arg(short, long, default_value = "owner_map.bin")]
output: PathBuf,
}
impl Command {
pub fn execute<N: NodeTypesWithDB>(
self,
data_dir: ChainPath<DataDirPath>,
_tool: &DbTool<N>,
) -> eyre::Result<()> {
let db_path = data_dir.db().join("mdbx.dat");
eyre::ensure!(db_path.exists(), "mdbx.dat not found at {:?}", db_path);
let t0 = Instant::now();
let file = fs::File::open(&db_path)
.wrap_err_with(|| format!("Failed to open {}", db_path.display()))?;
let mmap = unsafe { Mmap::map(&file)? };
let buf: &[u8] = &mmap;
if buf.len() < PAGEHDRSZ + 0xC0 {
eyre::bail!("File too small for MDBX meta page");
}
let flags0 = u16_le(buf, 0x0A);
if flags0 & P_META == 0 {
eyre::bail!("Page 0 missing P_META flag");
}
let magic = u64_le(buf, PAGEHDRSZ);
if (magic >> 8) != MDBX_MAGIC {
eyre::bail!("MDBX magic mismatch");
}
let candidates = [4096usize, 8192, 16384, 32768, 65536];
let mut ps = 4096usize;
let geo_now_raw = u32_le(buf, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
for &candidate in &candidates {
let mapped = geo_now_raw * candidate;
if mapped >= buf.len() / 2 && mapped <= buf.len() * 4 {
ps = candidate;
break;
}
}
let mut best_txnid: u64 = 0;
let mut best_meta_base: usize = 0;
for pgno in 0..3usize {
let page_base = pgno * ps;
let meta_base = page_base + PAGEHDRSZ;
if meta_base + 0xC0 > buf.len() { continue; }
let pflags = u16_le(buf, page_base + 0x0A);
if pflags & P_META == 0 { continue; }
let m = u64_le(buf, meta_base);
if (m >> 8) != MDBX_MAGIC { continue; }
let txnid_a = u64_le(buf, meta_base + META_TXNID_A);
let txnid_b = u64_le(buf, meta_base + META_TXNID_B);
if txnid_a == txnid_b && txnid_a > best_txnid {
best_txnid = txnid_a;
best_meta_base = meta_base;
}
}
eyre::ensure!(best_txnid > 0, "No valid meta page found");
let page_count = u32_le(buf, best_meta_base + META_GEO + GEO_NOW) as usize;
let free_tree = TreeDescriptor::parse(buf, best_meta_base + META_FREE_DB);
let main_tree = TreeDescriptor::parse(buf, best_meta_base + META_MAIN_DB);
println!("MDBX owner map generator (parallel)");
println!(" page_size: {ps}");
println!(" page_count: {page_count}");
println!(" txnid: {best_txnid}");
println!(" FreeDB root: {} ({} items)", free_tree.root, free_tree.items);
println!(" MainDB root: {} ({} items)", main_tree.root, main_tree.items);
println!();
let mut owner_map = vec![UNREFERENCED; page_count];
let mut conflicts: u64 = 0;
for pgno in 0..std::cmp::min(3, page_count) {
owner_map[pgno] = DBI_META;
}
println!("Discovering named DBIs via MainDB walk...");
let discovered = discover_named_dbis(
buf, ps, main_tree.root, &mut owner_map, &mut conflicts, 1, 2,
);
let mut name_to_dbi: HashMap<&str, u8> = HashMap::new();
for (idx, table) in Tables::ALL.iter().enumerate() {
name_to_dbi.insert(table.name(), (idx + 2) as u8);
}
let mut named_dbis: Vec<DBIInfo> = Vec::new();
let mut remap: HashMap<u8, u8> = HashMap::new();
for dbi in &discovered {
if let Some(&real_dbi) = name_to_dbi.get(dbi.name.as_str()) {
remap.insert(dbi.dbi_index, real_dbi);
named_dbis.push(DBIInfo {
name: dbi.name.clone(),
dbi_index: real_dbi,
tree: dbi.tree.clone(),
});
} else {
named_dbis.push(dbi.clone());
}
}
for byte in owner_map.iter_mut() {
if let Some(&new_dbi) = remap.get(byte) {
*byte = new_dbi;
}
}
println!("Found {} named DBI(s)", named_dbis.len());
println!("Walking FreeDB / GC...");
let free_marked = mark_free_pages(buf, ps, free_tree.root, &mut owner_map, &mut conflicts, 0);
println!("FreeDB: {} pages marked", free_marked);
println!("Walking {} named DBIs in parallel...", named_dbis.len());
let t_par = Instant::now();
let results: Vec<(String, u8, Vec<(usize, u8)>)> = named_dbis
.par_iter()
.filter(|dbi| dbi.tree.root != P_INVALID)
.map(|dbi| {
let pages = walk_tree_collect(buf, ps, dbi.tree.root, page_count, dbi.dbi_index);
(dbi.name.clone(), dbi.dbi_index, pages)
})
.collect();
for (name, dbi_index, pages) in &results {
let count = pages.len();
for &(pgno, idx) in pages {
mark(&mut owner_map, pgno, idx, &mut conflicts);
}
println!(" [{:2}] {:30} {:>10} pages", dbi_index, name, count);
}
println!("Parallel walk done in {:.2}s", t_par.elapsed().as_secs_f64());
let elapsed = t0.elapsed();
let mut counts: HashMap<u8, u64> = HashMap::new();
let mut unreferenced: u64 = 0;
for &b in &owner_map {
if b == UNREFERENCED {
unreferenced += 1;
} else {
*counts.entry(b).or_default() += 1;
}
}
println!();
println!("Walk complete in {:.2}s", elapsed.as_secs_f64());
println!(" Total pages: {page_count}");
println!(" Unreferenced: {unreferenced}");
println!(" Conflicts: {conflicts}");
println!();
if let Some(&c) = counts.get(&DBI_META) {
println!(" [0xFE] {:30} {:>10} pages", "<meta>", c);
}
if let Some(&c) = counts.get(&0) {
println!(" [ 0] {:30} {:>10} pages", "<free/GC>", c);
}
if let Some(&c) = counts.get(&1) {
println!(" [ 1] {:30} {:>10} pages", "<main>", c);
}
let mut sorted_dbis = named_dbis.clone();
sorted_dbis.sort_by_key(|d| d.dbi_index);
for dbi in &sorted_dbis {
let walked = counts.get(&dbi.dbi_index).copied().unwrap_or(0);
let expected = dbi.tree.total_pages();
let mismatch = if expected > 0 && walked != expected {
format!(" !! MISMATCH (tree_t says {})", expected)
} else {
String::new()
};
println!(" [{:3}] {:30} {:>10} pages{}", dbi.dbi_index, dbi.name, walked, mismatch);
}
println!(" [0xFF] {:30} {:>10} pages", "<unreferenced>", unreferenced);
let mut out = fs::File::create(&self.output)?;
out.write_all(&owner_map)?;
println!();
println!("Written {} bytes to {}", owner_map.len(), self.output.display());
Ok(())
}
}

View File

@@ -21,6 +21,7 @@ mod settings;
mod state;
mod static_file_header;
mod stats;
pub mod gen_ownermap;
/// DB List TUI
mod tui;
@@ -71,6 +72,8 @@ pub enum Subcommands {
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
/// Generate a page owner map for MDBX page visualization
GenOwnermap(gen_ownermap::Command),
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
@@ -214,6 +217,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::GenOwnermap(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(data_dir, &tool)?;
});
}
}
Ok(())

View File

@@ -73,6 +73,7 @@ reth-prune-types = { workspace = true, optional = true }
reth-stages = { workspace = true, optional = true }
reth-static-file = { workspace = true, optional = true }
reth-tracing = { workspace = true, optional = true }
reth-mdbx-viz = { workspace = true, optional = true }
[dev-dependencies]
# reth
@@ -150,6 +151,7 @@ rocksdb = [
"reth-e2e-test-utils/rocksdb",
]
edge = ["rocksdb"]
pageviz = ["dep:reth-mdbx-viz"]
[[test]]
name = "e2e_testsuite"

View File

@@ -2677,8 +2677,23 @@ where
let start = Instant::now();
#[cfg(feature = "pageviz")]
reth_mdbx_viz::pageviz_emit_block_marker(block_num_hash.number, true, 0, 0, 0);
#[cfg(feature = "pageviz")]
let pageviz_start = std::time::Instant::now();
let executed = execute(&mut self.payload_validator, input, ctx)?;
#[cfg(feature = "pageviz")]
reth_mdbx_viz::pageviz_emit_block_marker(
block_num_hash.number,
false,
executed.recovered_block().senders().len() as u16,
pageviz_start.elapsed().as_millis() as u32,
executed.recovered_block().gas_used(),
);
// if the parent is the canonical head, we can insert the block as the pending block
if self.state.tree_state.canonical_block_hash() == executed.recovered_block().parent_hash()
{

View File

@@ -129,3 +129,4 @@ op = [
"reth-evm/op",
"reth-primitives-traits/op",
]
pageviz = ["reth-engine-tree/pageviz"]

View File

@@ -27,6 +27,8 @@ alloy-primitives.workspace = true
# mdbx
reth-libmdbx = { workspace = true, optional = true, features = ["return-borrowed", "read-tx-timeouts"] }
eyre = { workspace = true, optional = true }
reth-mdbx-viz = { workspace = true, optional = true }
tokio = { workspace = true, optional = true, features = ["full"] }
# metrics
reth-metrics = { workspace = true, optional = true }
@@ -97,6 +99,7 @@ op = [
"reth-db-api/op",
"reth-primitives-traits/op",
]
pageviz = ["reth-libmdbx/pageviz", "dep:reth-mdbx-viz", "dep:tokio", "mdbx"]
disable-lock = []
[[bench]]

View File

@@ -522,6 +522,81 @@ impl DatabaseEnv {
self
}
/// Starts the real-time page access visualization server.
///
/// This enables the C-level page access instrumentation hooks,
/// starts a background drainer thread that coalesces events,
/// and launches an HTTP+WebSocket server on the given port.
///
/// The server serves an interactive HTML visualization at `http://localhost:{port}/`
/// showing page accesses in real-time with fading highlights:
/// - Blue = read, Red = write, Yellow = free
#[cfg(feature = "pageviz")]
pub fn start_pageviz(&self, port: u16, db_path: std::path::PathBuf) {
use reth_libmdbx::pageviz::PageVizDrainer;
let max_dbi = self.dbis.values().copied().max().unwrap_or(0) as usize;
let mut dbi_names = vec![String::new(); max_dbi + 1];
dbi_names[0] = "FREE_DBI".to_string();
if dbi_names.len() > 1 {
dbi_names[1] = "MAIN_DBI".to_string();
}
for (name, &dbi) in self.dbis.iter() {
dbi_names[dbi as usize] = name.to_string();
}
let mut name_to_dbi = std::collections::HashMap::new();
for (&name, &dbi) in self.dbis.iter() {
name_to_dbi.insert(name, dbi as u8);
}
let mdbx_dat = db_path.join("mdbx.dat");
let walk = match reth_mdbx_viz::walker::walk_mdbx(&mdbx_dat, &name_to_dbi) {
Ok(w) => w,
Err(e) => {
reth_tracing::tracing::error!("pageviz: failed to walk B-tree: {e}");
return;
}
};
let page_count = walk.page_count;
let page_size = walk.page_size;
let mut drainer = PageVizDrainer::new();
drainer.enable();
let rx = drainer.start(60);
let config = reth_mdbx_viz::VizConfig {
port,
page_count: page_count as u64,
page_size: page_size as u32,
dbi_names,
owner_map: walk.owner_map,
tree_info: walk.tree_info,
db_path: mdbx_dat,
};
// Spawn the viz server on a background tokio runtime
std::thread::Builder::new()
.name("mdbx-viz-server".into())
.spawn(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.expect("failed to build tokio runtime for pageviz");
rt.block_on(async {
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
reth_tracing::tracing::error!("pageviz server error: {e}");
}
});
// Keep drainer alive while server runs
drop(drainer);
})
.expect("failed to spawn pageviz server thread");
reth_tracing::tracing::info!("pageviz server started on port {port}");
}
/// Creates all the tables defined in [`Tables`], if necessary.
///
/// This keeps tracks of the created table handles and stores them for better efficiency.

View File

@@ -45,11 +45,22 @@ pub fn init_db_for<P: AsRef<Path>, TS: TableSet>(
path: P,
args: DatabaseArguments,
) -> eyre::Result<DatabaseEnv> {
let db_path = path.as_ref().to_path_buf();
let client_version = args.client_version().clone();
let mut db = create_db(path, args)?;
db.create_and_track_tables_for::<TS>()?;
db.record_client_version(client_version)?;
drop_orphan_tables(&db);
#[cfg(feature = "pageviz")]
{
let port = std::env::var("RETH_PAGEVIZ_PORT")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(3141u16);
db.start_pageviz(port, db_path);
}
Ok(db)
}

View File

@@ -28,6 +28,7 @@ dashmap = { workspace = true, features = ["inline"], optional = true }
default = []
return-borrowed = []
read-tx-timeouts = ["dep:dashmap"]
pageviz = ["reth-mdbx-sys/pageviz"]
[dev-dependencies]
criterion.workspace = true

View File

@@ -11,3 +11,7 @@ repository.workspace = true
[build-dependencies]
cc.workspace = true
bindgen = { workspace = true, features = ["runtime"] }
[features]
default = []
pageviz = []

View File

@@ -30,6 +30,11 @@ fn main() {
#[cfg(not(debug_assertions))]
cc.define("MDBX_DEBUG", "0").define("NDEBUG", None);
// Enable page visualization instrumentation
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
cc.define("MDBX_PAGEVIZ", "1");
}
// Propagate `-C target-cpu=native`
let rustflags = env::var("CARGO_ENCODED_RUSTFLAGS").unwrap();
if rustflags.contains("target-cpu=native") &&
@@ -38,7 +43,11 @@ fn main() {
cc.flag("-march=native");
}
cc.file(mdbx.join("mdbx.c")).compile("libmdbx.a");
cc.file(mdbx.join("mdbx.c"));
if std::env::var("CARGO_FEATURE_PAGEVIZ").is_ok() {
cc.file(mdbx.join("mdbx_pageviz.c"));
}
cc.compile("libmdbx.a");
}
fn generate_bindings(mdbx: &Path, out_file: &Path) {

View File

@@ -13,6 +13,7 @@
#include MDBX_CONFIG_H
#endif
/* Undefine the NDEBUG if debugging is enforced by MDBX_DEBUG */
#if (defined(MDBX_DEBUG) && MDBX_DEBUG > 0) || (defined(MDBX_FORCE_ASSERTIONS) && MDBX_FORCE_ASSERTIONS)
#undef NDEBUG
@@ -46,6 +47,7 @@
#define _DARWIN_C_SOURCE
#endif /* _DARWIN_C_SOURCE */
#include "mdbx_pageviz.h"
#if (defined(__MINGW__) || defined(__MINGW32__) || defined(__MINGW64__)) && !defined(__USE_MINGW_ANSI_STDIO)
#define __USE_MINGW_ANSI_STDIO 1
#endif /* MinGW */
@@ -6193,6 +6195,7 @@ MDBX_INTERNAL int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const p
static inline int page_touch(MDBX_cursor *mc) {
page_t *const mp = mc->pg[mc->top];
MDBX_txn *txn = mc->txn;
MDBX_PAGEVIZ_WRITE(mc, mp->pgno);
tASSERT(txn, mc->txn->flags & MDBX_TXN_DIRTY);
tASSERT(txn, F_ISSET(*cursor_dbi_state(mc), DBI_LINDO | DBI_VALID | DBI_DIRTY));
@@ -19926,6 +19929,9 @@ __cold int dxb_resize(MDBX_env *const env, const pgno_t used_pgno, const pgno_t
eASSERT(env, env->dxb_mmap.limit >= env->dxb_mmap.current);
if (rc == MDBX_SUCCESS) {
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
#endif
eASSERT(env, limit_bytes == env->dxb_mmap.limit);
eASSERT(env, size_bytes <= env->dxb_mmap.filesize);
if (mode == explicit_resize)
@@ -20339,6 +20345,10 @@ __cold int dxb_setup(MDBX_env *env, const int lck_rc, const mdbx_mode_t mode_bit
if (unlikely(err != MDBX_SUCCESS))
return err;
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
mdbx_pageviz_set_mapping(env->dxb_mmap.base, env->dxb_mmap.current, env->ps);
#endif
#if defined(MADV_DONTDUMP)
err =
madvise(env->dxb_mmap.base, env->dxb_mmap.limit, MADV_DONTDUMP) ? ignore_enosys_and_eagain(errno) : MDBX_SUCCESS;
@@ -23090,6 +23100,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
return err;
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
pnl_append_prereserved(txn->tw.retired_pages, lp->pgno);
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
@@ -23104,6 +23115,7 @@ static int gcu_loose(MDBX_txn *txn, gcu_t *ctx) {
for (page_t *lp = txn->tw.loose_pages; lp; lp = page_next(lp)) {
tASSERT(txn, lp->flags == P_LOOSE);
loose[++count] = lp->pgno;
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, 0, lp->pgno);
MDBX_ASAN_UNPOISON_MEMORY_REGION(&page_next(lp), sizeof(page_t *));
VALGRIND_MAKE_MEM_DEFINED(&page_next(lp), sizeof(page_t *));
}
@@ -31626,6 +31638,7 @@ static __always_inline pgr_t page_get_inline(const uint16_t ILL, const MDBX_curs
if (unlikely(r.err != MDBX_SUCCESS))
goto bailout;
#endif /* MDBX_DISABLE_VALIDATION */
MDBX_PAGEVIZ_READ(mc, pgno);
return r;
}
@@ -31842,6 +31855,7 @@ pgr_t page_new(MDBX_cursor *mc, const unsigned flags) {
return ret;
DEBUG("db %zu allocated new page %" PRIaPGNO, cursor_dbi(mc), ret.page->pgno);
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
ret.page->flags = (uint16_t)flags;
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
@@ -31870,6 +31884,7 @@ pgr_t page_new_large(MDBX_cursor *mc, const size_t npages) {
return ret;
DEBUG("dbi %zu allocated new large-page %" PRIaPGNO ", num %zu", cursor_dbi(mc), ret.page->pgno, npages);
MDBX_PAGEVIZ_WRITE(mc, ret.page->pgno);
ret.page->flags = P_LARGE;
cASSERT(mc, *cursor_dbi_state(mc) & DBI_DIRTY);
cASSERT(mc, mc->txn->flags & MDBX_TXN_DIRTY);
@@ -32015,6 +32030,7 @@ __hot int page_touch_unmodifable(MDBX_txn *txn, MDBX_cursor *mc, const page_t *c
DEBUG("touched db %d page %" PRIaPGNO " -> %" PRIaPGNO, cursor_dbi_dbg(mc), mp->pgno, pgno);
tASSERT(txn, mp->pgno != pgno);
pnl_append_prereserved(txn->tw.retired_pages, mp->pgno);
MDBX_PAGEVIZ_FREE(mc, mp->pgno);
/* Update the parent page, if any, to point to the new page */
if (likely(mc->top)) {
page_t *parent = mc->pg[mc->top - 1];
@@ -32211,6 +32227,7 @@ int page_retire_ex(MDBX_cursor *mc, const pgno_t pgno, page_t *mp /* maybe null
unsigned pageflags /* maybe unknown/zero */) {
int rc;
MDBX_txn *const txn = mc->txn;
MDBX_PAGEVIZ_FREE(mc, pgno);
tASSERT(txn, !mp || (mp->pgno == pgno && mp->flags == pageflags));
/* During deleting entire subtrees, it is reasonable and possible to avoid

View File

@@ -0,0 +1,119 @@
#include "mdbx_pageviz.h"
#if defined(MDBX_PAGEVIZ) && MDBX_PAGEVIZ
/* ── Global state ─────────────────────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_global = NULL;
/* ── Thread-local ring index ──────────────────────────────────────────── */
MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring = MDBX_PAGEVIZ_TLS_UNREGISTERED;
/* ── Lifecycle ────────────────────────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_create(void) {
mdbx_pageviz_state_t *state = calloc(1, sizeof(mdbx_pageviz_state_t));
if (state)
mdbx_pageviz_global = state;
return state;
}
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state) {
if (!state)
return;
if (mdbx_pageviz_global == state)
mdbx_pageviz_global = NULL;
free(state);
}
/* ── Enable / Disable ─────────────────────────────────────────────────── */
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state) {
atomic_store_explicit(&state->enabled, 1, memory_order_release);
}
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state) {
atomic_store_explicit(&state->enabled, 0, memory_order_release);
}
/* ── Drain ────────────────────────────────────────────────────────────── */
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
uint64_t *out_buf, uint32_t max_count) {
if (!state || ring_idx >= MDBX_PAGEVIZ_MAX_RINGS || !out_buf || max_count == 0)
return 0;
mdbx_pageviz_ring_t *ring = &state->rings[ring_idx];
uint32_t head = atomic_load_explicit(&ring->published_head,
memory_order_acquire);
uint32_t tail = atomic_load_explicit(&ring->consumer_tail,
memory_order_relaxed);
if (head == tail)
return 0;
uint32_t avail = head - tail;
uint32_t count = avail < max_count ? avail : max_count;
for (uint32_t i = 0; i < count; i++) {
uint32_t slot = (tail + i) & MDBX_PAGEVIZ_RING_MASK;
out_buf[i] = ring->events[slot];
}
atomic_store_explicit(&ring->consumer_tail, tail + count,
memory_order_release);
return count;
}
/* ── Queries ──────────────────────────────────────────────────────────── */
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state) {
return atomic_load_explicit(&state->ring_count, memory_order_relaxed);
}
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx) {
return atomic_load_explicit(&state->rings[ring_idx].dropped,
memory_order_relaxed);
}
/* ── Mapping info ────────────────────────────────────────────────────── */
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (!state)
return;
state->mapping.mdbx_page_size = mdbx_page_size;
state->mapping.sys_page_size = (uint32_t)sysconf(_SC_PAGESIZE);
state->mapping.len = len;
/* base written last so consumers see consistent len+page_size first */
__atomic_store_n((void *volatile *)&state->mapping.base, base, __ATOMIC_RELEASE);
}
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (!state)
return 0;
void *b = __atomic_load_n((void *volatile *)&state->mapping.base, __ATOMIC_ACQUIRE);
if (!b)
return 0;
*out_base = b;
*out_len = state->mapping.len;
*out_mdbx_ps = state->mapping.mdbx_page_size;
*out_sys_ps = state->mapping.sys_page_size;
return 1;
}
/* ── Block marker emit (non-inline wrapper for Rust FFI) ──────────────── */
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
uint16_t tx_count, uint8_t duration_encoded,
uint8_t gas_encoded) {
mdbx_pageviz_emit(op, ((uint32_t)gas_encoded << 24) | ((uint32_t)duration_encoded << 16) | (uint32_t)tx_count, block_number);
}
#else /* !MDBX_PAGEVIZ */
typedef int mdbx_pageviz_empty_tu_;
#endif /* MDBX_PAGEVIZ */

View File

@@ -0,0 +1,195 @@
/* mdbx_pageviz.h — Standalone page-access visualization ring buffer for MDBX.
*
* Self-contained: no dependencies on MDBX internals.
* Gate everything behind MDBX_PAGEVIZ; when not defined, all macros expand to
* nothing (zero overhead).
*
* Compatible with C11 and C17. */
#ifndef MDBX_PAGEVIZ_H
#define MDBX_PAGEVIZ_H
/* ── Disabled stub macros ─────────────────────────────────────────────── */
#if !defined(MDBX_PAGEVIZ) || !(MDBX_PAGEVIZ)
#define MDBX_PAGEVIZ_READ(mc, pgno) ((void)0)
#define MDBX_PAGEVIZ_WRITE(mc, pgno) ((void)0)
#define MDBX_PAGEVIZ_FREE(mc, pgno) ((void)0)
#else /* MDBX_PAGEVIZ enabled ───────────────────────────────────────────── */
#include <stdatomic.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
/* ── Tunables ─────────────────────────────────────────────────────────── */
#define MDBX_PAGEVIZ_RING_CAPACITY 65536u /* must be power-of-2 */
#define MDBX_PAGEVIZ_MAX_RINGS 128u
#define MDBX_PAGEVIZ_PUBLISH_INTERVAL 32u
/* ── Event encoding ───────────────────────────────────────────────────── */
/* Layout of a 64-bit event word:
* bits 63..56 op (1=READ, 2=WRITE, 3=FREE)
* bits 55..32 dbi (lower 24 bits of a uint32_t)
* bits 31..0 pgno (uint32_t page number) */
#define MDBX_PAGEVIZ_OP_READ 1
#define MDBX_PAGEVIZ_OP_WRITE 2
#define MDBX_PAGEVIZ_OP_FREE 3
#define MDBX_PAGEVIZ_OP_BLOCK_START 4
#define MDBX_PAGEVIZ_OP_BLOCK_END 5
#define MDBX_PAGEVIZ_ENCODE(op, dbi, pgno) \
(((uint64_t)(op) << 56) | ((uint64_t)((dbi) & 0x00FFFFFFu) << 32) | \
(uint64_t)(uint32_t)(pgno))
#define MDBX_PAGEVIZ_DECODE_OP(ev) ((uint8_t)((ev) >> 56))
#define MDBX_PAGEVIZ_DECODE_DBI(ev) ((uint32_t)(((ev) >> 32) & 0x00FFFFFFu))
#define MDBX_PAGEVIZ_DECODE_PGNO(ev) ((uint32_t)(ev))
/* ── Ring-buffer mask (power-of-2 capacity) ───────────────────────────── */
#define MDBX_PAGEVIZ_RING_MASK (MDBX_PAGEVIZ_RING_CAPACITY - 1u)
/* ── Platform TLS ─────────────────────────────────────────────────────── */
#if defined(_MSC_VER)
# define MDBX_PAGEVIZ_TLS __declspec(thread)
#elif defined(__GNUC__) || defined(__clang__)
# define MDBX_PAGEVIZ_TLS __thread
#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L
# define MDBX_PAGEVIZ_TLS _Thread_local
#else
# error "No thread-local storage support detected"
#endif
/* ── Structures ───────────────────────────────────────────────────────── */
typedef struct mdbx_pageviz_ring {
uint64_t events[MDBX_PAGEVIZ_RING_CAPACITY];
_Atomic uint32_t published_head; /* consumer reads (acquire) */
uint32_t local_head; /* producer-only, no atomic needed */
_Atomic uint32_t consumer_tail; /* consumer updates after draining */
_Atomic uint64_t dropped; /* unused, kept for ABI compat */
uint32_t _pad[10]; /* pad to separate cache lines */
} mdbx_pageviz_ring_t;
typedef struct mdbx_pageviz_mapping {
volatile void *base;
volatile size_t len;
volatile uint32_t mdbx_page_size;
volatile uint32_t sys_page_size;
} mdbx_pageviz_mapping_t;
typedef struct mdbx_pageviz_state {
mdbx_pageviz_ring_t rings[MDBX_PAGEVIZ_MAX_RINGS];
_Atomic uint32_t ring_count; /* how many rings registered */
_Atomic uint32_t enabled; /* runtime enable/disable */
mdbx_pageviz_mapping_t mapping;
} mdbx_pageviz_state_t;
/* ── Global state (defined in .c) ─────────────────────────────────────── */
extern mdbx_pageviz_state_t *mdbx_pageviz_global;
/* ── Thread-local ring index (defined in .c) ──────────────────────────── */
#define MDBX_PAGEVIZ_TLS_UNREGISTERED UINT32_MAX
extern MDBX_PAGEVIZ_TLS uint32_t mdbx_pageviz_tls_ring;
/* ── Inline helpers (private) ─────────────────────────────────────────── */
static inline uint32_t mdbx_pageviz_register_ring_(mdbx_pageviz_state_t *s) {
uint32_t idx =
atomic_fetch_add_explicit(&s->ring_count, 1, memory_order_relaxed);
if (idx >= MDBX_PAGEVIZ_MAX_RINGS) {
atomic_fetch_sub_explicit(&s->ring_count, 1, memory_order_relaxed);
return MDBX_PAGEVIZ_TLS_UNREGISTERED;
}
mdbx_pageviz_ring_t *r = &s->rings[idx];
atomic_store_explicit(&r->published_head, 0, memory_order_relaxed);
r->local_head = 0;
atomic_store_explicit(&r->consumer_tail, 0, memory_order_relaxed);
atomic_store_explicit(&r->dropped, 0, memory_order_relaxed);
return idx;
}
/* ── Hot-path emit (inlined) ──────────────────────────────────────────── */
static inline void mdbx_pageviz_emit(uint8_t op, uint32_t dbi,
uint32_t pgno) {
mdbx_pageviz_state_t *state = mdbx_pageviz_global;
if (__builtin_expect(
state == NULL ||
!atomic_load_explicit(&state->enabled, memory_order_relaxed),
1))
return;
uint32_t idx = mdbx_pageviz_tls_ring;
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0)) {
idx = mdbx_pageviz_register_ring_(state);
if (__builtin_expect(idx == MDBX_PAGEVIZ_TLS_UNREGISTERED, 0))
return;
mdbx_pageviz_tls_ring = idx;
}
mdbx_pageviz_ring_t *r = &state->rings[idx];
/* Block (spin) until the consumer drains enough space. */
while (__builtin_expect(
(uint32_t)(r->local_head -
atomic_load_explicit(&r->consumer_tail, memory_order_acquire))
>= MDBX_PAGEVIZ_RING_CAPACITY, 0)) {
/* Flush our writes so the consumer can make progress. */
atomic_store_explicit(&r->published_head, r->local_head,
memory_order_release);
#if defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)
__builtin_ia32_pause();
#elif defined(__aarch64__) || defined(_M_ARM64)
__asm__ volatile("yield");
#endif
}
r->events[r->local_head & MDBX_PAGEVIZ_RING_MASK] =
MDBX_PAGEVIZ_ENCODE(op, dbi, pgno);
r->local_head++;
if ((r->local_head & (MDBX_PAGEVIZ_PUBLISH_INTERVAL - 1u)) == 0)
atomic_store_explicit(&r->published_head, r->local_head,
memory_order_release);
}
/* ── Macro hooks (inserted into mdbx.c) ──────────────────────────────── */
#define MDBX_PAGEVIZ_READ(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_READ, (uint32_t)cursor_dbi(mc), (pgno))
#define MDBX_PAGEVIZ_WRITE(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_WRITE, (uint32_t)cursor_dbi(mc), (pgno))
#define MDBX_PAGEVIZ_FREE(mc, pgno) \
mdbx_pageviz_emit(MDBX_PAGEVIZ_OP_FREE, (uint32_t)cursor_dbi(mc), (pgno))
/* ── Public API (defined in .c) ───────────────────────────────────────── */
mdbx_pageviz_state_t *mdbx_pageviz_create(void);
void mdbx_pageviz_destroy(mdbx_pageviz_state_t *state);
void mdbx_pageviz_enable(mdbx_pageviz_state_t *state);
void mdbx_pageviz_disable(mdbx_pageviz_state_t *state);
uint32_t mdbx_pageviz_drain(mdbx_pageviz_state_t *state, uint32_t ring_idx,
uint64_t *out_buf, uint32_t max_count);
uint32_t mdbx_pageviz_ring_count(mdbx_pageviz_state_t *state);
uint64_t mdbx_pageviz_dropped(mdbx_pageviz_state_t *state, uint32_t ring_idx);
void mdbx_pageviz_set_mapping(void *base, size_t len, uint32_t mdbx_page_size);
int mdbx_pageviz_get_mapping(void **out_base, size_t *out_len,
uint32_t *out_mdbx_ps, uint32_t *out_sys_ps);
void mdbx_pageviz_emit_block_marker(uint8_t op, uint32_t block_number,
uint16_t tx_count, uint8_t duration_encoded,
uint8_t gas_encoded);
#endif /* MDBX_PAGEVIZ */
#endif /* MDBX_PAGEVIZ_H */

View File

@@ -36,6 +36,9 @@ mod flags;
mod transaction;
mod txn_manager;
#[cfg(feature = "pageviz")]
pub mod pageviz;
#[cfg(test)]
mod test_utils {
use super::*;

View File

@@ -0,0 +1,226 @@
use std::{
collections::HashMap,
sync::{
atomic::{AtomicBool, Ordering},
mpsc, Arc,
},
time::{Duration, Instant},
};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[repr(u8)]
pub enum PageOp {
Read = 1,
Write = 2,
Free = 3,
BlockStart = 4,
BlockEnd = 5,
}
impl PageOp {
fn from_u8(v: u8) -> Option<Self> {
match v {
1 => Some(Self::Read),
2 => Some(Self::Write),
3 => Some(Self::Free),
4 => Some(Self::BlockStart),
5 => Some(Self::BlockEnd),
_ => None,
}
}
pub fn is_block_marker(self) -> bool {
matches!(self, Self::BlockStart | Self::BlockEnd)
}
}
#[derive(Debug, Clone, Copy)]
pub struct PageEvent {
pub pgno: u32,
pub dbi: u32,
pub op: PageOp,
}
impl PageEvent {
fn decode(raw: u64) -> Option<Self> {
let op_byte = (raw >> 56) as u8;
let op = PageOp::from_u8(op_byte)?;
let dbi = ((raw >> 32) & 0x00FF_FFFF) as u32;
let pgno = raw as u32;
Some(Self { pgno, dbi, op })
}
}
unsafe extern "C" {
fn mdbx_pageviz_create() -> *mut std::ffi::c_void;
fn mdbx_pageviz_destroy(state: *mut std::ffi::c_void);
fn mdbx_pageviz_enable(state: *mut std::ffi::c_void);
fn mdbx_pageviz_disable(state: *mut std::ffi::c_void);
fn mdbx_pageviz_drain(
state: *mut std::ffi::c_void,
ring_idx: u32,
out_buf: *mut u64,
max_count: u32,
) -> u32;
fn mdbx_pageviz_ring_count(state: *mut std::ffi::c_void) -> u32;
fn mdbx_pageviz_dropped(state: *mut std::ffi::c_void, ring_idx: u32) -> u64;
}
#[derive(Debug)]
pub struct PageVizStats {
pub ring_count: u32,
pub total_dropped: u64,
}
#[derive(Debug, Clone, Copy)]
struct StatePtr(*mut std::ffi::c_void);
// SAFETY: The C state is thread-safe (uses atomics internally).
unsafe impl Send for StatePtr {}
unsafe impl Sync for StatePtr {}
#[derive(Debug)]
pub struct PageVizDrainer {
state: StatePtr,
running: Arc<AtomicBool>,
handle: Option<std::thread::JoinHandle<()>>,
}
const DRAIN_BUF_LEN: usize = 4096;
const OVERLOAD_THRESHOLD: usize = 200_000;
const READ_SAMPLE_RATE: usize = 8;
impl PageVizDrainer {
pub fn new() -> Self {
let state = StatePtr(unsafe { mdbx_pageviz_create() });
Self { state, running: Arc::new(AtomicBool::new(false)), handle: None }
}
pub fn enable(&self) {
unsafe { mdbx_pageviz_enable(self.state.0) }
}
pub fn disable(&self) {
unsafe { mdbx_pageviz_disable(self.state.0) }
}
pub fn start(&mut self, tick_hz: u32) -> mpsc::Receiver<Vec<PageEvent>> {
let (tx, rx) = mpsc::channel();
let running = self.running.clone();
running.store(true, Ordering::SeqCst);
let state_addr = self.state.0 as usize;
let tick_interval = Duration::from_micros(1_000_000 / u64::from(tick_hz));
let handle = std::thread::Builder::new()
.name("mdbx-pageviz".into())
.spawn(move || {
let state = state_addr as *mut std::ffi::c_void;
let mut buf = [0u64; DRAIN_BUF_LEN];
let mut coalesced: HashMap<u32, PageEvent> = HashMap::new();
let mut markers: Vec<PageEvent> = Vec::new();
let mut read_counter: usize = 0;
while running.load(Ordering::Relaxed) {
let tick_start = Instant::now();
let ring_count = unsafe { mdbx_pageviz_ring_count(state) };
for ring_idx in 0..ring_count {
let dropped = unsafe { mdbx_pageviz_dropped(state, ring_idx) };
if dropped > 0 {
tracing::warn!(
target: "libmdbx::pageviz",
ring_idx,
dropped,
"pageviz ring dropped events"
);
}
loop {
let count = unsafe {
mdbx_pageviz_drain(
state,
ring_idx,
buf.as_mut_ptr(),
DRAIN_BUF_LEN as u32,
)
};
if count == 0 {
break;
}
for &raw in &buf[..count as usize] {
if let Some(evt) = PageEvent::decode(raw) {
if evt.op.is_block_marker() {
markers.push(evt);
} else {
coalesced.insert(evt.pgno, evt);
}
}
}
}
}
if !coalesced.is_empty() || !markers.is_empty() {
let mut events: Vec<PageEvent> = if coalesced.len() > OVERLOAD_THRESHOLD {
coalesced
.drain()
.filter(|(_, evt)| {
if evt.op == PageOp::Read {
read_counter += 1;
read_counter % READ_SAMPLE_RATE == 0
} else {
true
}
})
.map(|(_, evt)| evt)
.collect()
} else {
coalesced.drain().map(|(_, evt)| evt).collect()
};
events.extend(markers.drain(..));
if tx.send(events).is_err() {
break;
}
}
let elapsed = tick_start.elapsed();
if let Some(remaining) = tick_interval.checked_sub(elapsed) {
std::thread::sleep(remaining);
}
}
})
.expect("failed to spawn mdbx-pageviz thread");
self.handle = Some(handle);
rx
}
pub fn stop(&mut self) {
self.running.store(false, Ordering::SeqCst);
if let Some(handle) = self.handle.take() {
let _ = handle.join();
}
}
pub fn stats(&self) -> PageVizStats {
let ring_count = unsafe { mdbx_pageviz_ring_count(self.state.0) };
let mut total_dropped = 0u64;
for ring_idx in 0..ring_count {
total_dropped += unsafe { mdbx_pageviz_dropped(self.state.0, ring_idx) };
}
PageVizStats { ring_count, total_dropped }
}
}
impl Drop for PageVizDrainer {
fn drop(&mut self) {
self.stop();
unsafe {
mdbx_pageviz_disable(self.state.0);
mdbx_pageviz_destroy(self.state.0);
}
}
}

1381
crates/storage/mdbx-viz/Cargo.lock generated Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,23 @@
[package]
name = "reth-mdbx-viz"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license = "Apache-2.0"
homepage.workspace = true
repository.workspace = true
description = "Real-time MDBX page access visualization server"
[dependencies]
reth-libmdbx = { workspace = true, features = ["pageviz"] }
reth-mdbx-sys = { workspace = true, features = ["pageviz"] }
axum = { version = "0.7", features = ["ws"] }
tokio = { workspace = true, features = ["full"] }
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
clap = { version = "4", features = ["derive"] }
tracing.workspace = true
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
rayon.workspace = true
eyre.workspace = true
libc = "0.2"

View File

@@ -0,0 +1,952 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>MDBX Live Viz</title>
<style>
*{margin:0;padding:0;box-sizing:border-box}
html,body{background:#0d1117;color:#c9d1d9;font-family:'SF Mono','Fira Code','Cascadia Code',monospace;height:100%;overflow:hidden}
body{display:flex;flex-direction:row;height:100%}
#main-panel{flex:1;display:flex;flex-direction:column;padding:16px;overflow:hidden;min-width:0}
#side-panel{flex:0 0 540px;border-left:1px solid #30363d;overflow-y:auto;padding:16px;background:#161b22}
h1{color:#f0f6fc;font-size:18px;margin-bottom:2px}
.subtitle{color:#8b949e;font-size:12px;margin-bottom:10px}
.sub-row{display:flex;align-items:baseline;gap:12px;margin-bottom:6px;flex-wrap:wrap}
.sub-row .subtitle{margin-bottom:0;flex-shrink:0}
.sub-row .viz-controls{margin-bottom:0;margin-left:auto}
.controls{margin-bottom:6px;display:flex;gap:8px;align-items:center;flex-wrap:wrap}
.controls button{background:#21262d;border:1px solid #30363d;color:#c9d1d9;padding:3px 10px;border-radius:4px;cursor:pointer;font-family:inherit;font-size:12px}
.controls button:hover{background:#30363d}
.zoom-info{font-size:11px;color:#8b949e}
.status-bar{display:flex;gap:12px;align-items:center;margin-bottom:6px;font-size:11px}
.dot{display:inline-block;width:8px;height:8px;border-radius:50%;margin-right:4px;vertical-align:middle}
.dot.connected{background:#4ade80}.dot.disconnected{background:#f87171}.dot.connecting{background:#facc15}
.stat-label{color:#8b949e}.stat-value{color:#c9d1d9;font-weight:600}
#canvas-wrap{border:1px solid #30363d;border-radius:6px;overflow:hidden;background:#000;position:relative;cursor:crosshair;flex:1;min-height:0}
canvas{display:block;image-rendering:pixelated;width:100%;height:100%}
#tooltip{position:fixed;background:#1c2128;border:1px solid #30363d;border-radius:6px;padding:8px 12px;font-size:12px;pointer-events:none;display:none;z-index:100;box-shadow:0 4px 12px rgba(0,0,0,.4);max-width:360px}
#tooltip .tt-row{display:flex;align-items:center;gap:6px}
#tooltip .tt-swatch{width:10px;height:10px;border-radius:2px;flex-shrink:0;border:1px solid rgba(255,255,255,0.2)}
#tooltip .tt-name{color:#f0f6fc;font-weight:600}
#tooltip .tt-detail{color:#8b949e;margin-top:2px}
#side-panel h2{color:#f0f6fc;font-size:14px;margin-bottom:8px}
.viz-controls{margin-bottom:12px;font-size:11px}
.viz-controls label{display:inline-flex;align-items:center;gap:4px;margin-right:10px;cursor:pointer;color:#8b949e}
.viz-controls label:hover{color:#c9d1d9}
.viz-controls input[type=checkbox]{accent-color:#58a6ff}
.viz-controls input[type=range]{width:100px;accent-color:#58a6ff}
.fade-val{color:#c9d1d9;min-width:35px;display:inline-block}
table{width:100%;border-collapse:collapse;font-size:11px;background:#0d1117;border:1px solid #30363d;border-radius:6px;overflow:hidden;table-layout:fixed}
td,th{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
th{text-align:left;padding:6px 8px;background:#1c2128;color:#8b949e;font-weight:500;border-bottom:1px solid #30363d;font-size:10px;text-transform:uppercase;position:sticky;top:0}
td{padding:5px 8px;border-bottom:1px solid #21262d}
tr.tbl-row{cursor:pointer;transition:background 0.1s}
tr.tbl-row:hover td{background:#1c2128}
tr.tbl-row.active td{background:#1c2128;outline:1px solid #58a6ff}
.pct-bar{display:inline-block;height:6px;border-radius:3px;margin-right:6px;vertical-align:middle}
.swatch-cell{width:12px;height:12px;border-radius:2px;flex-shrink:0}
#loading{display:flex;flex-direction:column;align-items:center;justify-content:center;height:100%;color:#8b949e;font-size:14px;gap:12px}
#load-bar-wrap{width:280px;height:6px;background:#21262d;border-radius:3px;overflow:hidden;display:none}
#load-bar{height:100%;width:0;background:#58a6ff;border-radius:3px;transition:width 0.15s}
#flame-wrap{border:1px solid #30363d;border-radius:0 0 6px 6px;margin-top:-1px;background:#0d1117}
#flame-wrap canvas{display:block}
.spark-row{display:flex;align-items:center;gap:6px;margin-bottom:3px;font-size:10px}
.spark-label{width:100px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
.spark-cv{border:1px solid #21262d;border-radius:2px;background:#0d1117}
.spark-val{color:#c9d1d9;min-width:30px;text-align:right}
.tree-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:4px;font-size:10px;cursor:pointer}
.tree-card:hover{border-color:#58a6ff}
.tree-card .tree-name{color:#f0f6fc;font-weight:600;font-size:11px}
.tree-card .tree-stat{color:#8b949e;margin-top:2px}
.tree-card .tree-bar{display:flex;gap:1px;margin-top:3px;height:6px;border-radius:2px;overflow:hidden}
.tree-card .tree-seg{height:100%}
.cache-stat{display:flex;justify-content:space-between;align-items:center;font-size:10px;padding:2px 0}
.cache-bar{height:4px;border-radius:2px;background:#21262d;flex:1;margin:0 6px;position:relative;overflow:hidden}
.cache-fill{height:100%;border-radius:2px;background:#4ade80;position:absolute;left:0;top:0}
.cache-pct{color:#4ade80;font-weight:600;min-width:36px;text-align:right}
.cache-label{color:#8b949e;min-width:80px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
.fault-cnt{color:#d946ef;font-size:10px;margin-left:4px}
.block-card{background:#1c2128;border:1px solid #30363d;border-radius:4px;padding:6px 8px;margin-bottom:3px;font-size:10px;cursor:pointer;transition:border-color 0.15s}
.block-card:hover{border-color:#58a6ff}
.block-card.replaying{border-color:#f59e0b;background:#1e1b13}
.block-card.recording{border-color:#4ade80;animation:pulse-border 1s infinite}
.block-card .block-num{color:#f0f6fc;font-weight:600;font-size:11px}
.block-card .block-stat{color:#8b949e;margin-top:1px}
@keyframes pulse-border{0%,100%{border-color:#4ade80}50%{border-color:#166534}}
#block-live{cursor:pointer;font-size:10px;margin-left:6px;user-select:none}
.utbl-row{display:flex;align-items:center;gap:6px;padding:3px 4px;font-size:10px;cursor:pointer}
.utbl-row:hover{background:#1c2128}
.utbl-row.active{background:#1c2128;outline:1px solid #58a6ff}
.utbl-sw{width:8px;height:8px;border-radius:2px;flex-shrink:0}
.utbl-name{width:110px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:#8b949e}
.utbl-spark{flex-shrink:0}
.utbl-cache{flex:1;min-width:0}
.utbl-cache-bar-wrap{width:100%;height:6px;background:#21262d;border-radius:3px;overflow:hidden}
.utbl-cache-fill{height:100%;border-radius:3px;transition:width 0.3s}
.utbl-cache-pct{text-align:center;font-size:9px;color:#4ade80;font-weight:600;margin-top:1px;line-height:1}
.utbl-diff{width:52px;text-align:right;font-size:9px}
.utbl-faults{width:36px;text-align:right;color:#d946ef;font-size:9px}
#block-panel::-webkit-scrollbar,#side-panel::-webkit-scrollbar{width:6px}
#block-panel::-webkit-scrollbar-track,#side-panel::-webkit-scrollbar-track{background:#161b22}
#block-panel::-webkit-scrollbar-thumb,#side-panel::-webkit-scrollbar-thumb{background:#30363d;border-radius:3px}
#block-panel::-webkit-scrollbar-thumb:hover,#side-panel::-webkit-scrollbar-thumb:hover{background:#484f58}
#block-panel,#side-panel{scrollbar-width:thin;scrollbar-color:#30363d #161b22}
</style>
</head>
<body>
<div id="main-panel">
<h1>MDBX Live Viz</h1>
<div class="sub-row">
<div class="subtitle" id="subtitle">Loading...</div>
<div class="viz-controls">
<label><input type="checkbox" id="chk-read" checked> <span style="color:#3c82f6">&#9632;</span> Reads</label>
<label><input type="checkbox" id="chk-write" checked> <span style="color:#ef4444">&#9632;</span> Writes</label>
<label><input type="checkbox" id="chk-free" checked> <span style="color:#eab308">&#9632;</span> Frees</label>
<label><input type="checkbox" id="chk-fixw"> Fixed Width</label>
<label><input type="checkbox" id="chk-trails" checked> Trails</label>
<label><input type="checkbox" id="chk-heatmap"> Heatmap</label>
<label><input type="checkbox" id="chk-flame"> Timeline</label>
<label><input type="checkbox" id="chk-cache"> Page Cache</label>
<label>Fade: <input type="range" id="fade-slider" min="200" max="8000" value="2000" step="100"> <span class="fade-val" id="fade-val">2.0s</span></label>
<label>Trail W: <input type="range" id="trail-width" min="0.5" max="6" value="1.0" step="0.5"> <span class="fade-val" id="trail-w-val">1.0</span></label>
</div>
</div>
<div class="status-bar">
<span><span class="dot connecting" id="ws-dot"></span><span id="ws-status">Connecting</span></span>
<span class="stat-label">Events/s: <span class="stat-value" id="evt-rate">0</span></span>
<span class="stat-label">Total: <span class="stat-value" id="evt-total">0</span></span>
<span class="stat-label">Active: <span class="stat-value" id="active-count">0</span></span>
<span class="stat-label">FPS: <span class="stat-value" id="fps-counter">-</span></span>
</div>
<div class="status-bar">
<span class="stat-label" style="color:#3c82f6">R: <span class="stat-value" id="cnt-read">0</span></span>
<span class="stat-label" style="color:#ef4444">W: <span class="stat-value" id="cnt-write">0</span></span>
<span class="stat-label" style="color:#eab308">F: <span class="stat-value" id="cnt-free">0</span></span>
<span class="stat-label">Latest: <span class="stat-value" id="evt-latest">-</span></span>
<span id="lag-warn" style="display:none;color:#f87171;font-weight:600">&#9888; Lagged <span id="lag-count">0</span> msgs</span>
</div>
<div class="controls">
<button id="btn-zin">Zoom +</button>
<button id="btn-zout">Zoom &minus;</button>
<button id="btn-reset">Reset</button>
<button id="btn-clear">Clear Highlights</button>
<span class="zoom-info" id="zoom-info"></span>
</div>
<div id="canvas-wrap">
<div id="loading"><span id="load-text">Loading page map&hellip;</span><div id="load-bar-wrap"><div id="load-bar"></div></div></div>
<canvas id="map" style="display:none"></canvas>
</div>
<div id="flame-wrap" style="height:0;overflow:hidden;transition:height 0.2s">
<canvas id="flame"></canvas>
</div>
</div>
<div id="side-panel">
<h2>Blocks <span id="block-live" style="color:#4ade80" onclick="window._goLive&&window._goLive()">&#x25CF; LIVE</span></h2>
<div id="block-panel" style="max-height:172px;overflow-y:auto"></div>
<h2 style="margin-top:12px">Tables <span id="cache-total" style="color:#8b949e;font-weight:400;font-size:11px"></span> <span id="cache-ago" style="color:#8b949e;font-weight:400;font-size:10px"></span></h2>
<div id="unified-tables"></div>
<h2 style="margin-top:12px">Table Details</h2>
<table><thead><tr><th></th><th>Table</th><th>Pages</th><th>Size</th><th>%</th></tr></thead><tbody id="tbl-body"></tbody></table>
<h2 style="margin-top:12px">B-Tree Structure</h2>
<div id="tree-info"></div>
</div>
<div id="tooltip">
<div class="tt-row"><div class="tt-swatch" id="tt-swatch"></div><div class="tt-name" id="tt-name"></div></div>
<div class="tt-detail" id="tt-detail"></div>
</div>
<script>
(function(){
"use strict";
function hsl2r(h,s,l){h/=360;s/=100;l/=100;let r,g,b;if(!s){r=g=b=l}else{const q=l<.5?l*(1+s):l+s-l*s,p=2*l-q;const f=(p,q,t)=>{if(t<0)t+=1;if(t>1)t-=1;if(t<1/6)return p+(q-p)*6*t;if(t<.5)return q;if(t<2/3)return p+(q-p)*(2/3-t)*6;return p};r=f(p,q,h+1/3);g=f(p,q,h);b=f(p,q,h-1/3)}return[Math.round(r*255),Math.round(g*255),Math.round(b*255)]}
const DBI_HUES=new Float32Array(256);
for(let i=2;i<254;i++)DBI_HUES[i]=((i-2)*137.508)%360;
const PAL=new Array(256),PAL_DESAT=new Array(256),PAL_DARK=new Array(256),PAL_DD=new Array(256);
PAL[0]=hsl2r(220,15,82);PAL_DESAT[0]=hsl2r(220,5,72);PAL_DARK[0]=hsl2r(220,15,28);PAL_DD[0]=hsl2r(220,5,22);
PAL[1]=hsl2r(220,30,24);PAL_DESAT[1]=hsl2r(220,8,22);PAL_DARK[1]=hsl2r(220,30,12);PAL_DD[1]=hsl2r(220,8,12);
for(let i=2;i<254;i++){const h=DBI_HUES[i];PAL[i]=hsl2r(h,70,55);PAL_DESAT[i]=hsl2r(h,15,50);PAL_DARK[i]=hsl2r(h,70,22);PAL_DD[i]=hsl2r(h,15,20)}
PAL[0xFE]=[20,20,23];PAL_DESAT[0xFE]=[20,20,23];PAL_DARK[0xFE]=[12,12,14];PAL_DD[0xFE]=[12,12,14];
PAL[0xFF]=[28,28,32];PAL_DESAT[0xFF]=[28,28,32];PAL_DARK[0xFF]=[16,16,18];PAL_DD[0xFF]=[16,16,18];
const FLASH_TINT={1:[140,180,255],2:[255,140,140],3:[255,220,100]};
let NP=0,PS=4096,GC=0,GR=0,DBIS=[],DBI_IDX={},oMap=null,lOp=null,lTs=null,FADE=2000;
let cntR=0,cntW=0,cntF=0;
let sR=true,sW=true,sF=true,fW=false,showTrails=true,trailW=1.0;
const TRAIL_CAP=4096;let trailBuf=new Array(TRAIL_CAP);let trailStart=0,trailN=0;
const TRC={1:"60,130,246",2:"255,80,80",3:"234,179,8"};
let activePgs=0;
const activeSet=new Set();
let oMapDirty=false;
let unrefCount=0;
let outR=0,outG=0,outB=0;
let cacheTotalCached=0,cacheCachedPer=null,cachePrevPer=null,cachePrevTotal=0,cacheStale=true;
let hmMode=false,hmCounts=null;
let cacheMode=false,resMap=null,faultCounts={},faultWindow=[],faultCurrent={};
let cacheLastUpdate=0;
let zbCv=null,zbCx=null,zbDirty=true,zbW=0,zbH=0,zbPX=0,zbPY=0,zbSc=0,zbFlt=false;
const BASE_LUT=new Uint32Array(256*32*2);
function buildLUT(){
for(let dbi=0;dbi<256;dbi++){for(let hq=0;hq<32;hq++){const ht=hq/31;for(let c=0;c<2;c++){
let cf,cd;
if(c){cf=PAL[dbi];cd=PAL_DESAT[dbi]}else{cf=PAL_DARK[dbi];cd=PAL_DD[dbi]}
if(!cf)cf=PAL[255];if(!cd)cd=PAL_DESAT[255];
const r=Math.round(cd[0]+(cf[0]-cd[0])*ht);
const g=Math.round(cd[1]+(cf[1]-cd[1])*ht);
const b=Math.round(cd[2]+(cf[2]-cd[2])*ht);
BASE_LUT[(dbi*32+hq)*2+c]=(255<<24)|(b<<16)|(g<<8)|r;
}}}
}
buildLUT();
function heatQ(pg){if(!hmMode||!hmCounts)return 31;const cnt=hmCounts[pg];return cnt>0?Math.min(31,Math.floor(Math.log2(cnt+1)*31/16)):0}
function rollingFaults(dbi){let s=0;for(const w of faultWindow)s+=(w[dbi]||0);return s}
function markZbDirty(){zbDirty=true}
let blockRecording=null,blockHistory=[],replayMode=false,replayBlockRef=null,replayTimer=null,liveSnapshot=null;
const BLOCK_HISTORY_CAP=15;
let blockPanelDirty=false;
let showFlame=false;const FLAME_SECS=120;let flameData={},flameHead=0;
const SPARK_SECS=60;let sparkData={},sparkAccum={};
const cv=document.getElementById("map"),cx=cv.getContext("2d"),wr=document.getElementById("canvas-wrap");
const ldEl=document.getElementById("loading"),zmEl=document.getElementById("zoom-info");
const tt=document.getElementById("tooltip"),ttSw=document.getElementById("tt-swatch");
const ttNm=document.getElementById("tt-name"),ttDt=document.getElementById("tt-detail");
const tb=document.getElementById("tbl-body");
let W=0,H=0,iD=null,oC=null,oCx=null,pX=0,pY=0,sc=1,mS=.01;const xS=128;
const sel=new Set();let tEv=0,rEv=0,lRT=performance.now();
function cl(v,a,b){return Math.max(a,Math.min(b,v))}
function growTo(newNP){
if(newNP<=NP)return;
const oldNP=NP;
const nOm=new Uint8Array(newNP);nOm.set(oMap);nOm.fill(0xFF,oldNP);oMap=nOm;
const nOp=new Uint8Array(newNP);nOp.set(lOp);lOp=nOp;
const nTs=new Float64Array(newNP);nTs.set(lTs);lTs=nTs;
if(hmCounts){const nHm=new Uint32Array(newNP);nHm.set(hmCounts);hmCounts=nHm}
if(resMap){const nRm=new Uint8Array(newNP);nRm.set(resMap);resMap=nRm}
unrefCount+=newNP-oldNP;
NP=newNP;GR=Math.ceil(NP/GC);
oMapDirty=true;
iD=null;oC=null;markZbDirty();
}
function oX(){return 0}
function oY(){return Math.max(0,(H-GR*sc)/2)}
function cP(){const vw=W/sc,vh=H/sc;pX=cl(pX,0,Math.max(0,GC-vw));pY=cl(pY,0,Math.max(0,GR-vh))}
function s2w(sx,sy){return{wx:pX+(sx-oX())/sc,wy:pY+(sy-oY())/sc}}
function w2p(wx,wy){const px=Math.floor(wx),py=Math.floor(wy);if(px<0||py<0||px>=GC||py>=GR)return-1;const i=py*GC+px;return i<NP?i:-1}
function trailPush(from,to,op,ts){
const idx=(trailStart+trailN)%TRAIL_CAP;
if(!trailBuf[idx])trailBuf[idx]={from,to,op,ts};
else{trailBuf[idx].from=from;trailBuf[idx].to=to;trailBuf[idx].op=op;trailBuf[idx].ts=ts}
if(trailN<TRAIL_CAP)trailN++;
else trailStart=(trailStart+1)%TRAIL_CAP;
}
function trailExpire(now){
while(trailN>0){
const t=trailBuf[trailStart];
if(now-t.ts>=FADE){trailStart=(trailStart+1)%TRAIL_CAP;trailN--}
else break;
}
}
function pCBase(pg,flt){
if(pg<0||pg>=NP)return false;
const ow=oMap[pg];
if(flt&&!sel.has(ow)){outR=25;outG=25;outB=28;return true}
const hq=heatQ(pg);
const cached=(cacheMode&&resMap&&pg<resMap.length)?resMap[pg]?1:0:1;
const rgba=BASE_LUT[(ow*32+hq)*2+cached];
outR=rgba&0xFF;outG=(rgba>>8)&0xFF;outB=(rgba>>16)&0xFF;
return true;
}
function pC(pg,flt,now){
if(!pCBase(pg,flt))return false;
const op=lOp[pg];
if(op&&((op===1&&sR)||(op===2&&sW)||(op===3&&sF))){
const age=now-lTs[pg];
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);activeSet.delete(pg)}
else{const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
outR=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
outG=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
outB=Math.min(255,Math.round(outB+(fl[2]-outB)*fi))}
}
return true;
}
function pg2s(pg){const col=pg%GC,row=Math.floor(pg/GC);return{sx:oX()+(col-pX+.5)*sc,sy:oY()+(row-pY+.5)*sc}}
function drawTrails(now){
if(!showTrails||trailN===0)return;
cx.lineCap="round";cx.lineWidth=trailW;
const paths={1:[],2:[],3:[]};
for(let j=0;j<trailN;j++){
const idx=(trailStart+j)%TRAIL_CAP;
const t=trailBuf[idx];
const age=now-t.ts;if(age>=FADE)continue;
if((t.op===1&&!sR)||(t.op===2&&!sW)||(t.op===3&&!sF))continue;
const a=1-Math.pow(age/FADE,2);
const f=pg2s(t.from),e=pg2s(t.to);
(paths[t.op]||(paths[t.op]=[])).push({fx:f.sx,fy:f.sy,ex:e.sx,ey:e.sy,a});
}
for(const op of [1,2,3]){
const segs=paths[op];if(!segs||!segs.length)continue;
const groups={};
for(const s of segs){
const ak=Math.round(s.a*10);
if(!groups[ak])groups[ak]=[];
groups[ak].push(s);
}
const c=TRC[op]||TRC[1];
for(const [ak,gs] of Object.entries(groups)){
cx.strokeStyle=`rgba(${c},${(ak/10*0.6).toFixed(2)})`;
cx.beginPath();
for(const s of gs){cx.moveTo(s.fx,s.fy);cx.lineTo(s.ex,s.ey)}
cx.stroke();
}
}
}
function rebuildZoomBase(flt){
if(!zbCv){zbCv=document.createElement("canvas");zbCx=zbCv.getContext("2d")}
zbCv.width=W;zbCv.height=H;zbW=W;zbH=H;zbPX=pX;zbPY=pY;zbSc=sc;zbFlt=flt;
const zd=zbCx.createImageData(W,H),d=zd.data,ox=oX(),oy=oY(),ips=1/sc;
for(let sy=0;sy<H;sy++){
const wy0=Math.max(0,Math.floor(pY+(sy-oy)*ips)),wy1=Math.min(GR-1,Math.floor(pY+(sy-oy+1)*ips));
for(let sx=0;sx<W;sx++){
const wx0=Math.max(0,Math.floor(pX+(sx-ox)*ips)),wx1=Math.min(GC-1,Math.floor(pX+(sx-ox+1)*ips));
const o=(sy*W+sx)<<2;
if(wx1<0||wy1<0||wx0>=GC||wy0>=GR){d[o]=0;d[o+1]=0;d[o+2]=0;d[o+3]=255;continue}
const cpx=Math.floor((wx0+wx1)/2),cpy=Math.floor((wy0+wy1)/2);
const ci=cpy*GC+cpx;
if(ci>=0&&ci<NP&&pCBase(ci,flt)){d[o]=outR;d[o+1]=outG;d[o+2]=outB}
d[o+3]=255;
}
}
zbCx.putImageData(zd,0,0);
}
function drawActiveOverlay(now,flt){
if(activeSet.size===0)return;
const ox=oX(),oy=oY(),ips=1/sc;
const expired=[];
for(const pg of activeSet){
const op=lOp[pg];
if(!op){expired.push(pg);continue}
if((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF))continue;
const age=now-lTs[pg];
if(age>=FADE){lOp[pg]=0;activePgs=Math.max(0,activePgs-1);expired.push(pg);continue}
pCBase(pg,flt);
const a2=Math.max(0,age);const t=1-(a2/FADE)*(a2/FADE);const fl=FLASH_TINT[op];const fi=t*0.7;
const r=Math.min(255,Math.round(outR+(fl[0]-outR)*fi));
const g=Math.min(255,Math.round(outG+(fl[1]-outG)*fi));
const b=Math.min(255,Math.round(outB+(fl[2]-outB)*fi));
const col=pg%GC,row=Math.floor(pg/GC);
const sx=ox+(col-pX)*sc,sy=oy+(row-pY)*sc;
if(sx<-sc||sy<-sc||sx>W||sy>H)continue;
const psz=Math.max(1,Math.ceil(sc));
cx.fillStyle=`rgb(${r},${g},${b})`;
cx.fillRect(Math.floor(sx),Math.floor(sy),psz,psz);
}
for(const pg of expired)activeSet.delete(pg);
}
function render(){
if(!oMap||W<1||H<1)return;
const flt=sel.size>0,now=performance.now();
if(sc>=1){
const vw=W/sc,vh=H/sc,x0=Math.floor(pX),y0=Math.floor(pY);
const x1=Math.min(GC,Math.ceil(pX+vw)+1),y1=Math.min(GR,Math.ceil(pY+vh)+1);
const vW=Math.max(1,x1-x0),vH=Math.max(1,y1-y0);
if(!oC){oC=document.createElement("canvas");oCx=oC.getContext("2d")}
if(oC.width!==vW||oC.height!==vH){oC.width=vW;oC.height=vH}
const od=oCx.createImageData(vW,vH),d=od.data;
for(let y=0;y<vH;y++){const gy=y0+y;for(let x=0;x<vW;x++){
const i=gy*GC+(x0+x),o=(y*vW+x)<<2;
const ok=(i>=0&&i<NP)?pC(i,flt,now):false;
if(ok){d[o]=outR;d[o+1]=outG;d[o+2]=outB}else{d[o]=0;d[o+1]=0;d[o+2]=0}d[o+3]=255;
}}
oCx.putImageData(od,0,0);cx.imageSmoothingEnabled=false;cx.clearRect(0,0,W,H);
cx.drawImage(oC,oX()-(pX-x0)*sc,oY()-(pY-y0)*sc,vW*sc,vH*sc);
drawTrails(now);
zmEl.textContent=sc.toFixed(sc>=10?0:1)+" px/page";
}else{
if(zbDirty||!zbCv||zbW!==W||zbH!==H||zbPX!==pX||zbPY!==pY||zbSc!==sc||zbFlt!==flt){
rebuildZoomBase(flt);zbDirty=false;
}
cx.clearRect(0,0,W,H);
cx.drawImage(zbCv,0,0);
drawActiveOverlay(now,flt);
drawTrails(now);
const pp=1/sc;zmEl.textContent="1 px = "+pp.toFixed(pp>=10?0:1)+" pages";
}
}
let anim=false;
let fpsFrames=0,fpsLast=performance.now();
function rLoop(){
render();let act=0;const now=performance.now();
act+=activePgs;
if(showTrails){trailExpire(now);act+=trailN}
document.getElementById("active-count").textContent=act.toLocaleString();
fpsFrames++;
const fpsNow=performance.now();
if(fpsNow-fpsLast>=500){
document.getElementById("fps-counter").textContent=Math.round(fpsFrames/((fpsNow-fpsLast)/1000));
fpsFrames=0;fpsLast=fpsNow;
}
if(act>0&&!(replayMode&&!replayTimer))requestAnimationFrame(rLoop);else{anim=false;document.getElementById("fps-counter").textContent="-"}
}
function kick(){if(!anim){anim=true;requestAnimationFrame(rLoop)}}
let schedPending=false;
function sched(){if(!schedPending){schedPending=true;requestAnimationFrame(()=>{schedPending=false;render()})}}
function szCv(){const w=wr.clientWidth,h=wr.clientHeight;if(w<1||h<1)return;W=w;H=h;cv.width=W;cv.height=H;iD=null}
function fmtSz(mb){if(mb>=1024)return(mb/1024).toFixed(1)+" GB";if(mb>=1)return mb.toFixed(1)+" MB";return(mb*1024).toFixed(0)+" KB"}
function bldTbl(){
tb.innerHTML="";const mx=Math.max(...DBIS.map(d=>d.pct),.1);
DBIS.forEach(d=>{const tr=document.createElement("tr");tr.className="tbl-row";tr.dataset.dbiIndex=d.dbi_index;
const c=PAL[d.dbi_index]||PAL[255];
tr.innerHTML=`<td><div class="swatch-cell" style="background:rgb(${c[0]},${c[1]},${c[2]})"></div></td><td>${d.name}</td><td>${d.pages.toLocaleString()}</td><td>${fmtSz(d.size_mb)}</td><td><span class="pct-bar" style="width:${d.pct/mx*60}px;background:rgb(${c[0]},${c[1]},${c[2]})"></span>${d.pct}%</td>`;
tr.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
tb.appendChild(tr);
});
}
function uAR(){document.querySelectorAll(".tbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")});document.querySelectorAll(".utbl-row").forEach(r=>{sel.has(parseInt(r.dataset.dbiIndex,10))?r.classList.add("active"):r.classList.remove("active")})}
function reflow(){if(!fW)return;GC=Math.max(1,Math.floor(W/sc));GR=Math.ceil(NP/GC);pX=0;cP()}
function boot(){szCv();const aspect=W>0&&H>0?W/H:1;GC=Math.max(1,Math.ceil(Math.sqrt(NP*aspect)));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);sc=mS;pX=0;pY=0;render()}
function zA(ns,sx,sy){ns=cl(ns,mS,xS);if(Math.abs(ns-sc)<1e-6)return;const b=s2w(sx,sy);sc=ns;if(fW){reflow();pY=b.wy-(sy-oY())/sc;cP()}else{pX=b.wx-(sx-oX())/sc;pY=b.wy-(sy-oY())/sc;cP()}sched()}
document.getElementById("btn-zin").addEventListener("click",()=>zA(sc*1.3,W>>1,H>>1));
document.getElementById("btn-zout").addEventListener("click",()=>zA(sc/1.3,W>>1,H>>1));
document.getElementById("btn-reset").addEventListener("click",()=>{sc=mS;pX=0;pY=0;sel.clear();uAR();markZbDirty();render()});
document.getElementById("btn-clear").addEventListener("click",()=>{if(lOp)lOp.fill(0);if(hmCounts)hmCounts.fill(0);activePgs=0;activeSet.clear();markZbDirty();sched()});
wr.addEventListener("wheel",e=>{e.preventDefault();const r=cv.getBoundingClientRect();zA(sc*Math.exp(-(e.deltaMode===1?e.deltaY*16:e.deltaY)*.002),(e.clientX-r.left)/r.width*W,(e.clientY-r.top)/r.height*H)},{passive:false});
let drag=false,lCX=0,lCY=0;
cv.addEventListener("pointerdown",e=>{drag=true;lCX=e.clientX;lCY=e.clientY;cv.setPointerCapture(e.pointerId);cv.style.cursor="grabbing"});
cv.addEventListener("pointermove",e=>{
if(drag){if(!fW)pX-=(e.clientX-lCX)/sc;pY-=(e.clientY-lCY)/sc;lCX=e.clientX;lCY=e.clientY;cP();sched();tt.style.display="none";return}
const r=cv.getBoundingClientRect(),sx=(e.clientX-r.left)/r.width*W,sy=(e.clientY-r.top)/r.height*H;
const w=s2w(sx,sy),pg=w2p(w.wx,w.wy);if(pg<0){tt.style.display="none";return}
const ow=oMap[pg],inf=DBI_IDX[ow],c=PAL[ow]||PAL[255];
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
let ex="";const op=lOp[pg];if(op===1)ex=" [READ]";else if(op===2)ex=" [WRITE]";else if(op===3)ex=" [FREE]";
ttNm.textContent=`Page ${pg.toLocaleString()} \u2014 ${inf?inf.name:"idx "+ow}${ex}`;
let dtxt=inf?`${inf.pages.toLocaleString()} pages, ${inf.pct}%`:"";
if(hmCounts&&pg<hmCounts.length&&hmCounts[pg]>0)dtxt+=(dtxt?" \u00b7 ":"")+hmCounts[pg].toLocaleString()+" hits";
if(resMap&&pg<resMap.length){
const cached=resMap[pg];
dtxt+=(dtxt?" \u00b7 ":"")+(cached?'<span style="color:#28c840;font-weight:600">CACHED</span>':'<span style="color:#f04040;font-weight:600">NOT CACHED</span>');
}
ttDt.innerHTML=dtxt;
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
});
cv.addEventListener("pointerup",()=>{drag=false;cv.style.cursor="crosshair"});
cv.addEventListener("pointercancel",()=>{drag=false;cv.style.cursor="crosshair"});
cv.addEventListener("mouseleave",()=>{tt.style.display="none"});
document.getElementById("chk-read").addEventListener("change",e=>{sR=e.target.checked;sched()});
document.getElementById("chk-write").addEventListener("change",e=>{sW=e.target.checked;sched()});
document.getElementById("chk-free").addEventListener("change",e=>{sF=e.target.checked;sched()});
document.getElementById("fade-slider").addEventListener("input",e=>{FADE=parseInt(e.target.value);document.getElementById("fade-val").textContent=(FADE/1000).toFixed(1)+"s"});
document.getElementById("chk-trails").addEventListener("change",e=>{showTrails=e.target.checked;if(!showTrails){trailStart=0;trailN=0}sched()});
document.getElementById("trail-width").addEventListener("input",e=>{trailW=parseFloat(e.target.value);document.getElementById("trail-w-val").textContent=trailW.toFixed(1);sched()});
document.getElementById("chk-heatmap").addEventListener("change",e=>{hmMode=e.target.checked;markZbDirty();sched()});
document.getElementById("chk-cache").addEventListener("change",e=>{cacheMode=e.target.checked;markZbDirty();sched()});
document.getElementById("chk-flame").addEventListener("change",e=>{
showFlame=e.target.checked;const fw=document.getElementById("flame-wrap");
const at=DBIS.filter(d=>d.pages>0);fw.style.height=showFlame?Math.min(at.length*8+24,200)+"px":"0";
if(showFlame){szFlame();drawFlame()}
});
document.getElementById("chk-fixw").addEventListener("change",e=>{fW=e.target.checked;if(fW){reflow()}else{GC=Math.ceil(Math.sqrt(NP));GR=Math.ceil(NP/GC);mS=Math.min(W/GC,H/GR,1);pX=0;cP()}sched()});
window.addEventListener("resize",()=>{if(!oMap)return;szCv();reflow();mS=Math.min(W/GC,H/GR,1);if(sc<mS)sc=mS;cP();render();if(showFlame){szFlame();drawFlame()}});
const wD=document.getElementById("ws-dot"),wS=document.getElementById("ws-status");
function connectWS(){
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws`);
ws.binaryType="arraybuffer";
ws.onopen=()=>{wD.className="dot connected";wS.textContent="Connected"};
ws.onclose=()=>{wD.className="dot disconnected";wS.textContent="Reconnecting...";setTimeout(connectWS,1000)};
ws.onerror=()=>ws.close();
let lagTimer=0;
ws.onmessage=msg=>{
if(typeof msg.data==="string"){try{const j=JSON.parse(msg.data);if(j.lagged){const el=document.getElementById("lag-warn"),lc=document.getElementById("lag-count");lc.textContent=j.lagged.toLocaleString();el.style.display="inline";clearTimeout(lagTimer);lagTimer=setTimeout(()=>{el.style.display="none"},5000)}}catch(e){}return}
if(!oMap)return;const v=new DataView(msg.data);const now=performance.now();let cnt=0;
let lastPg=-1,lastDbi=0,lastOp=0,prevPg=-1;
{let scan=0,maxPg=NP;while(scan+8<=msg.data.byteLength){const pg=v.getUint32(scan,true),op=v.getUint8(scan+6);scan+=8;if(op>=4)continue;if(pg>=maxPg)maxPg=pg+1}if(maxPg>NP)growTo(maxPg)}
let off=0;
while(off+8<=msg.data.byteLength){
const pg=v.getUint32(off,true),dbi=v.getUint16(off+4,true),op=v.getUint8(off+6),rsv=v.getUint8(off+7);off+=8;
if(op===4){if(!replayMode){blockRecording={number:pg,events:[],before:{},faults:0,startTime:performance.now()};blockPanelDirty=true}continue}
if(op===5){
if(!replayMode&&blockRecording&&blockRecording.number===pg){
const txCount=dbi&0xFF,gasEnc=(dbi>>8)&0xFF;
blockRecording.txCount=txCount;
blockRecording.duration=rsv>0?Math.pow(2,rsv/20.0):0;
blockRecording.gasUsed=gasEnc>0?Math.pow(2,gasEnc/10.0):0;
let r=0,w=0,f=0;
for(const e of blockRecording.events){if(e.op===1)r++;else if(e.op===2)w++;else if(e.op===3)f++}
blockRecording.reads=r;blockRecording.writes=w;blockRecording.frees=f;
blockRecording.uniquePages=Object.keys(blockRecording.before).length;
blockRecording.panelSnapshot={
dbiPages:DBIS.map(d=>d.pages),
cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,
cachePrevPer:cachePrevPer?cachePrevPer.slice():null,
cacheTotalCached,cachePrevTotal,unrefCount,
faultWindow:faultWindow.map(w=>({...w})),
faultCurrent:{...faultCurrent}
};
blockHistory.push(blockRecording);
if(blockHistory.length>BLOCK_HISTORY_CAP)blockHistory.shift();
blockRecording=null;
blockPanelDirty=true;
}
continue;
}
if(blockRecording){
if(!(pg in blockRecording.before))blockRecording.before[pg]={om:oMap[pg],rm:resMap?resMap[pg]:0};
blockRecording.events.push({pg,dbi,op});
}
if(pg>=NP)continue;cnt++;
if(op===1)cntR++;else if(op===2)cntW++;else if(op===3)cntF++;
if(cacheMode&&resMap&&op===1&&pg<resMap.length&&!resMap[pg]){faultCounts[dbi]=(faultCounts[dbi]||0)+1;faultCurrent[dbi]=(faultCurrent[dbi]||0)+1;if(blockRecording){blockRecording.faults=(blockRecording.faults||0)+1}}
if(resMap&&pg<resMap.length&&op!==3)resMap[pg]=1;
if(hmCounts)hmCounts[pg]++;
if(sparkAccum[dbi]!==undefined)sparkAccum[dbi]++;
if(op===2&&dbi>0&&dbi<0xFE){const old=oMap[pg];if(old!==dbi){oMap[pg]=dbi;oMapDirty=true;
const di=DBI_IDX[dbi];if(di)di.pages++;
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
if(old===0xFF)unrefCount--;
}}
if(op===3){const old=oMap[pg];if(old!==1){oMap[pg]=1;oMapDirty=true;
const di=DBI_IDX[1];if(di)di.pages++;
const oi=DBI_IDX[old];if(oi&&oi.pages>0)oi.pages--;
if(old===0xFF)unrefCount--;
}}
if(replayMode)continue;
const prev=lOp[pg],prevLive=prev&&(now-lTs[pg])<FADE;
if(!prevLive||op>=prev){
if(prev===0)activePgs++;
lOp[pg]=op;lTs[pg]=now;
activeSet.add(pg);
}
if(showTrails&&prevPg>=0&&prevPg!==pg&&!((op===1&&!sR)||(op===2&&!sW)||(op===3&&!sF)))trailPush(prevPg,pg,op,now);
prevPg=pg;
if(op>=2){lastPg=pg;lastDbi=dbi;lastOp=op}else if(lastPg<0){lastPg=pg;lastDbi=dbi;lastOp=op}
}
tEv+=cnt;rEv+=cnt;document.getElementById("evt-total").textContent=tEv.toLocaleString();
document.getElementById("cnt-read").textContent=cntR.toLocaleString();
document.getElementById("cnt-write").textContent=cntW.toLocaleString();
document.getElementById("cnt-free").textContent=cntF.toLocaleString();
if(lastPg>=0){const inf=DBI_IDX[lastDbi];const nm=inf?inf.name:"dbi"+lastDbi;const opN=lastOp===1?"R":lastOp===2?"W":"F";document.getElementById("evt-latest").textContent=opN+" pg"+lastPg.toLocaleString()+" "+nm}
const el=now-lRT;if(el>=1000){document.getElementById("evt-rate").textContent=Math.round(rEv/(el/1000)).toLocaleString();rEv=0;lRT=now}
if(cnt>0&&!replayMode)kick();
};
}
function szFlame(){
const fw=document.getElementById("flame-wrap"),fc=document.getElementById("flame");
if(!fw||!fc)return;fc.width=fw.clientWidth||300;fc.height=fw.clientHeight||100;
}
function drawFlame(){
const fc=document.getElementById("flame");if(!fc)return;
const ctx=fc.getContext("2d"),cw=fc.width,ch=fc.height;ctx.clearRect(0,0,cw,ch);
const at=DBIS.filter(d=>d.pages>0);if(!at.length)return;
const lw=42,rw=cw-lw,rowH=Math.max(4,Math.floor((ch-14)/Math.max(1,at.length)));
ctx.font="7px monospace";
for(let r=0;r<at.length;r++){
const d=at[r],fd=flameData[d.dbi_index];if(!fd)continue;
const c=PAL[d.dbi_index]||PAL[255],y=r*rowH;
let mx=1;for(let s=0;s<FLAME_SECS;s++){const v=fd[s];if(v>mx)mx=v}
for(let s=0;s<FLAME_SECS;s++){
const idx=((flameHead-FLAME_SECS+s)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
const ops=fd[idx];if(ops<=0)continue;
const x=lw+(s/FLAME_SECS)*rw,w=Math.ceil(rw/FLAME_SECS);
const a=Math.min(1,Math.log2(ops+1)/Math.log2(mx+1));
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},${a.toFixed(2)})`;
ctx.fillRect(x,y,w,rowH-1);
}
ctx.fillStyle="#8b949e";ctx.fillText(d.name.slice(0,6),2,y+rowH-2);
}
const ay=at.length*rowH+2;ctx.fillStyle="#30363d";ctx.fillRect(lw,ay,rw,1);
ctx.fillStyle="#8b949e";ctx.font="7px monospace";
for(let s=30;s<FLAME_SECS;s+=30){
const x=lw+((FLAME_SECS-s)/FLAME_SECS)*rw;
ctx.fillText("-"+s+"s",x,ay+10);
}
}
function buildUnifiedPanel(){
const el=document.getElementById("unified-tables");if(!el)return;el.innerHTML="";
const hdr=document.createElement("div");hdr.className="utbl-row";hdr.style.cssText="cursor:default;border-bottom:1px solid #30363d;margin-bottom:2px;padding-bottom:4px";
hdr.innerHTML='<div class="utbl-sw" style="visibility:hidden"></div><div class="utbl-name" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Table</div><div class="utbl-spark" style="width:80px;text-align:center;color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Activity</div><div class="utbl-cache" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px;text-align:center">Cache</div><div class="utbl-diff" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px">Diff</div><div class="utbl-faults" style="color:#8b949e;font-weight:500;text-transform:uppercase;font-size:9px" title="Page faults over last 3 residency updates">Fault<span style="font-size:7px;text-transform:none"> (3)</span></div>';
el.appendChild(hdr);
DBIS.forEach(d=>{
if(d.pages<=0&&d.dbi_index>2)return;
const c=PAL[d.dbi_index]||PAL[255];
const row=document.createElement("div");row.className="utbl-row";
row.dataset.dbiIndex=d.dbi_index;
const sw=document.createElement("div");sw.className="utbl-sw";
sw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
const nm=document.createElement("div");nm.className="utbl-name";nm.textContent=d.name;
const cv=document.createElement("canvas");cv.className="utbl-spark";cv.width=80;cv.height=16;
cv.style.cssText="width:80px;height:16px;border:1px solid #21262d;border-radius:2px;background:#0d1117";
const cp=document.createElement("div");cp.className="utbl-cache";
const cpBar=document.createElement("div");cpBar.className="utbl-cache-bar-wrap";
const cpFill=document.createElement("div");cpFill.className="utbl-cache-fill";cpFill.style.width="0%";cpFill.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
cpBar.appendChild(cpFill);
const cpPct=document.createElement("div");cpPct.className="utbl-cache-pct";
cp.append(cpBar,cpPct);
const cd=document.createElement("div");cd.className="utbl-diff";
const ft=document.createElement("div");ft.className="utbl-faults";
row.append(sw,nm,cv,cp,cd,ft);
row.addEventListener("click",()=>{const i=d.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
el.appendChild(row);
d._sparkCv=cv;d._sparkCtx=cv.getContext("2d");
d._cacheEl=cp;d._cacheFill=cpFill;d._cachePct=cpPct;d._diffEl=cd;d._faultEl=ft;
});
}
function drawSparkline(d){
const cv=d._sparkCv,ctx=d._sparkCtx;if(!cv||!ctx)return;
const sd=sparkData[d.dbi_index];if(!sd)return;
const w=cv.width,h=cv.height,c=PAL[d.dbi_index]||PAL[255];
ctx.clearRect(0,0,w,h);
let mx=1;for(let i=0;i<SPARK_SECS;i++){const v=sd.ring[i];if(v>mx)mx=v}
const latest=sd.ring[((sd.head-1)%SPARK_SECS+SPARK_SECS)%SPARK_SECS];
ctx.beginPath();ctx.moveTo(0,h);
for(let i=0;i<SPARK_SECS;i++){
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
if(i===0)ctx.lineTo(x,y);else ctx.lineTo(x,y);
}
ctx.lineTo(w,h);ctx.closePath();
ctx.fillStyle=`rgba(${c[0]},${c[1]},${c[2]},0.3)`;ctx.fill();
ctx.beginPath();
for(let i=0;i<SPARK_SECS;i++){
const idx=((sd.head-SPARK_SECS+i)%SPARK_SECS+SPARK_SECS)%SPARK_SECS;
const x=(i/(SPARK_SECS-1))*w,y=h-(sd.ring[idx]/mx)*h;
if(i===0)ctx.moveTo(x,y);else ctx.lineTo(x,y);
}
ctx.strokeStyle=`rgba(${c[0]},${c[1]},${c[2]},0.8)`;ctx.lineWidth=1;ctx.stroke();
if(d._sparkVal)d._sparkVal.textContent=latest.toLocaleString();
}
function buildTreeInfo(info){
const el=document.getElementById("tree-info");if(!el)return;el.innerHTML="";
info.forEach(ti=>{
if(ti.height===0&&ti.branch_pages===0&&ti.leaf_pages===0&&ti.large_pages===0)return;
const card=document.createElement("div");card.className="tree-card";
const total=ti.branch_pages+ti.leaf_pages+ti.large_pages;
const ipl=ti.leaf_pages>0?Math.round(ti.items/ti.leaf_pages):0;
let html=`<div class="tree-name">${ti.name} <span style="color:#8b949e;font-weight:400">depth ${ti.height}</span></div>`;
html+=`<div class="tree-stat">B:${ti.branch_pages.toLocaleString()} L:${ti.leaf_pages.toLocaleString()} O:${ti.large_pages.toLocaleString()} items:${ti.items.toLocaleString()}</div>`;
if(total>0){
const bp=Math.max(1,Math.round(ti.branch_pages/total*100)),lp=Math.max(1,Math.round(ti.leaf_pages/total*100)),op=Math.max(1,100-bp-lp);
html+=`<div class="tree-bar"><div class="tree-seg" style="width:${bp}%;background:#3c82f6"></div><div class="tree-seg" style="width:${lp}%;background:#4ade80"></div><div class="tree-seg" style="width:${op}%;background:#f58231"></div></div>`;
}
if(ipl>0)html+=`<div class="tree-stat">~${ipl.toLocaleString()} items/leaf</div>`;
card.innerHTML=html;
card.addEventListener("click",()=>{const i=ti.dbi_index;if(sel.has(i))sel.delete(i);else sel.add(i);uAR();markZbDirty();sched()});
el.appendChild(card);
});
}
function fmtGas(g){if(g>=1e6)return(g/1e6).toFixed(1)+"M";if(g>=1e3)return(g/1e3).toFixed(0)+"K";return g.toFixed(0)}
function updateBlockPanel(){
blockPanelDirty=false;
const el=document.getElementById("block-panel");if(!el)return;
el.innerHTML="";
if(blockRecording){
const c=document.createElement("div");c.className="block-card recording";
const elapsed=performance.now()-blockRecording.startTime;
const elStr=elapsed<1000?elapsed.toFixed(0)+"ms":(elapsed/1000).toFixed(1)+"s";
c.innerHTML=`<div class="block-num">&#x23F1; #${blockRecording.number.toLocaleString()} recording\u2026</div><div class="block-stat">${blockRecording.events.length.toLocaleString()} events &nbsp;${elStr}</div>`;
el.appendChild(c);
}
for(let i=blockHistory.length-1;i>=0;i--){
const b=blockHistory[i],isReplaying=replayMode&&replayBlockRef===b;
const c=document.createElement("div");c.className="block-card"+(isReplaying?" replaying":"");
const total=b.reads+b.writes+b.frees;
const dur=b.duration>0?(b.duration<1000?b.duration.toFixed(0)+"ms":(b.duration/1000).toFixed(1)+"s"):"";
const gas=b.gasUsed>0?fmtGas(b.gasUsed)+"gas":"";
c.innerHTML=`<div class="block-num">&#x25B6; #${b.number.toLocaleString()} &nbsp;${b.txCount}tx${dur?" &nbsp;"+dur:""}${gas?" &nbsp;"+gas:""} &nbsp;${total.toLocaleString()} ops</div><div class="block-stat"><span style="color:#3c82f6">R:${b.reads.toLocaleString()}</span> <span style="color:#ef4444">W:${b.writes.toLocaleString()}</span> <span style="color:#eab308">F:${b.frees.toLocaleString()}</span>${b.faults?` <span style="color:#d946ef">${b.faults}f</span>`:""} &nbsp;${(b.uniquePages||0).toLocaleString()} pages</div>`;
c.addEventListener("click",()=>startReplay(b));
el.appendChild(c);
}
}
function snapPanelState(){
return{dbiPages:DBIS.map(d=>d.pages),cacheCachedPer:cacheCachedPer?cacheCachedPer.slice():null,cachePrevPer:cachePrevPer?cachePrevPer.slice():null,cacheTotalCached,cachePrevTotal,unrefCount,faultWindow:faultWindow.map(w=>({...w})),faultCurrent:{...faultCurrent}};
}
function applyPanelSnap(snap){
DBIS.forEach((d,i)=>{d.pages=snap.dbiPages[i]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
if(snap.cacheCachedPer&&cacheCachedPer){for(let i=0;i<cacheCachedPer.length;i++)cacheCachedPer[i]=snap.cacheCachedPer[i]||0}
cachePrevPer=snap.cachePrevPer?snap.cachePrevPer.slice():null;
cacheTotalCached=snap.cacheTotalCached;cachePrevTotal=snap.cachePrevTotal;
unrefCount=snap.unrefCount;
faultWindow=snap.faultWindow.map(w=>({...w}));faultCurrent={...snap.faultCurrent};
cacheStale=true;bldTbl();updateCacheStats();
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
}
function startReplay(block){
if(replayMode&&replayBlockRef===block&&replayTimer)return;
if(!liveSnapshot)liveSnapshot=snapPanelState();
replayMode=true;replayBlockRef=block;
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
lOp.fill(0);activeSet.clear();activePgs=0;
if(hmCounts)hmCounts.fill(0);
trailStart=0;trailN=0;
for(const pgStr in block.before){
const pg=parseInt(pgStr,10);if(pg>=NP)continue;
const snap=block.before[pg];
oMap[pg]=snap.om;
if(resMap&&pg<resMap.length)resMap[pg]=snap.rm;
}
if(block.panelSnapshot)applyPanelSnap(block.panelSnapshot);
markZbDirty();sched();
document.getElementById("block-live").style.color="#8b949e";
updateBlockPanel();
const events=block.events;
const batchSize=Math.max(1,Math.ceil(events.length/60));
let idx=0;
function step(){
if(!replayMode){replayTimer=null;return}
const end=Math.min(idx+batchSize,events.length);
const now=performance.now();
for(let i=idx;i<end;i++){
const e=events[i];
if(e.pg>=NP)continue;
if(hmCounts)hmCounts[e.pg]++;
if(cacheMode&&resMap&&e.op===1&&e.pg<resMap.length&&!resMap[e.pg]){faultCounts[e.dbi]=(faultCounts[e.dbi]||0)+1;faultCurrent[e.dbi]=(faultCurrent[e.dbi]||0)+1}
if(resMap&&e.pg<resMap.length&&e.op!==3)resMap[e.pg]=1;
lOp[e.pg]=e.op;lTs[e.pg]=now;
if(!activeSet.has(e.pg)){activeSet.add(e.pg);activePgs++}
if(e.op===2&&e.dbi>0&&e.dbi<0xFE)oMap[e.pg]=e.dbi;
if(showTrails&&i>0&&!((e.op===1&&!sR)||(e.op===2&&!sW)||(e.op===3&&!sF))){const p=events[i-1];if(p.pg!==e.pg)trailPush(p.pg,e.pg,e.op,now)}
}
idx=end;
markZbDirty();kick();
if(idx<events.length){replayTimer=requestAnimationFrame(step)}
else{
replayTimer=null;
const pin=performance.now()+FADE*10;
for(const pg of activeSet)lTs[pg]=pin;
markZbDirty();sched();
updateBlockPanel();
}
}
replayTimer=requestAnimationFrame(step);
}
function goLive(){
replayMode=false;replayBlockRef=null;
if(replayTimer){cancelAnimationFrame(replayTimer);replayTimer=null}
lOp.fill(0);activeSet.clear();activePgs=0;
trailStart=0;trailN=0;
if(liveSnapshot){applyPanelSnap(liveSnapshot);liveSnapshot=null}
oMapDirty=true;cacheStale=true;
markZbDirty();sched();
document.getElementById("block-live").style.color="#4ade80";
updateBlockPanel();updateCacheStats();
}
window._goLive=goLive;
function connectResidencyWS(){
const ws=new WebSocket(`${location.protocol==="https:"?"wss:":"ws:"}//${location.host}/ws_residency`);
ws.binaryType="arraybuffer";
ws.onclose=()=>setTimeout(connectResidencyWS,3000);
ws.onerror=()=>ws.close();
ws.onmessage=msg=>{
if(!msg.data||msg.data.byteLength<1)return;
if(replayMode)return;
const v=new DataView(msg.data);
const msgType=v.getUint8(0);
if(msgType===0){
const src=new Uint8Array(msg.data,1);
const newLen=src.length;
if(!resMap||resMap.length!==newLen){
const oldMap=resMap;const oldLen=oldMap?oldMap.length:0;
resMap=new Uint8Array(newLen);
if(oldMap){
const shared=Math.min(oldLen,newLen,NP);
for(let i=0;i<shared;i++){
const ns=src[i],os=oldMap[i];
if(ns!==os){
const dbi=oMap?oMap[i]:0;
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
}
resMap[i]=ns;
}
for(let i=shared;i<newLen;i++){
resMap[i]=src[i];
if(src[i]){cacheTotalCached++;if(cacheCachedPer&&oMap){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
}
}else{
resMap.set(src);
cacheTotalCached=0;if(cacheCachedPer)cacheCachedPer.fill(0);
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
for(let i=0;i<lim;i++){
if(src[i]){cacheTotalCached++;if(cacheCachedPer){const dbi=oMap[i];if(dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}}
}
}
}else{
const lim=Math.min(newLen,NP,oMap?oMap.length:0);
for(let i=0;i<lim;i++){
const ns=src[i],os=resMap[i];
if(ns!==os){
const dbi=oMap?oMap[i]:0;
if(ns&&!os){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
else if(!ns&&os){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
}
resMap[i]=ns;
}
}
cacheStale=true;
cacheLastUpdate=performance.now();
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
if(cacheMode)markZbDirty();
updateCacheStats();
if(cacheMode)sched();
}else if(msgType===1){
if(!resMap)return;
const spanCount=v.getUint32(1,true);
let off=5;
for(let s=0;s<spanCount;s++){
const start=v.getUint32(off,true);off+=4;
const len=v.getUint32(off,true);off+=4;
const state=v.getUint8(off);off+=1;
for(let p=start;p<start+len&&p<resMap.length;p++){
const oldState=resMap[p];
if(oldState!==state){
const dbi=oMap?oMap[p]:0;
if(state&&!oldState){cacheTotalCached++;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]++}
else if(!state&&oldState){cacheTotalCached--;if(cacheCachedPer&&dbi<cacheCachedPer.length)cacheCachedPer[dbi]--}
}
resMap[p]=state;
}
}
cacheStale=true;
cacheLastUpdate=performance.now();
faultWindow.push({...faultCurrent});if(faultWindow.length>3)faultWindow.shift();faultCurrent={};
if(cacheMode)markZbDirty();
updateCacheStats();
if(cacheMode)sched();
}
};
}
let cacheStatsPending=false;
function updateCacheStats(){
if(!cacheStale||!cacheCachedPer)return;
if(cacheStatsPending)return;
cacheStatsPending=true;
requestAnimationFrame(()=>{
cacheStatsPending=false;
if(!cacheStale||!cacheCachedPer)return;
cacheStale=false;
const limit=Math.min(resMap?resMap.length:0,NP);
const totalPct=limit>0?(cacheTotalCached/limit*100).toFixed(1):"0";
const prevTotalPct=limit>0?(cachePrevTotal/limit*100):0;
const totalDiff=limit>0?(cacheTotalCached/limit*100-prevTotalPct):0;
const tdS=cachePrevPer?(totalDiff>=0?"+":"")+totalDiff.toFixed(2)+"%":"";
const tdC=totalDiff>0?"#4ade80":totalDiff<0?"#f87171":"#8b949e";
document.getElementById("cache-total").innerHTML=`${cacheTotalCached.toLocaleString()}/${limit.toLocaleString()} (${totalPct}%) ${cachePrevPer?`<span style="color:${tdC}">${tdS}</span>`:""}`;
DBIS.forEach(d=>{
if(!d._cacheEl)return;
const total=d.pages;if(total<=0){if(d._cachePct)d._cachePct.textContent="-";if(d._cacheFill)d._cacheFill.style.width="0%";d._diffEl.textContent="";d._faultEl.textContent="";return}
const cached=cacheCachedPer[d.dbi_index]||0;
const pct=total>0?(cached/total*100):0;
if(d._cacheFill)d._cacheFill.style.width=pct.toFixed(1)+"%";
if(d._cachePct){d._cachePct.textContent=pct.toFixed(1)+"%";d._cachePct.style.color=pct>50?"#4ade80":"#f87171"}
const prevCached=cachePrevPer?cachePrevPer[d.dbi_index]||0:0;
const prevPct=total>0?(prevCached/total*100):0;
const diff=cachePrevPer?(pct-prevPct):0;
if(cachePrevPer){
d._diffEl.textContent=(diff>=0?"+":"")+diff.toFixed(2)+"%";
d._diffEl.style.color=diff>0?"#4ade80":diff<0?"#f87171":"#8b949e";
}
const fc=rollingFaults(d.dbi_index);
d._faultEl.textContent=fc>0?fc+"f":"";
});
cachePrevPer=cacheCachedPer.slice();
cachePrevTotal=cacheTotalCached;
});
}
async function init(){
try{
const r=await fetch("/api/info"),info=await r.json();NP=info.page_count;PS=info.page_size;
DBIS=info.dbi_names.map((n,i)=>({name:n,dbi_index:i,pages:0,pct:0,size_mb:0}));
DBI_IDX={};DBIS.forEach(d=>{DBI_IDX[d.dbi_index]=d});
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 loading owner map\u2026`;
}catch(e){document.getElementById("subtitle").textContent="Failed: "+e.message}
try{
const lt=document.getElementById("load-text"),lbw=document.getElementById("load-bar-wrap"),lb=document.getElementById("load-bar");
lt.textContent="Downloading owner map\u2026";lbw.style.display="block";
const r=await fetch("/api/owner_map");
if(r.ok){
const total=parseInt(r.headers.get("content-length")||"0",10)||NP;
const reader=r.body.getReader();
const buf=new Uint8Array(total||NP);let pos=0;
for(;;){const{done,value}=await reader.read();if(done)break;buf.set(value,pos);pos+=value.length;
const pct=total>0?Math.min(100,pos/total*100):0;
lb.style.width=pct.toFixed(1)+"%";
lt.textContent=`Downloading owner map\u2026 ${(pos/(1024*1024)).toFixed(1)}/${(total/(1024*1024)).toFixed(1)} MB`;
}
oMap=buf.length===pos?buf:buf.slice(0,pos);
lt.textContent="Processing owner map\u2026";lb.style.width="100%";
await new Promise(r=>setTimeout(r,0));
const ct={};for(let i=0;i<NP;i++){const o=oMap[i];ct[o]=(ct[o]||0)+1}
DBIS.forEach(d=>{d.pages=ct[d.dbi_index]||0;d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
}else{oMap=new Uint8Array(NP);oMap.fill(0xFF)}
}catch(e){oMap=new Uint8Array(NP);oMap.fill(0xFF)}
unrefCount=oMap.reduce((n,b)=>n+(b===0xFF?1:0),0);const ref=NP-unrefCount;
const gb=(NP*PS/(1024**3)).toFixed(2);
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${ref.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
lOp=new Uint8Array(NP);lTs=new Float64Array(NP);hmCounts=new Uint32Array(NP);
cacheCachedPer=new Array(DBIS.length).fill(0);
DBIS.forEach(d=>{sparkData[d.dbi_index]={ring:new Array(SPARK_SECS).fill(0),head:0};sparkAccum[d.dbi_index]=0;flameData[d.dbi_index]=new Array(FLAME_SECS).fill(0)});
bldTbl();buildUnifiedPanel();
let treeInfo=[];
try{const r=await fetch("/api/tree_info");if(r.ok)treeInfo=await r.json()}catch(e){}
if(treeInfo.length>0)buildTreeInfo(treeInfo);
ldEl.style.display="none";cv.style.display="block";boot();connectWS();connectResidencyWS();
const flCv=document.getElementById("flame");
flCv.addEventListener("mousemove",e=>{
if(!showFlame)return;
const at=DBIS.filter(d=>d.pages>0);if(!at.length){tt.style.display="none";return}
const rect=flCv.getBoundingClientRect();
const mx=e.clientX-rect.left,my=e.clientY-rect.top;
const lw=42,rw=flCv.width-lw;
const rowH=Math.max(4,Math.floor((flCv.height-14)/Math.max(1,at.length)));
const ri=Math.floor(my/rowH);
if(ri<0||ri>=at.length||mx<lw){tt.style.display="none";return}
const d=at[ri],c=PAL[d.dbi_index]||PAL[255];
const secIdx=Math.floor((mx-lw)/rw*FLAME_SECS);
const idx=((flameHead-FLAME_SECS+secIdx)%FLAME_SECS+FLAME_SECS)%FLAME_SECS;
const fd=flameData[d.dbi_index];
const ops=fd?fd[idx]:0;
const ago=FLAME_SECS-secIdx;
ttSw.style.background=`rgb(${c[0]},${c[1]},${c[2]})`;
ttNm.textContent=d.name;
ttDt.innerHTML=`${ops.toLocaleString()} ops/s \u00b7 ${ago}s ago`;
tt.style.display="block";tt.style.left=(e.clientX+14)+"px";tt.style.top=(e.clientY+14)+"px";
});
flCv.addEventListener("mouseleave",()=>{tt.style.display="none"});
setInterval(()=>{
DBIS.forEach(d=>{
const v=sparkAccum[d.dbi_index]||0;
const sd=sparkData[d.dbi_index];if(sd){sd.ring[sd.head%SPARK_SECS]=v;sd.head++}
const fd=flameData[d.dbi_index];if(fd)fd[flameHead%FLAME_SECS]=v;
sparkAccum[d.dbi_index]=0;
});
flameHead++;
DBIS.forEach(d=>drawSparkline(d));
if(showFlame)drawFlame();
},1000);
setInterval(()=>{if(!oMap||!oMapDirty||replayMode)return;oMapDirty=false;
DBIS.forEach(d=>{d.pct=NP>0?Math.round(d.pages/NP*10000)/100:0;d.size_mb=Math.round(d.pages*PS/(1024*1024)*10)/10});
bldTbl();uAR();markZbDirty();sched();
const rf=NP-unrefCount,gb=(NP*PS/(1024**3)).toFixed(2);
document.getElementById("subtitle").textContent=`${NP.toLocaleString()} pages \u00b7 ${rf.toLocaleString()} ref \u00b7 ${unrefCount.toLocaleString()} unref \u00b7 ${PS>=1024?(PS/1024)+"K":PS} \u00b7 ${gb} GB \u00b7 ${DBIS.length} tables`;
},2000);
setInterval(()=>{
if(!cacheLastUpdate)return;
const ago=Math.round((performance.now()-cacheLastUpdate)/1000);
let txt;
if(ago<60)txt=ago+"s ago";
else if(ago<3600)txt=Math.floor(ago/60)+"m "+ago%60+"s ago";
else txt=Math.floor(ago/3600)+"h ago";
document.getElementById("cache-ago").textContent=txt;
},1000);
setInterval(()=>{if(blockPanelDirty)updateBlockPanel()},250);
setInterval(()=>{
if(activeSet.size<5000)return;
const now=performance.now();
for(const pg of activeSet){
if(!lOp[pg]||now-lTs[pg]>=FADE){activeSet.delete(pg);if(lOp[pg]){lOp[pg]=0;activePgs=Math.max(0,activePgs-1)}}
}
},5000);
}
init();
})();
</script>
</body>
</html>

View File

@@ -0,0 +1,577 @@
use std::path::PathBuf;
use std::sync::{
atomic::{AtomicUsize, Ordering},
mpsc, Arc, RwLock,
};
use axum::{
extract::{
ws::{Message, WebSocket, WebSocketUpgrade},
State,
},
http::{header, StatusCode},
response::{Html, IntoResponse},
routing::get,
Router,
};
use reth_libmdbx::pageviz::{PageEvent, PageOp};
use serde::Serialize;
use tokio::sync::broadcast;
pub mod walker;
unsafe extern "C" {
fn mdbx_pageviz_emit_block_marker(op: u8, block_number: u32, tx_count: u16, duration_encoded: u8, gas_encoded: u8);
}
pub fn pageviz_emit_block_marker(block_number: u64, start: bool, tx_count: u16, duration_ms: u32, gas_used: u64) {
let dur_enc = if duration_ms == 0 { 0u8 } else { ((duration_ms as f64).log2() * 20.0).round().min(255.0) as u8 };
let gas_enc = if gas_used == 0 { 0u8 } else { ((gas_used as f64).log2() * 10.0).round().min(255.0) as u8 };
unsafe {
let op = if start { 4u8 } else { 5u8 };
mdbx_pageviz_emit_block_marker(op, block_number as u32, tx_count, dur_enc, gas_enc);
}
}
#[derive(Clone, Default, Serialize)]
struct ResidencyStats {
total_pages: u64,
cached_pages: u64,
pct: f64,
per_table: Vec<TableResidency>,
}
#[derive(Clone, Default, Serialize)]
struct TableResidency {
dbi: usize,
name: String,
total: u64,
cached: u64,
pct: f64,
}
#[derive(Clone)]
struct AppState {
info: VizInfo,
owner_map: Arc<RwLock<Vec<u8>>>,
tree_info: Arc<Vec<walker::TreeInfo>>,
subscribers: Arc<AtomicUsize>,
event_tx: broadcast::Sender<Vec<u8>>,
residency: Arc<RwLock<Vec<u8>>>,
residency_stats: Arc<RwLock<ResidencyStats>>,
residency_tx: broadcast::Sender<Vec<u8>>,
residency_subscribers: Arc<AtomicUsize>,
}
#[derive(Debug, Clone, Serialize)]
pub struct VizInfo {
pub page_count: u64,
pub page_size: u32,
pub dbi_names: Vec<String>,
}
pub struct VizConfig {
pub port: u16,
pub page_count: u64,
pub page_size: u32,
pub dbi_names: Vec<String>,
pub owner_map: Vec<u8>,
pub tree_info: Vec<walker::TreeInfo>,
pub db_path: PathBuf,
}
const WIRE_EVENT_SIZE: usize = 8;
fn encode_events(events: &[PageEvent]) -> Vec<u8> {
let mut buf = Vec::with_capacity(events.len() * WIRE_EVENT_SIZE);
for ev in events {
buf.extend_from_slice(&ev.pgno.to_le_bytes());
if ev.op.is_block_marker() {
let tx_lo = (ev.dbi & 0xFF) as u8;
let gas = ((ev.dbi >> 24) & 0xFF) as u8;
buf.extend_from_slice(&u16::to_le_bytes((gas as u16) << 8 | tx_lo as u16));
buf.push(ev.op as u8);
buf.push(((ev.dbi >> 16) & 0xFF) as u8);
} else {
buf.extend_from_slice(&(ev.dbi as u16).to_le_bytes());
buf.push(ev.op as u8);
buf.push(0u8);
}
}
buf
}
struct ReadOnlyMapping {
base: *mut libc::c_void,
len: usize,
}
impl ReadOnlyMapping {
fn new(path: &std::path::Path) -> Option<Self> {
use std::os::unix::io::AsRawFd;
let file = std::fs::File::open(path).ok()?;
let len = file.metadata().ok()?.len() as usize;
if len == 0 {
return None;
}
let base = unsafe {
libc::mmap(
std::ptr::null_mut(),
len,
libc::PROT_READ,
libc::MAP_SHARED,
file.as_raw_fd(),
0,
)
};
if base == libc::MAP_FAILED {
return None;
}
Some(Self { base, len })
}
fn remap(&mut self, path: &std::path::Path) -> bool {
use std::os::unix::io::AsRawFd;
let file = match std::fs::File::open(path) {
Ok(f) => f,
Err(_) => return false,
};
let new_len = match file.metadata() {
Ok(m) => m.len() as usize,
Err(_) => return false,
};
if new_len == 0 {
return false;
}
unsafe {
libc::munmap(self.base, self.len);
}
let base = unsafe {
libc::mmap(
std::ptr::null_mut(),
new_len,
libc::PROT_READ,
libc::MAP_SHARED,
file.as_raw_fd(),
0,
)
};
if base == libc::MAP_FAILED {
return false;
}
self.base = base;
self.len = new_len;
true
}
}
impl Drop for ReadOnlyMapping {
fn drop(&mut self) {
unsafe {
libc::munmap(self.base, self.len);
}
}
}
unsafe impl Send for ReadOnlyMapping {}
pub async fn start_viz_server(
config: VizConfig,
rx: mpsc::Receiver<Vec<PageEvent>>,
) -> std::io::Result<()> {
let (event_tx, _) = broadcast::channel::<Vec<u8>>(256);
let (residency_tx, _) = broadcast::channel::<Vec<u8>>(64);
let owner_map = Arc::new(RwLock::new(config.owner_map));
let subscribers = Arc::new(AtomicUsize::new(0));
let state = AppState {
info: VizInfo {
page_count: config.page_count,
page_size: config.page_size,
dbi_names: config.dbi_names,
},
owner_map: owner_map.clone(),
tree_info: Arc::new(config.tree_info),
subscribers: subscribers.clone(),
event_tx: event_tx.clone(),
residency: Arc::new(RwLock::new(Vec::new())),
residency_stats: Arc::new(RwLock::new(ResidencyStats::default())),
residency_tx: residency_tx.clone(),
residency_subscribers: Arc::new(AtomicUsize::new(0)),
};
let bridge_tx = event_tx.clone();
std::thread::Builder::new()
.name("viz-bridge".into())
.spawn(move || {
while let Ok(events) = rx.recv() {
if events.is_empty() {
continue;
}
{
let mut map = owner_map.write().unwrap();
for ev in &events {
let pg = ev.pgno as usize;
if ev.op == PageOp::Write && ev.dbi > 0 && ev.dbi < 0xFE && pg < map.len()
{
map[pg] = ev.dbi as u8;
}
}
}
if subscribers.load(Ordering::Relaxed) > 0 {
let encoded = encode_events(&events);
let _ = bridge_tx.send(encoded);
}
}
})
.expect("failed to spawn viz-bridge thread");
let res_map = state.residency.clone();
let res_stats = state.residency_stats.clone();
let res_tx = state.residency_tx.clone();
let res_subs = state.residency_subscribers.clone();
let own_map = state.owner_map.clone();
let info_clone = state.info.clone();
let db_path = config.db_path;
let mdbx_ps = config.page_size;
let sys_ps = unsafe { libc::sysconf(libc::_SC_PAGESIZE) } as u32;
std::thread::Builder::new()
.name("residency-poller".into())
.spawn(move || {
let mut mapping: Option<ReadOnlyMapping> = None;
let mut prev: Vec<u8> = Vec::new();
if db_path.as_os_str().is_empty() {
tracing::warn!("residency-poller: no db_path configured, residency polling disabled");
return;
}
loop {
std::thread::sleep(std::time::Duration::from_secs(3));
if res_subs.load(Ordering::Relaxed) == 0 {
continue;
}
if mapping.is_none() {
match ReadOnlyMapping::new(&db_path) {
Some(m) => {
tracing::info!(
"residency-poller: mapped {} ({} bytes, mdbx_ps={}, sys_ps={})",
db_path.display(),
m.len,
mdbx_ps,
sys_ps,
);
mapping = Some(m);
}
None => {
tracing::warn!(
"residency-poller: failed to mmap {}",
db_path.display()
);
continue;
}
}
}
let file_len = match std::fs::metadata(&db_path) {
Ok(m) => m.len() as usize,
Err(_) => continue,
};
let m = mapping.as_mut().unwrap();
if file_len > m.len {
tracing::info!(
"residency-poller: file grew {} -> {}, remapping",
m.len,
file_len,
);
if !m.remap(&db_path) {
tracing::warn!("residency-poller: remap failed");
mapping = None;
continue;
}
}
let m = mapping.as_ref().unwrap();
let base = m.base;
let len = m.len;
if mdbx_ps == 0 || sys_ps == 0 || len == 0 {
continue;
}
let sys_pages = (len + sys_ps as usize - 1) / sys_ps as usize;
let mdbx_page_count = len / mdbx_ps as usize;
let mut mincore_vec = vec![0u8; sys_pages];
let ret = unsafe { libc::mincore(base, len, mincore_vec.as_mut_ptr()) };
if ret != 0 {
continue;
}
let mut cur = vec![0u8; mdbx_page_count];
if mdbx_ps == sys_ps {
for i in 0..mdbx_page_count.min(sys_pages) {
cur[i] = mincore_vec[i] & 1;
}
} else if mdbx_ps > sys_ps {
let ratio = mdbx_ps as usize / sys_ps as usize;
for i in 0..mdbx_page_count {
let base_idx = i * ratio;
let mut all_resident = true;
for j in 0..ratio {
if base_idx + j >= sys_pages
|| (mincore_vec[base_idx + j] & 1) == 0
{
all_resident = false;
break;
}
}
cur[i] = if all_resident { 1 } else { 0 };
}
} else {
let ratio = sys_ps as usize / mdbx_ps as usize;
for i in 0..mdbx_page_count {
let sys_idx = i / ratio;
cur[i] = if sys_idx < sys_pages {
mincore_vec[sys_idx] & 1
} else {
0
};
}
}
{
let mut rm = res_map.write().unwrap();
rm.clear();
rm.extend_from_slice(&cur);
}
{
let omap = own_map.read().unwrap();
let max_dbi = info_clone.dbi_names.len();
let mut total_per = vec![0u64; max_dbi];
let mut cached_per = vec![0u64; max_dbi];
let mut total_cached = 0u64;
let limit = mdbx_page_count.min(omap.len());
for i in 0..limit {
let dbi = omap[i] as usize;
if dbi < max_dbi {
total_per[dbi] += 1;
if cur[i] == 1 {
cached_per[dbi] += 1;
total_cached += 1;
}
}
}
let mut stats = res_stats.write().unwrap();
stats.total_pages = mdbx_page_count as u64;
stats.cached_pages = total_cached;
stats.pct = if mdbx_page_count > 0 {
(total_cached as f64 / mdbx_page_count as f64) * 100.0
} else {
0.0
};
stats.per_table = (0..max_dbi)
.filter(|&d| total_per[d] > 0)
.map(|d| TableResidency {
dbi: d,
name: info_clone.dbi_names.get(d).cloned().unwrap_or_default(),
total: total_per[d],
cached: cached_per[d],
pct: if total_per[d] > 0 {
(cached_per[d] as f64 / total_per[d] as f64) * 100.0
} else {
0.0
},
})
.collect();
}
if prev.len() != cur.len() {
let mut buf = Vec::with_capacity(1 + cur.len());
buf.push(0u8);
buf.extend_from_slice(&cur);
let _ = res_tx.send(buf);
tracing::info!(
"residency-poller: broadcast full snapshot ({} pages)",
cur.len(),
);
} else {
let mut spans: Vec<u8> = Vec::new();
let mut span_count: u32 = 0;
spans.extend_from_slice(&[1u8]);
spans.extend_from_slice(&[0u8; 4]);
let mut i = 0;
while i < cur.len() {
if cur[i] != prev[i] {
let state_val = cur[i];
let start = i as u32;
let mut end = i + 1;
while end < cur.len()
&& cur[end] != prev[end]
&& cur[end] == state_val
{
end += 1;
}
let run_len = (end - i) as u32;
spans.extend_from_slice(&start.to_le_bytes());
spans.extend_from_slice(&run_len.to_le_bytes());
spans.push(state_val);
span_count += 1;
i = end;
} else {
i += 1;
}
}
if span_count > 0 {
spans[1..5].copy_from_slice(&span_count.to_le_bytes());
let _ = res_tx.send(spans);
}
}
prev = cur;
}
})
.expect("failed to spawn residency-poller thread");
let app = Router::new()
.route("/", get(index_handler))
.route("/ws", get(ws_handler))
.route("/api/info", get(info_handler))
.route("/api/owner_map", get(owner_map_handler))
.route("/api/tree_info", get(tree_info_handler))
.route("/api/residency", get(residency_handler))
.route("/api/residency_stats", get(residency_stats_handler))
.route("/ws_residency", get(ws_residency_handler))
.with_state(state);
let addr = format!("0.0.0.0:{}", config.port);
tracing::info!("mdbx-viz server listening on {addr}");
let listener = tokio::net::TcpListener::bind(&addr).await?;
axum::serve(listener, app).await?;
Ok(())
}
async fn index_handler() -> Html<&'static str> {
Html(include_str!("index.html"))
}
async fn info_handler(State(state): State<AppState>) -> impl IntoResponse {
axum::Json(state.info)
}
async fn owner_map_handler(State(state): State<AppState>) -> impl IntoResponse {
let map = state.owner_map.read().unwrap();
(
StatusCode::OK,
[(header::CONTENT_TYPE, "application/octet-stream")],
map.clone(),
)
.into_response()
}
async fn tree_info_handler(State(state): State<AppState>) -> impl IntoResponse {
axum::Json(state.tree_info.as_ref().clone())
}
async fn ws_handler(
ws: WebSocketUpgrade,
State(state): State<AppState>,
) -> impl IntoResponse {
ws.on_upgrade(move |socket| handle_ws(socket, state))
}
async fn handle_ws(mut socket: WebSocket, state: AppState) {
state.subscribers.fetch_add(1, Ordering::Relaxed);
let mut rx = state.event_tx.subscribe();
loop {
match rx.recv().await {
Ok(data) => {
if socket.send(Message::Binary(data.into())).await.is_err() {
break;
}
}
Err(broadcast::error::RecvError::Lagged(n)) => {
tracing::warn!("ws client lagged by {n} messages");
let msg = format!("{{\"lagged\":{n}}}");
if socket.send(Message::Text(msg)).await.is_err() {
break;
}
}
Err(broadcast::error::RecvError::Closed) => break,
}
}
state.subscribers.fetch_sub(1, Ordering::Relaxed);
}
async fn residency_handler(State(state): State<AppState>) -> impl IntoResponse {
let map = state.residency.read().unwrap();
(
StatusCode::OK,
[(header::CONTENT_TYPE, "application/octet-stream")],
map.clone(),
)
.into_response()
}
async fn residency_stats_handler(State(state): State<AppState>) -> impl IntoResponse {
let stats = state.residency_stats.read().unwrap();
axum::Json(stats.clone())
}
async fn ws_residency_handler(
ws: WebSocketUpgrade,
State(state): State<AppState>,
) -> impl IntoResponse {
ws.on_upgrade(move |socket| handle_ws_residency(socket, state))
}
async fn handle_ws_residency(mut socket: WebSocket, state: AppState) {
state.residency_subscribers.fetch_add(1, Ordering::Relaxed);
{
let msg = {
let map = state.residency.read().unwrap();
if !map.is_empty() {
let mut buf = Vec::with_capacity(1 + map.len());
buf.push(0u8);
buf.extend_from_slice(&map);
Some(buf)
} else {
None
}
};
if let Some(msg) = msg {
let _ = socket.send(Message::Binary(msg.into())).await;
}
}
let mut rx = state.residency_tx.subscribe();
loop {
match rx.recv().await {
Ok(data) => {
if socket.send(Message::Binary(data.into())).await.is_err() {
break;
}
}
Err(broadcast::error::RecvError::Lagged(_)) => continue,
Err(broadcast::error::RecvError::Closed) => break,
}
}
state.residency_subscribers.fetch_sub(1, Ordering::Relaxed);
}

View File

@@ -0,0 +1,93 @@
use std::sync::mpsc;
use std::time::Duration;
use clap::Parser;
use reth_libmdbx::pageviz::{PageEvent, PageOp};
#[derive(Parser)]
#[command(name = "reth-mdbx-viz", about = "Real-time MDBX page visualization")]
struct Args {
/// Path to the .bin owner map file (generated by mdbx-viz Python tool)
#[arg(short, long)]
bin: Option<String>,
/// Page size in bytes
#[arg(long, default_value = "4096")]
page_size: u32,
/// Port to serve on
#[arg(short, long, default_value = "3141")]
port: u16,
/// Generate fake events for testing
#[arg(long)]
demo: bool,
}
#[tokio::main]
async fn main() {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| "info".into()),
)
.init();
let args = Args::parse();
let (owner_map, page_count) = if let Some(ref bin_path) = args.bin {
let data = std::fs::read(bin_path).expect("failed to read .bin file");
let pc = data.len() as u64;
tracing::info!("loaded owner map: {pc} pages from {bin_path}");
(data, pc)
} else {
tracing::info!("no --bin provided, starting with empty owner map");
(vec![0xFFu8; 1_000_000], 1_000_000)
};
let (tx, rx) = mpsc::channel();
if args.demo {
let np = page_count as u32;
std::thread::Builder::new()
.name("demo-events".into())
.spawn(move || {
let mut pgno: u32 = 0;
loop {
let mut batch = Vec::with_capacity(256);
for _ in 0..256 {
let p = pgno % np;
batch.push(PageEvent {
pgno: p,
dbi: (p % 8) + 2,
op: match p % 10 {
0..=6 => PageOp::Read,
7..=8 => PageOp::Write,
_ => PageOp::Free,
},
});
pgno = pgno.wrapping_add(37);
}
if tx.send(batch).is_err() {
break;
}
std::thread::sleep(Duration::from_millis(33));
}
})
.expect("failed to spawn demo thread");
}
let config = reth_mdbx_viz::VizConfig {
port: args.port,
page_count,
page_size: args.page_size,
dbi_names: vec![],
tree_info: vec![],
owner_map,
db_path: std::path::PathBuf::new(),
};
if let Err(e) = reth_mdbx_viz::start_viz_server(config, rx).await {
tracing::error!("server error: {e}");
}
}

View File

@@ -0,0 +1,757 @@
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use std::{
collections::HashMap,
io::{self, Read, Write},
os::unix::io::AsRawFd,
path::Path,
sync::atomic::{AtomicU8, AtomicU64, Ordering},
time::Instant,
};
const PAGEHDRSZ: usize = 20;
const NODESIZE: usize = 8;
const P_BRANCH: u16 = 0x01;
const P_LEAF: u16 = 0x02;
const P_LARGE: u16 = 0x04;
const P_META: u16 = 0x08;
const P_DUPFIX: u16 = 0x20;
const N_BIG: u8 = 0x01;
const N_TREE: u8 = 0x02;
const N_DUP: u8 = 0x04;
const P_INVALID: u32 = 0xFFFFFFFF;
const UNREFERENCED: u8 = 0xFF;
const DBI_META: u8 = 0xFE;
const META_GEO: usize = 0x14;
const META_FREE_DB: usize = 0x28;
const META_MAIN_DB: usize = 0x58;
const META_TXNID_A: usize = 0x08;
const META_TXNID_B: usize = 0xB0;
const GEO_NOW: usize = 0x0C;
const TREE_ROOT: usize = 0x08;
const TREE_BRANCH_PAGES: usize = 0x0C;
const TREE_LEAF_PAGES: usize = 0x10;
const TREE_LARGE_PAGES: usize = 0x14;
const TREE_ITEMS: usize = 0x20;
const MDBX_MAGIC: u64 = 0x59659DBDEF4C11;
const CACHE_MAGIC: u64 = 0x5056495A_43414348; // "PVIZC ACH"
const CACHE_VERSION: u32 = 1;
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct TreeInfo {
pub name: String,
pub dbi_index: u8,
pub height: u16,
pub branch_pages: u32,
pub leaf_pages: u32,
pub large_pages: u32,
pub items: u64,
pub root_pgno: u32,
}
pub struct WalkResult {
pub owner_map: Vec<u8>,
pub page_count: usize,
pub page_size: usize,
pub tree_info: Vec<TreeInfo>,
}
#[allow(dead_code)]
struct TreeDescriptor {
flags: u16,
height: u16,
dupfix_size: u32,
root: u32,
branch_pages: u32,
leaf_pages: u32,
large_pages: u32,
sequence: u64,
items: u64,
mod_txnid: u64,
}
struct DBIInfo {
name: String,
dbi_index: u8,
tree: TreeDescriptor,
}
struct MmapFile {
ptr: *mut u8,
len: usize,
}
impl MmapFile {
fn open(path: &Path) -> io::Result<Self> {
let file = std::fs::File::open(path)?;
let len = file.metadata()?.len() as usize;
if len == 0 {
return Err(io::Error::new(io::ErrorKind::InvalidData, "empty file"));
}
let ptr = unsafe {
libc::mmap(
std::ptr::null_mut(),
len,
libc::PROT_READ,
libc::MAP_SHARED,
file.as_raw_fd(),
0,
)
};
if ptr == libc::MAP_FAILED {
return Err(io::Error::last_os_error());
}
Ok(Self { ptr: ptr as *mut u8, len })
}
#[inline]
fn page(&self, pgno: usize, ps: usize) -> Option<&[u8]> {
let off = pgno.checked_mul(ps)?;
if off + ps > self.len {
return None;
}
Some(unsafe { std::slice::from_raw_parts(self.ptr.add(off), ps) })
}
fn evict(&self) {
unsafe {
libc::madvise(self.ptr as *mut libc::c_void, self.len, libc::MADV_DONTNEED);
}
tracing::info!(size_gb = self.len / (1024 * 1024 * 1024), "evicted walker mmap from page cache");
}
}
impl Drop for MmapFile {
fn drop(&mut self) {
unsafe { libc::munmap(self.ptr as *mut libc::c_void, self.len); }
}
}
unsafe impl Send for MmapFile {}
unsafe impl Sync for MmapFile {}
#[inline]
fn u16_le(buf: &[u8], off: usize) -> u16 {
u16::from_le_bytes([buf[off], buf[off + 1]])
}
#[inline]
fn u32_le(buf: &[u8], off: usize) -> u32 {
u32::from_le_bytes([buf[off], buf[off + 1], buf[off + 2], buf[off + 3]])
}
#[inline]
fn u64_le(buf: &[u8], off: usize) -> u64 {
u64::from_le_bytes([
buf[off], buf[off + 1], buf[off + 2], buf[off + 3],
buf[off + 4], buf[off + 5], buf[off + 6], buf[off + 7],
])
}
#[inline]
fn page_flags(buf: &[u8]) -> u16 {
u16_le(buf, 0x0A)
}
#[inline]
fn page_nkeys(buf: &[u8]) -> usize {
u16_le(buf, 0x0C) as usize / 2
}
#[inline]
fn page_overflow_count(buf: &[u8]) -> u32 {
u32_le(buf, 0x0C)
}
fn parse_tree_descriptor(buf: &[u8], off: usize) -> TreeDescriptor {
TreeDescriptor {
flags: u16_le(buf, off),
height: u16_le(buf, off + 0x02),
dupfix_size: u32_le(buf, off + 0x04),
root: u32_le(buf, off + TREE_ROOT),
branch_pages: u32_le(buf, off + TREE_BRANCH_PAGES),
leaf_pages: u32_le(buf, off + TREE_LEAF_PAGES),
large_pages: u32_le(buf, off + TREE_LARGE_PAGES),
sequence: u64_le(buf, off + 0x18),
items: u64_le(buf, off + TREE_ITEMS),
mod_txnid: u64_le(buf, off + 0x28),
}
}
#[inline]
fn claim(owner_map: &[AtomicU8], pgno: usize, dbi_index: u8, conflicts: &AtomicU64) -> bool {
if pgno >= owner_map.len() {
return false;
}
match owner_map[pgno].compare_exchange(UNREFERENCED, dbi_index, Ordering::Relaxed, Ordering::Relaxed) {
Ok(_) => true,
Err(existing) => {
if existing != dbi_index {
conflicts.fetch_add(1, Ordering::Relaxed);
}
false
}
}
}
fn walk_tree_claim(
mmap: &MmapFile,
ps: usize,
root_pgno: u32,
page_count: usize,
dbi_index: u8,
owner_map: &[AtomicU8],
conflicts: &AtomicU64,
) {
if root_pgno == P_INVALID || root_pgno as usize >= page_count {
return;
}
let mut stack: Vec<usize> = vec![root_pgno as usize];
while let Some(pgno) = stack.pop() {
if pgno >= page_count {
continue;
}
if !claim(owner_map, pgno, dbi_index, conflicts) {
continue;
}
let buf = match mmap.page(pgno, ps) {
Some(b) => b,
None => continue,
};
let flags = page_flags(buf);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let node_off = PAGEHDRSZ + node_rel;
if node_off + NODESIZE > ps {
continue;
}
let mn_flags = buf[node_off + 4];
let ksize = u16_le(buf, node_off + 6) as usize;
let data_off = node_off + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > ps {
continue;
}
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= page_count {
continue;
}
let ov_buf = match mmap.page(ov_pgno, ps) {
Some(b) => b,
None => continue,
};
let ov_count = page_overflow_count(ov_buf) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if op < page_count {
claim(owner_map, op, dbi_index, conflicts);
}
}
} else if (mn_flags & (N_DUP | N_TREE)) == (N_DUP | N_TREE) {
if data_off + TREE_ROOT + 4 > ps {
continue;
}
let sub_root = u32_le(buf, data_off + TREE_ROOT);
if sub_root != P_INVALID {
stack.push(sub_root as usize);
}
}
}
} else if flags & P_LARGE != 0 {
let ov_count = page_overflow_count(buf) as usize;
for op in (pgno + 1)..pgno + ov_count {
if op < page_count {
claim(owner_map, op, dbi_index, conflicts);
}
}
}
}
}
fn discover_named_dbis(
mmap: &MmapFile,
ps: usize,
main_root: u32,
page_count: usize,
owner_map: &[AtomicU8],
conflicts: &AtomicU64,
dbi_main: u8,
dbi_start: u8,
) -> Vec<DBIInfo> {
let mut named = Vec::new();
if main_root == P_INVALID || main_root as usize >= page_count {
return named;
}
let mut next_index = dbi_start;
let mut stack: Vec<usize> = vec![main_root as usize];
while let Some(pgno) = stack.pop() {
if pgno >= page_count {
continue;
}
if !claim(owner_map, pgno, dbi_main, conflicts) {
continue;
}
let buf = match mmap.page(pgno, ps) {
Some(b) => b,
None => continue,
};
let flags = page_flags(buf);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let node_off = PAGEHDRSZ + node_rel;
if node_off + NODESIZE > ps {
continue;
}
let mn_flags = buf[node_off + 4];
let ksize = u16_le(buf, node_off + 6) as usize;
let data_off = node_off + NODESIZE + ksize;
if mn_flags & N_TREE != 0 {
let key_off = node_off + NODESIZE;
if key_off + ksize > ps || data_off + 0x30 > ps {
continue;
}
let key_bytes = &buf[key_off..key_off + ksize];
let name = match key_bytes.iter().position(|&b| b == 0) {
Some(nul) => &key_bytes[..nul],
None => key_bytes,
};
let name = String::from_utf8_lossy(name).into_owned();
let tree = parse_tree_descriptor(buf, data_off);
named.push(DBIInfo { name, dbi_index: next_index, tree });
next_index = next_index.wrapping_add(1);
} else if mn_flags & N_BIG != 0 {
if data_off + 4 > ps {
continue;
}
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= page_count {
continue;
}
let ov_buf = match mmap.page(ov_pgno, ps) {
Some(b) => b,
None => continue,
};
let ov_count = page_overflow_count(ov_buf) as usize;
for op in ov_pgno..ov_pgno + ov_count {
claim(owner_map, op, dbi_main, conflicts);
}
}
}
}
}
named
}
fn mark_free_pages(
mmap: &MmapFile,
ps: usize,
free_root: u32,
page_count: usize,
owner_map: &[AtomicU8],
conflicts: &AtomicU64,
dbi_free: u8,
) -> u64 {
let mut marked: u64 = 0;
if free_root == P_INVALID || free_root as usize >= page_count {
return marked;
}
let mut stack: Vec<usize> = vec![free_root as usize];
while let Some(pgno) = stack.pop() {
if pgno >= page_count {
continue;
}
if !claim(owner_map, pgno, dbi_free, conflicts) {
continue;
}
marked += 1;
let buf = match mmap.page(pgno, ps) {
Some(b) => b,
None => continue,
};
let flags = page_flags(buf);
if flags & P_BRANCH != 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let child_pgno = u32_le(buf, PAGEHDRSZ + node_rel) as usize;
stack.push(child_pgno);
}
} else if flags & P_LEAF != 0 && flags & P_DUPFIX == 0 {
let nkeys = page_nkeys(buf);
for i in 0..nkeys {
let node_rel = u16_le(buf, PAGEHDRSZ + i * 2) as usize;
let node_off = PAGEHDRSZ + node_rel;
if node_off + NODESIZE > ps {
continue;
}
let mn_flags = buf[node_off + 4];
let ksize = u16_le(buf, node_off + 6) as usize;
let dsize = u32_le(buf, node_off) as usize;
let data_off = node_off + NODESIZE + ksize;
if mn_flags & N_BIG != 0 {
if data_off + 4 > ps {
continue;
}
let ov_pgno = u32_le(buf, data_off) as usize;
if ov_pgno >= page_count {
continue;
}
let ov_buf = match mmap.page(ov_pgno, ps) {
Some(b) => b,
None => continue,
};
let ov_count = page_overflow_count(ov_buf) as usize;
for op in ov_pgno..ov_pgno + ov_count {
if claim(owner_map, op, dbi_free, conflicts) {
marked += 1;
}
}
let ov_data_off = PAGEHDRSZ;
if dsize >= 4 && ov_data_off + 4 <= ps {
let pnl_count = u32_le(ov_buf, ov_data_off) as usize;
let max_entries = (ps - ov_data_off - 4) / 4;
let pnl_count = pnl_count.min(max_entries);
for j in 0..pnl_count {
let off = ov_data_off + 4 + j * 4;
if off + 4 > ps { break; }
let fp = u32_le(ov_buf, off) as usize;
if fp < page_count {
if claim(owner_map, fp, dbi_free, conflicts) {
marked += 1;
}
}
}
}
} else {
if dsize >= 4 && data_off + 4 <= ps {
let pnl_count = u32_le(buf, data_off) as usize;
let max_entries = (ps - data_off - 4) / 4;
let pnl_count = pnl_count.min(max_entries);
for j in 0..pnl_count {
let off = data_off + 4 + j * 4;
if off + 4 > ps { break; }
let fp = u32_le(buf, off) as usize;
if fp < page_count {
if claim(owner_map, fp, dbi_free, conflicts) {
marked += 1;
}
}
}
}
}
}
}
}
marked
}
fn cache_path(mdbx_path: &Path) -> std::path::PathBuf {
mdbx_path.with_file_name("mdbx_pageviz_cache.bin")
}
fn try_load_cache(
path: &Path,
file_size: u64,
txnid: u64,
page_size: usize,
page_count: usize,
) -> Option<WalkResult> {
let cp = cache_path(path);
let mut f = std::fs::File::open(&cp).ok()?;
let mut hdr = [0u8; 40];
f.read_exact(&mut hdr).ok()?;
let magic = u64::from_le_bytes(hdr[0..8].try_into().unwrap());
let ver = u32::from_le_bytes(hdr[8..12].try_into().unwrap());
let c_txnid = u64::from_le_bytes(hdr[12..20].try_into().unwrap());
let c_pc = u64::from_le_bytes(hdr[20..28].try_into().unwrap());
let c_ps = u32::from_le_bytes(hdr[28..32].try_into().unwrap());
let c_fsz = u64::from_le_bytes(hdr[32..40].try_into().unwrap());
if magic != CACHE_MAGIC || ver != CACHE_VERSION {
return None;
}
if c_txnid != txnid || c_pc as usize != page_count || c_ps as usize != page_size || c_fsz != file_size {
tracing::info!(
cache_txnid = c_txnid, current_txnid = txnid,
"cache stale, will re-walk"
);
return None;
}
let mut ti_len_buf = [0u8; 4];
f.read_exact(&mut ti_len_buf).ok()?;
let ti_len = u32::from_le_bytes(ti_len_buf) as usize;
let mut ti_buf = vec![0u8; ti_len];
f.read_exact(&mut ti_buf).ok()?;
let tree_info: Vec<TreeInfo> = serde_json::from_slice(&ti_buf).ok()?;
let mut owner_map = vec![0u8; page_count];
f.read_exact(&mut owner_map).ok()?;
tracing::info!(page_count, "loaded owner_map from cache");
Some(WalkResult { owner_map, page_count, page_size, tree_info })
}
fn save_cache(
path: &Path,
file_size: u64,
txnid: u64,
result: &WalkResult,
) {
let cp = cache_path(path);
let tmp = cp.with_extension("tmp");
let write_inner = || -> io::Result<()> {
let mut f = std::fs::File::create(&tmp)?;
f.write_all(&CACHE_MAGIC.to_le_bytes())?;
f.write_all(&CACHE_VERSION.to_le_bytes())?;
f.write_all(&txnid.to_le_bytes())?;
f.write_all(&(result.page_count as u64).to_le_bytes())?;
f.write_all(&(result.page_size as u32).to_le_bytes())?;
f.write_all(&file_size.to_le_bytes())?;
let ti_json = serde_json::to_vec(&result.tree_info).map_err(|e| io::Error::new(io::ErrorKind::Other, e))?;
f.write_all(&(ti_json.len() as u32).to_le_bytes())?;
f.write_all(&ti_json)?;
f.write_all(&result.owner_map)?;
f.flush()?;
Ok(())
};
match write_inner() {
Ok(()) => {
if let Err(e) = std::fs::rename(&tmp, &cp) {
tracing::warn!("failed to rename cache file: {e}");
let _ = std::fs::remove_file(&tmp);
} else {
tracing::info!(
path = %cp.display(),
size_mb = result.owner_map.len() / (1024 * 1024),
"saved owner_map cache"
);
}
}
Err(e) => {
tracing::warn!("failed to write cache: {e}");
let _ = std::fs::remove_file(&tmp);
}
}
}
pub fn walk_mdbx(path: &Path, name_to_dbi: &HashMap<&str, u8>) -> eyre::Result<WalkResult> {
let t0 = Instant::now();
let mmap = MmapFile::open(path)?;
let file_size = mmap.len as u64;
let meta0 = mmap.page(0, 4096).ok_or_else(|| eyre::eyre!("cannot read page 0"))?;
let pflags = page_flags(meta0);
eyre::ensure!(pflags & P_META != 0, "page 0 missing P_META flag");
let magic = u64_le(meta0, PAGEHDRSZ);
eyre::ensure!((magic >> 8) == MDBX_MAGIC, "MDBX magic mismatch");
let candidates: &[usize] = &[4096, 8192, 16384, 32768, 65536, 1024, 2048];
let geo_now_raw = u32_le(meta0, PAGEHDRSZ + META_GEO + GEO_NOW) as usize;
let mut ps = 4096usize;
for &candidate in candidates {
let mapped = geo_now_raw * candidate;
if mapped >= mmap.len / 2 && mapped <= mmap.len * 4 {
ps = candidate;
break;
}
}
tracing::info!(page_size = ps, file_size_gb = file_size / (1024 * 1024 * 1024), "detected page size");
let meta_pages: Vec<&[u8]> = (0..3)
.map(|i| mmap.page(i, ps).ok_or_else(|| eyre::eyre!("cannot read meta page {i}")))
.collect::<eyre::Result<Vec<_>>>()?;
let mut best_idx = 0usize;
let mut best_txnid = 0u64;
for (i, meta) in meta_pages.iter().enumerate() {
let m = u64_le(meta, PAGEHDRSZ);
if (m >> 8) != MDBX_MAGIC {
continue;
}
let txnid_a = u64_le(meta, PAGEHDRSZ + META_TXNID_A);
let txnid_b = u64_le(meta, PAGEHDRSZ + META_TXNID_B);
if txnid_a == txnid_b && txnid_a > best_txnid {
best_txnid = txnid_a;
best_idx = i;
}
}
eyre::ensure!(best_txnid > 0, "no valid meta page found");
let best = meta_pages[best_idx];
let mb = PAGEHDRSZ;
let geo_now = u32_le(best, mb + META_GEO + GEO_NOW) as usize;
let page_count = geo_now;
tracing::info!(best_meta = best_idx, txnid = best_txnid, page_count, "selected best meta");
if let Some(cached) = try_load_cache(path, file_size, best_txnid, ps, page_count) {
let elapsed = t0.elapsed();
tracing::info!(elapsed_ms = elapsed.as_millis() as u64, "walk skipped (cache hit)");
return Ok(cached);
}
let free_tree = parse_tree_descriptor(best, mb + META_FREE_DB);
let main_tree = parse_tree_descriptor(best, mb + META_MAIN_DB);
tracing::info!(
page_count,
free_root = free_tree.root,
main_root = main_tree.root,
"parsed meta, starting walk"
);
let owner_map: Vec<AtomicU8> = (0..page_count).map(|_| AtomicU8::new(UNREFERENCED)).collect();
let conflicts = AtomicU64::new(0);
for pgno in 0..3.min(page_count) {
owner_map[pgno].store(DBI_META, Ordering::Relaxed);
}
let dbi_free: u8 = 1;
let dbi_main: u8 = 2;
let dbi_start: u8 = 3;
tracing::info!("discovering named DBIs via MainDB walk");
let mut named_dbis =
discover_named_dbis(&mmap, ps, main_tree.root, page_count, &owner_map, &conflicts, dbi_main, dbi_start);
tracing::info!(count = named_dbis.len(), "found named DBIs");
for dbi in &mut named_dbis {
if let Some(&mapped_idx) = name_to_dbi.get(dbi.name.as_str()) {
dbi.dbi_index = mapped_idx;
}
}
tracing::info!("walking FreeDB");
let free_marked =
mark_free_pages(&mmap, ps, free_tree.root, page_count, &owner_map, &conflicts, dbi_free);
tracing::info!(marked = free_marked, "FreeDB walk complete");
let walk_tasks: Vec<(u32, u8)> = named_dbis
.iter()
.filter(|d| d.tree.root != P_INVALID)
.map(|d| (d.tree.root, d.dbi_index))
.collect();
tracing::info!(count = walk_tasks.len(), "walking named DBIs in parallel");
walk_tasks
.par_iter()
.for_each(|&(root, dbi_idx)| {
walk_tree_claim(&mmap, ps, root, page_count, dbi_idx, &owner_map, &conflicts);
});
let final_conflicts = conflicts.load(Ordering::Relaxed);
let owner_map_vec: Vec<u8> = owner_map.iter().map(|a| a.load(Ordering::Relaxed)).collect();
let assigned = owner_map_vec.iter().filter(|&&b| b != UNREFERENCED).count();
let elapsed = t0.elapsed();
tracing::info!(
elapsed_ms = elapsed.as_millis() as u64,
assigned,
unreferenced = page_count - assigned,
conflicts = final_conflicts,
"walk complete"
);
let mut tree_info = Vec::new();
tree_info.push(TreeInfo {
name: "FreeDB".to_string(),
dbi_index: dbi_free,
height: free_tree.height,
branch_pages: free_tree.branch_pages,
leaf_pages: free_tree.leaf_pages,
large_pages: free_tree.large_pages,
items: free_tree.items,
root_pgno: free_tree.root,
});
tree_info.push(TreeInfo {
name: "MainDB".to_string(),
dbi_index: dbi_main,
height: main_tree.height,
branch_pages: main_tree.branch_pages,
leaf_pages: main_tree.leaf_pages,
large_pages: main_tree.large_pages,
items: main_tree.items,
root_pgno: main_tree.root,
});
for dbi in &named_dbis {
tree_info.push(TreeInfo {
name: dbi.name.clone(),
dbi_index: dbi.dbi_index,
height: dbi.tree.height,
branch_pages: dbi.tree.branch_pages,
leaf_pages: dbi.tree.leaf_pages,
large_pages: dbi.tree.large_pages,
items: dbi.tree.items,
root_pgno: dbi.tree.root,
});
}
let result = WalkResult { owner_map: owner_map_vec, page_count, page_size: ps, tree_info };
save_cache(path, file_size, best_txnid, &result);
mmap.evict();
Ok(result)
}