diff --git a/Cargo.lock b/Cargo.lock index ab30c41bee..a17d38d149 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -387,6 +387,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.64.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4243e6031260db77ede97ad86c27e501d646a27ab57b59a574f725d98ab1fb4" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2 1.0.60", + "quote 1.0.28", + "regex", + "rustc-hash", + "shlex", + "syn 1.0.109", +] + [[package]] name = "bindgen" version = "0.65.1" @@ -3157,6 +3177,17 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" +[[package]] +name = "jemalloc-ctl" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1891c671f3db85d8ea8525dd43ab147f9977041911d24a03e5a36187a7bfde9" +dependencies = [ + "jemalloc-sys", + "libc", + "paste", +] + [[package]] name = "jemalloc-sys" version = "0.5.3+5.3.0-patched" @@ -3431,6 +3462,17 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "libproc" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b18cbf29f8ff3542ba22bdce9ac610fcb75d74bb4e2b306b2a2762242025b4f" +dependencies = [ + "bindgen 0.64.0", + "errno 0.2.8", + "libc", +] + [[package]] name = "lifetimed-bytes" version = "0.1.0" @@ -3537,6 +3579,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "match_cfg" version = "0.1.0" @@ -3623,6 +3674,21 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "metrics-process" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99eab79be9f7c18565e889d6eaed6f1ebdafb2b6a88aef446d2fee5e7796ed10" +dependencies = [ + "libproc", + "mach2", + "metrics", + "once_cell", + "procfs", + "rlimit", + "windows", +] + [[package]] name = "metrics-util" version = "0.14.0" @@ -4478,6 +4544,19 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "procfs" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ca7f9f29bab5844ecd8fdb3992c5969b6622bb9609b9502fef9b4310e3f1f" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "hex", + "lazy_static", + "rustix 0.36.11", +] + [[package]] name = "proptest" version = "1.1.0" @@ -4851,8 +4930,10 @@ dependencies = [ "human_bytes", "humantime", "hyper", + "jemalloc-ctl", "jemallocator", "metrics-exporter-prometheus", + "metrics-process", "metrics-util", "pin-project", "pretty_assertions", @@ -5264,7 +5345,7 @@ dependencies = [ name = "reth-mdbx-sys" version = "0.1.0-alpha.1" dependencies = [ - "bindgen", + "bindgen 0.65.1", "cc", "libc", ] @@ -5910,6 +5991,15 @@ dependencies = [ "digest 0.10.6", ] +[[package]] +name = "rlimit" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a29d87a652dc4d43c586328706bb5cdff211f3f39a530f240b53f7221dab8e" +dependencies = [ + "libc", +] + [[package]] name = "rlp" version = "0.5.2" @@ -7819,6 +7909,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets 0.48.0", +] + [[package]] name = "windows-sys" version = "0.42.0" diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 29470c9d1a..d9955fa891 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -38,6 +38,7 @@ reth-basic-payload-builder = { path = "../../crates/payload/basic" } reth-discv4 = { path = "../../crates/net/discv4" } reth-metrics = { workspace = true } jemallocator = { version = "0.5.0", optional = true } +jemalloc-ctl = { version = "0.5.0", optional = true } # crypto secp256k1 = { workspace = true, features = ["global-context", "rand-std", "recovery"] } @@ -57,6 +58,7 @@ toml = { version = "0.7", features = ["display"] } # metrics metrics-exporter-prometheus = "0.11.0" metrics-util = "0.14.0" +metrics-process = "1.0.9" # test vectors generation proptest = "1.0" @@ -86,7 +88,7 @@ pretty_assertions = "1.3.0" humantime = "2.1.0" [features] -jemalloc = ["dep:jemallocator"] +jemalloc = ["dep:jemallocator", "dep:jemalloc-ctl"] jemalloc-prof = ["jemalloc", "jemallocator?/profiling"] min-error-logs = ["tracing/release_max_level_error"] min-warn-logs = ["tracing/release_max_level_warn"] diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 02c160f3ae..d7436e1525 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -481,8 +481,8 @@ impl Command { async fn start_metrics_endpoint(&self, db: Arc>) -> eyre::Result<()> { if let Some(listen_addr) = self.metrics { info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); - - prometheus_exporter::initialize_with_db_metrics(listen_addr, db).await?; + prometheus_exporter::initialize(listen_addr, db, metrics_process::Collector::default()) + .await?; } Ok(()) diff --git a/bin/reth/src/prometheus_exporter.rs b/bin/reth/src/prometheus_exporter.rs index 0c9c3bb481..2dba64b12b 100644 --- a/bin/reth/src/prometheus_exporter.rs +++ b/bin/reth/src/prometheus_exporter.rs @@ -14,19 +14,24 @@ use reth_db::{ use reth_metrics::metrics::{self, absolute_counter, describe_counter, Unit}; use std::{convert::Infallible, net::SocketAddr, sync::Arc}; -/// Installs Prometheus as the metrics recorder and serves it over HTTP with a hook. +pub(crate) trait Hook: Fn() + Send + Sync {} +impl Hook for T {} + +/// Installs Prometheus as the metrics recorder and serves it over HTTP with hooks. /// -/// The hook is called every time the metrics are requested at the given endpoint, and can be used +/// The hooks are called every time the metrics are requested at the given endpoint, and can be used /// to record values for pull-style metrics, i.e. metrics that are not automatically updated. -pub(crate) async fn initialize_with_hook( +pub(crate) async fn initialize_with_hooks( listen_addr: SocketAddr, - hook: F, + hooks: impl IntoIterator, ) -> eyre::Result<()> { let recorder = PrometheusBuilder::new().build_recorder(); let handle = recorder.handle(); + let hooks: Vec<_> = hooks.into_iter().collect(); + // Start endpoint - start_endpoint(listen_addr, handle, Arc::new(hook)) + start_endpoint(listen_addr, handle, Arc::new(move || hooks.iter().for_each(|hook| hook()))) .await .wrap_err("Could not start Prometheus endpoint")?; @@ -40,7 +45,7 @@ pub(crate) async fn initialize_with_hook( } /// Starts an endpoint at the given address to serve Prometheus metrics. -async fn start_endpoint( +async fn start_endpoint( listen_addr: SocketAddr, handle: PrometheusHandle, hook: Arc, @@ -64,14 +69,16 @@ async fn start_endpoint( Ok(()) } -/// Installs Prometheus as the metrics recorder and serves it over HTTP with database metrics. -pub(crate) async fn initialize_with_db_metrics( +/// Installs Prometheus as the metrics recorder and serves it over HTTP with database and process +/// metrics. +pub(crate) async fn initialize( listen_addr: SocketAddr, db: Arc>, + process: metrics_process::Collector, ) -> eyre::Result<()> { let db_stats = move || { // TODO: A generic stats abstraction for other DB types to deduplicate this and `reth db - // stats` + // stats` let _ = db.view(|tx| { for table in tables::Tables::ALL.iter().map(|table| table.name()) { let table_db = @@ -99,12 +106,112 @@ pub(crate) async fn initialize_with_db_metrics( }); }; - initialize_with_hook(listen_addr, db_stats).await?; + // Clone `process` to move it into the hook and use the original `process` for describe below. + let cloned_process = process.clone(); + let hooks: Vec>> = vec![ + Box::new(db_stats), + Box::new(move || cloned_process.collect()), + Box::new(collect_memory_stats), + ]; + initialize_with_hooks(listen_addr, hooks).await?; // We describe the metrics after the recorder is installed, otherwise this information is not // registered describe_counter!("db.table_size", Unit::Bytes, "The size of a database table (in bytes)"); describe_counter!("db.table_pages", "The number of database pages for a table"); + process.describe(); + describe_memory_stats(); Ok(()) } + +#[cfg(feature = "jemalloc")] +fn collect_memory_stats() { + use jemalloc_ctl::{epoch, stats}; + use reth_metrics::metrics::gauge; + use tracing::error; + + if epoch::advance().map_err(|error| error!(?error, "Failed to advance jemalloc epoch")).is_err() + { + return + } + + if let Ok(value) = stats::active::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.active")) + { + gauge!("jemalloc.active", value as f64); + } + + if let Ok(value) = stats::allocated::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.allocated")) + { + gauge!("jemalloc.allocated", value as f64); + } + + if let Ok(value) = stats::mapped::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.mapped")) + { + gauge!("jemalloc.mapped", value as f64); + } + + if let Ok(value) = stats::metadata::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.metadata")) + { + gauge!("jemalloc.metadata", value as f64); + } + + if let Ok(value) = stats::resident::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.resident")) + { + gauge!("jemalloc.resident", value as f64); + } + + if let Ok(value) = stats::retained::read() + .map_err(|error| error!(?error, "Failed to read jemalloc.stats.retained")) + { + gauge!("jemalloc.retained", value as f64); + } +} + +#[cfg(feature = "jemalloc")] +fn describe_memory_stats() { + use reth_metrics::metrics::describe_gauge; + + describe_gauge!( + "jemalloc.active", + Unit::Bytes, + "Total number of bytes in active pages allocated by the application" + ); + describe_gauge!( + "jemalloc.allocated", + Unit::Bytes, + "Total number of bytes allocated by the application" + ); + describe_gauge!( + "jemalloc.mapped", + Unit::Bytes, + "Total number of bytes in active extents mapped by the allocator" + ); + describe_gauge!( + "jemalloc.metadata", + Unit::Bytes, + "Total number of bytes dedicated to jemalloc metadata" + ); + describe_gauge!( + "jemalloc.resident", + Unit::Bytes, + "Total number of bytes in physically resident data pages mapped by the allocator" + ); + describe_gauge!( + "jemalloc.retained", + Unit::Bytes, + "Total number of bytes in virtual memory mappings that were retained rather than \ + being returned to the operating system via e.g. munmap(2)" + ); +} + +#[cfg(not(feature = "jemalloc"))] +fn collect_memory_stats() {} + +#[cfg(not(feature = "jemalloc"))] +fn describe_memory_stats() {} diff --git a/bin/reth/src/stage/run.rs b/bin/reth/src/stage/run.rs index 1150b20f9c..02d90514ef 100644 --- a/bin/reth/src/stage/run.rs +++ b/bin/reth/src/stage/run.rs @@ -127,7 +127,12 @@ impl Command { if let Some(listen_addr) = self.metrics { info!(target: "reth::cli", "Starting metrics endpoint at {}", listen_addr); - prometheus_exporter::initialize_with_db_metrics(listen_addr, Arc::clone(&db)).await?; + prometheus_exporter::initialize( + listen_addr, + Arc::clone(&db), + metrics_process::Collector::default(), + ) + .await?; } let batch_size = self.batch_size.unwrap_or(self.to - self.from + 1);