github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-01-08 23:08:19 -05:00
Code Issues Packages Projects Releases Wiki Activity

feat(metrics): configurable jeprof pprof dumps directory (#20834)

Browse Source
This commit is contained in:
Alexey Shekhirin
2026-01-08 11:21:42 +00:00
committed by GitHub
parent 0f0a181fe2
commit bcd74d021b
40 changed files with 176 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@@ -9551,15 +9551,19 @@ dependencies = [
"http-body-util",
"jemalloc_pprof",
"jsonrpsee-server",
"mappings",
"metrics",
"metrics-exporter-prometheus",
"metrics-process",
"metrics-util",
"pprof_util",
"procfs 0.17.0",
"reqwest",
"reth-fs-util",
"reth-metrics",
"reth-tasks",
"socket2 0.5.10",
"tempfile",
"tikv-jemalloc-ctl",
"tokio",
"tower",

2
crates/cli/commands/src/db/mod.rs
View File

@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone())?;
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

8
crates/cli/commands/src/db/repair_trie.rs
View File

@@ -9,7 +9,10 @@ use reth_db_api::{
transaction::{DbTx, DbTxMut},
};
use reth_db_common::DbTool;
use reth_node_core::version::version_metadata;
use reth_node_core::{
dirs::{ChainPath, DataDirPath},
version::version_metadata,
};
use reth_node_metrics::{
chain::ChainSpecInfo,
hooks::Hooks,
@@ -53,11 +56,13 @@ impl Command {
self,
tool: &DbTool<N>,
task_executor: TaskExecutor,
data_dir: &ChainPath<DataDirPath>,
) -> eyre::Result<()> {
// Set up metrics server if requested
let _metrics_handle = if let Some(listen_addr) = self.metrics {
let chain_name = tool.provider_factory.chain_spec().chain().to_string();
let executor = task_executor.clone();
let pprof_dump_dir = data_dir.pprof_dumps();
let handle = task_executor.spawn_critical("metrics server", async move {
let config = MetricServerConfig::new(
@@ -73,6 +78,7 @@ impl Command {
ChainSpecInfo { name: chain_name },
executor,
Hooks::builder().build(),
pprof_dump_dir,
);
// Spawn the metrics server

1
crates/cli/commands/src/stage/run.rs
View File

@@ -153,6 +153,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
}
})
.build(),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;

1
crates/ethereum/node/tests/it/testing.rs
View File

@@ -29,6 +29,7 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> {
.expect("valid datadir"),
static_files_path: Some(tempdir.path().join("static")),
rocksdb_path: Some(tempdir.path().join("rocksdb")),
pprof_dumps_path: Some(tempdir.path().join("pprof")),
};
let config = NodeConfig::test().with_datadir_args(datadir_args).with_rpc(rpc_args);
let db = create_test_rw_db();

1
crates/node/builder/src/launch/common.rs
View File

@@ -664,6 +664,7 @@ where
}
})
.build(),
self.data_dir().pprof_dumps(),
)
.with_push_gateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval);

4
crates/node/core/src/args/datadir_args.rs
View File

@@ -31,6 +31,10 @@ pub struct DatadirArgs {
/// The absolute path to store `RocksDB` database in.
#[arg(long = "datadir.rocksdb", value_name = "PATH", verbatim_doc_comment)]
pub rocksdb_path: Option<PathBuf>,
/// The absolute path to store pprof dumps in.
#[arg(long = "datadir.pprof-dumps", value_name = "PATH", verbatim_doc_comment)]
pub pprof_dumps_path: Option<PathBuf>,
}
impl DatadirArgs {

12
crates/node/core/src/dirs.rs
View File

@@ -313,6 +313,18 @@ impl<D> ChainPath<D> {
}
}
/// Returns the path to the directory for storing pprof dumps for this chain.
///
/// `<DIR>/<CHAIN_ID>/pprof`
pub fn pprof_dumps(&self) -> PathBuf {
let datadir_args = &self.2;
if let Some(pprof_dumps_path) = &datadir_args.pprof_dumps_path {
pprof_dumps_path.clone()
} else {
self.data_dir().join("pprof")
}
}
/// Returns the path to the reth p2p secret key for this chain.
///
/// `<DIR>/<CHAIN_ID>/discovery-secret`

8
crates/node/metrics/Cargo.toml
View File

@@ -29,8 +29,12 @@ tracing.workspace = true
eyre.workspace = true
[target.'cfg(unix)'.dependencies]
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
jemalloc_pprof = { workspace = true, optional = true }
mappings = { version = "0.7", optional = true }
pprof_util = { version = "0.8", optional = true }
reth-fs-util = { workspace = true, optional = true }
tempfile = { workspace = true, optional = true }
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = "0.17.0"
@@ -44,5 +48,5 @@ workspace = true
[features]
jemalloc = ["dep:tikv-jemalloc-ctl"]
jemalloc-prof = ["jemalloc", "dep:jemalloc_pprof"]
jemalloc-prof = ["jemalloc", "dep:jemalloc_pprof", "dep:mappings", "dep:pprof_util", "dep:reth-fs-util", "dep:tempfile"]
jemalloc-symbols = ["jemalloc-prof", "jemalloc_pprof?/symbolize"]

57
crates/node/metrics/src/server.rs
View File

@@ -13,7 +13,7 @@ use metrics_process::Collector;
use reqwest::Client;
use reth_metrics::metrics::Unit;
use reth_tasks::TaskExecutor;
use std::{convert::Infallible, net::SocketAddr, sync::Arc, time::Duration};
use std::{convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc, time::Duration};
/// Configuration for the [`MetricServer`]
#[derive(Debug)]
@@ -25,6 +25,7 @@ pub struct MetricServerConfig {
hooks: Hooks,
push_gateway_url: Option<String>,
push_gateway_interval: Duration,
pprof_dump_dir: PathBuf,
}
impl MetricServerConfig {
@@ -35,6 +36,7 @@ impl MetricServerConfig {
chain_spec_info: ChainSpecInfo,
task_executor: TaskExecutor,
hooks: Hooks,
pprof_dump_dir: PathBuf,
) -> Self {
Self {
listen_addr,
@@ -44,6 +46,7 @@ impl MetricServerConfig {
chain_spec_info,
push_gateway_url: None,
push_gateway_interval: Duration::from_secs(5),
pprof_dump_dir,
}
}
@@ -77,6 +80,7 @@ impl MetricServer {
chain_spec_info,
push_gateway_url,
push_gateway_interval,
pprof_dump_dir,
} = &self.config;
let hooks_for_endpoint = hooks.clone();
@@ -84,6 +88,7 @@ impl MetricServer {
*listen_addr,
Arc::new(move || hooks_for_endpoint.iter().for_each(|hook| hook())),
task_executor.clone(),
pprof_dump_dir.clone(),
)
.await
.wrap_err_with(|| format!("Could not start Prometheus endpoint at {listen_addr}"))?;
@@ -116,6 +121,7 @@ impl MetricServer {
listen_addr: SocketAddr,
hook: Arc<F>,
task_executor: TaskExecutor,
pprof_dump_dir: PathBuf,
) -> eyre::Result<()> {
let listener = tokio::net::TcpListener::bind(listen_addr)
.await
@@ -141,8 +147,10 @@ impl MetricServer {
let handle = install_prometheus_recorder();
let hook = hook.clone();
let pprof_dump_dir = pprof_dump_dir.clone();
let service = tower::service_fn(move |req: Request<_>| {
let response = handle_request(req.uri().path(), &*hook, handle);
let response =
handle_request(req.uri().path(), &*hook, handle, &pprof_dump_dir);
async move { Ok::<_, Infallible>(response) }
});
@@ -288,9 +296,10 @@ fn handle_request(
path: &str,
hook: impl Fn(),
handle: &crate::recorder::PrometheusRecorder,
pprof_dump_dir: &PathBuf,
) -> Response<Full<Bytes>> {
match path {
"/debug/pprof/heap" => handle_pprof_heap(),
"/debug/pprof/heap" => handle_pprof_heap(pprof_dump_dir),
_ => {
hook();
let metrics = handle.handle().render();
@@ -302,12 +311,12 @@ fn handle_request(
}
#[cfg(all(feature = "jemalloc-prof", unix))]
fn handle_pprof_heap() -> Response<Full<Bytes>> {
fn handle_pprof_heap(pprof_dump_dir: &PathBuf) -> Response<Full<Bytes>> {
use http::header::CONTENT_ENCODING;
match jemalloc_pprof::PROF_CTL.as_ref() {
Some(prof_ctl) => match prof_ctl.try_lock() {
Ok(mut ctl) => match ctl.dump_pprof() {
Ok(_) => match jemalloc_pprof_dump(pprof_dump_dir) {
Ok(pprof) => {
let mut response = Response::new(Full::new(Bytes::from(pprof)));
response
@@ -345,8 +354,34 @@ fn handle_pprof_heap() -> Response<Full<Bytes>> {
}
}
/// Equivalent to [`jemalloc_pprof::JemallocProfCtl::dump`], but accepts a directory that the
/// temporary pprof file will be written to. The file is deleted when the function exits.
#[cfg(all(feature = "jemalloc-prof", unix))]
fn jemalloc_pprof_dump(pprof_dump_dir: &PathBuf) -> eyre::Result<Vec<u8>> {
use std::{ffi::CString, io::BufReader};
use mappings::MAPPINGS;
use pprof_util::parse_jeheap;
use tempfile::NamedTempFile;
reth_fs_util::create_dir_all(pprof_dump_dir)?;
let f = NamedTempFile::new_in(pprof_dump_dir)?;
let path = CString::new(f.path().as_os_str().as_encoded_bytes()).unwrap();
// SAFETY: "prof.dump" is documented as being writable and taking a C string as input:
// http://jemalloc.net/jemalloc.3.html#prof.dump
unsafe { tikv_jemalloc_ctl::raw::write(b"prof.dump\0", path.as_ptr()) }?;
let dump_reader = BufReader::new(f);
let profile =
parse_jeheap(dump_reader, MAPPINGS.as_deref()).map_err(|err| eyre::eyre!(Box::new(err)))?;
let pprof = profile.to_pprof(("inuse_space", "bytes"), ("space", "bytes"), None);
Ok(pprof)
}
#[cfg(not(all(feature = "jemalloc-prof", unix)))]
fn handle_pprof_heap() -> Response<Full<Bytes>> {
fn handle_pprof_heap(_pprof_dump_dir: &PathBuf) -> Response<Full<Bytes>> {
let mut response = Response::new(Full::new(Bytes::from_static(
b"jemalloc pprof support not compiled. Rebuild with the jemalloc-prof feature.",
)));
@@ -390,8 +425,14 @@ mod tests {
let hooks = Hooks::builder().build();
let listen_addr = get_random_available_addr();
let config =
MetricServerConfig::new(listen_addr, version_info, chain_spec_info, executor, hooks);
let config = MetricServerConfig::new(
listen_addr,
version_info,
chain_spec_info,
executor,
hooks,
std::env::temp_dir(),
);
MetricServer::new(config).serve().await.unwrap();

3
docs/vocs/docs/pages/cli/op-reth/db.mdx
View File

@@ -46,6 +46,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/import-op.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/import-receipts-op.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/init-state.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/init.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/node.mdx
View File

@@ -74,6 +74,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
Networking:
-d, --disable-discovery
Disable the discovery service

3
docs/vocs/docs/pages/cli/op-reth/p2p/body.mdx
View File

@@ -251,6 +251,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use.

3
docs/vocs/docs/pages/cli/op-reth/p2p/header.mdx
View File

@@ -251,6 +251,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use.

3
docs/vocs/docs/pages/cli/op-reth/prune.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/re-execute.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/stage/drop.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/stage/dump.mdx
View File

@@ -37,6 +37,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/stage/run.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/op-reth/stage/unwind.mdx
View File

@@ -35,6 +35,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/db.mdx
View File

@@ -46,6 +46,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/download.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/export-era.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/import-era.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/import.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/init-state.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/init.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/node.mdx
View File

@@ -74,6 +74,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
Networking:
-d, --disable-discovery
Disable the discovery service

3
docs/vocs/docs/pages/cli/reth/p2p/body.mdx
View File

@@ -251,6 +251,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use.

3
docs/vocs/docs/pages/cli/reth/p2p/header.mdx
View File

@@ -251,6 +251,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use.

3
docs/vocs/docs/pages/cli/reth/prune.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/re-execute.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/stage/drop.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/stage/dump.mdx
View File

@@ -37,6 +37,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/stage/run.mdx
View File

@@ -30,6 +30,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use

3
docs/vocs/docs/pages/cli/reth/stage/unwind.mdx
View File

@@ -35,6 +35,9 @@ Datadir:
--datadir.rocksdb <PATH>
The absolute path to store `RocksDB` database in.
--datadir.pprof-dumps <PATH>
The absolute path to store pprof dumps in.
--config <FILE>
The path to the configuration file to use