From 4094d677e45eb6931a16df05f2d3cf49666fba79 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 25 Mar 2026 20:08:31 +0530 Subject: [PATCH] feat: enable jemalloc `override_allocator_on_supported_platforms` (#23214) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- Cargo.lock | 1 + Cargo.toml | 1 + Makefile | 2 +- bin/reth-bench/Cargo.toml | 2 +- bin/reth-bench/src/main.rs | 3 +++ bin/reth/Cargo.toml | 8 +------- bin/reth/src/lib.rs | 1 - bin/reth/src/main.rs | 6 +++++- crates/cli/util/Cargo.toml | 10 ++++++---- crates/cli/util/src/allocator.rs | 5 +++++ docs/vocs/docs/pages/run/faq/profiling.mdx | 18 ++++++++++-------- 11 files changed, 34 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 98b63159b8..c4578f94f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7825,6 +7825,7 @@ dependencies = [ "serde", "snmalloc-rs", "thiserror 2.0.18", + "tikv-jemalloc-sys", "tikv-jemallocator", "tracy-client", ] diff --git a/Cargo.toml b/Cargo.toml index c4b8c3cd91..2315ab9621 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -645,6 +645,7 @@ ethereum_ssz_derive = "0.10.1" # allocators jemalloc_pprof = { version = "0.8", default-features = false } tikv-jemalloc-ctl = "0.6" +tikv-jemalloc-sys = "0.6" tikv-jemallocator = "0.6" tracy-client = { version = "0.18.0", features = ["demangle"] } snmalloc-rs = { version = "0.3.7", features = ["build_cc"] } diff --git a/Makefile b/Makefile index c1c3df611b..fd04a20d35 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ build-%-reproducible: LC_ALL=C \ TZ=UTC \ JEMALLOC_OVERRIDE=/usr/lib/x86_64-linux-gnu/libjemalloc.a \ - cargo build --bin reth --features "$(FEATURES) jemalloc-unprefixed" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu + cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu .PHONY: build-debug build-debug: ## Build the reth binary into `target/debug` directory. diff --git a/bin/reth-bench/Cargo.toml b/bin/reth-bench/Cargo.toml index 9c501e8d1e..abcc47f625 100644 --- a/bin/reth-bench/Cargo.toml +++ b/bin/reth-bench/Cargo.toml @@ -86,7 +86,7 @@ jemalloc = [ "reth-cli-util/jemalloc", "reth-node-core/jemalloc", ] -jemalloc-prof = ["reth-cli-util/jemalloc-prof"] +jemalloc-prof = ["jemalloc", "reth-cli-util/jemalloc-prof"] tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"] tracy = [ "reth-node-core/tracy", diff --git a/bin/reth-bench/src/main.rs b/bin/reth-bench/src/main.rs index ee98115ba7..608007e086 100644 --- a/bin/reth-bench/src/main.rs +++ b/bin/reth-bench/src/main.rs @@ -14,6 +14,9 @@ #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); +#[cfg(all(feature = "jemalloc", unix))] +use reth_cli_util::allocator::tikv_jemalloc_sys as _; + pub mod authenticated_transport; pub mod bench; pub mod bench_mode; diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index d49ffc0e8e..3af90862a7 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -127,7 +127,7 @@ jemalloc = [ "reth-provider/jemalloc", ] jemalloc-prof = [ - "reth-cli-util/jemalloc", + "jemalloc", "reth-cli-util/jemalloc-prof", "reth-ethereum-cli/jemalloc-prof", "reth-node-metrics/jemalloc-prof", @@ -136,12 +136,6 @@ jemalloc-symbols = [ "jemalloc-prof", "reth-ethereum-cli/jemalloc-symbols", ] -jemalloc-unprefixed = [ - "reth-cli-util/jemalloc-unprefixed", - "reth-node-core/jemalloc", - "reth-node-metrics/jemalloc", - "reth-ethereum-cli/jemalloc", -] tracy-allocator = [ "reth-cli-util/tracy-allocator", "reth-ethereum-cli/tracy-allocator", diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 7d7bf43cce..f8a4be0424 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -22,7 +22,6 @@ //! and leak detection functionality. See [jemalloc's opt.prof](https://jemalloc.net/jemalloc.3.html#opt.prof) //! documentation for usage details. This is **not recommended on Windows**. //! - `jemalloc-symbols`: Enables jemalloc symbols for profiling. Includes `jemalloc-prof`. -//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols. //! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator //! integration for memory profiling. //! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator. diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 597a0fa597..838f4e1dc6 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -3,8 +3,12 @@ #[global_allocator] static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator(); +// Required for "override_allocator_on_supported_platforms". +#[cfg(all(feature = "jemalloc", unix))] +use reth_cli_util::allocator::tikv_jemalloc_sys as _; + #[cfg(all(feature = "jemalloc-prof", unix))] -#[unsafe(export_name = "_rjem_malloc_conf")] +#[unsafe(export_name = "malloc_conf")] static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0"; use clap::Parser; diff --git a/crates/cli/util/Cargo.toml b/crates/cli/util/Cargo.toml index 3e7fbed45b..d215716450 100644 --- a/crates/cli/util/Cargo.toml +++ b/crates/cli/util/Cargo.toml @@ -33,19 +33,21 @@ reth-tracing = { workspace = true, optional = true } rand.workspace = true [target.'cfg(unix)'.dependencies] +tikv-jemalloc-sys = { workspace = true, optional = true } tikv-jemallocator = { workspace = true, optional = true } snmalloc-rs = { workspace = true, optional = true } libc = "0.2" [features] -jemalloc = ["dep:tikv-jemallocator"] +jemalloc = [ + "dep:tikv-jemallocator", + "dep:tikv-jemalloc-sys", + "tikv-jemallocator?/override_allocator_on_supported_platforms", +] # Enables jemalloc profiling features jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"] -# Enables unprefixed malloc (reproducible builds support) -jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"] - # Wraps the selected allocator in the tracy profiling allocator tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"] diff --git a/crates/cli/util/src/allocator.rs b/crates/cli/util/src/allocator.rs index 004404a0e8..7541d64908 100644 --- a/crates/cli/util/src/allocator.rs +++ b/crates/cli/util/src/allocator.rs @@ -15,6 +15,11 @@ cfg_if::cfg_if! { } } +// Re-export jemalloc-sys so that binaries can `use` it in main.rs to make it +// visible to the linker, which is required for `override_allocator_on_supported_platforms`. +#[cfg(all(feature = "jemalloc", unix))] +pub use tikv_jemalloc_sys; + // This is to prevent clippy unused warnings when we do `--all-features` cfg_if::cfg_if! { if #[cfg(all(feature = "snmalloc", feature = "jemalloc", unix))] { diff --git a/docs/vocs/docs/pages/run/faq/profiling.mdx b/docs/vocs/docs/pages/run/faq/profiling.mdx index 4b8906922c..908baa2e4d 100644 --- a/docs/vocs/docs/pages/run/faq/profiling.mdx +++ b/docs/vocs/docs/pages/run/faq/profiling.mdx @@ -139,23 +139,25 @@ cgexec -g memory:rethMemory reth node ### Understanding allocation with jeprof -When reth is built with the `jemalloc-prof` feature and debug symbols, the profiling still needs to be configured and enabled at runtime. This is done with the `_RJEM_MALLOC_CONF` environment variable. Take the following -command to launch reth with jemalloc profiling enabled: +When reth is built with the `jemalloc-prof` feature, profiling is already enabled at compile time — the binary embeds `prof:true,prof_active:true,lg_prof_sample:19` via a `malloc_conf` static. No environment variable is needed to turn on profiling. + +To configure additional options like `lg_prof_interval` at runtime, use the `MALLOC_CONF` environment variable: ``` -_RJEM_MALLOC_CONF=prof:true,lg_prof_interval:32,lg_prof_sample:19 reth node +MALLOC_CONF=lg_prof_interval:32 reth node ``` -If reth is not built properly, you will see this when you try to run reth: +If you see errors like this: ``` -~/p/reth (dan/managing-memory)> _RJEM_MALLOC_CONF=prof:true,lg_prof_interval:32,lg_prof_sample:19 reth node : Invalid conf pair: prof:true -: Invalid conf pair: lg_prof_interval:32 -: Invalid conf pair: lg_prof_sample:19 ``` -If this happens, jemalloc likely needs to be rebuilt with the `jemalloc-prof` feature enabled. +It means reth was not built with the `jemalloc-prof` feature, so jemalloc does not have profiling support compiled in. Rebuild with: + +``` +cargo build --features jemalloc-prof +``` If everything is working, this will output `jeprof.*.heap` files while reth is running. [The jemalloc website](https://jemalloc.net/jemalloc.3.html#opt.abort) has a helpful overview of the options available, for example `lg_prof_interval`, `lg_prof_sample`, `prof_leak`, and `prof_final`.