Compare commits

...

3 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
d788306490 fix(prune): mark progress HasMoreData when breaking early due to limit
When the pruner loop breaks early because is_limit_reached() returns true,
the output.progress was not updated. This caused the CLI loop to think
pruning was finished when segments were still pending.

Now we explicitly set output.progress = limiter.progress(false) before
breaking, ensuring the caller knows there is more work to do.
2026-01-30 07:48:41 +00:00
Georgios Konstantopoulos
f462dce9eb fix: add cfg(feature = edge) to DEFAULT_DELETE_LIMIT and update docs
Amp-Thread-ID: https://ampcode.com/threads/T-019c0cf4-aa9c-7276-9c8a-b5d417b92f35
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 04:02:54 +00:00
Georgios Konstantopoulos
c06166f982 fix(prune): add --delete-limit flag with smart defaults based on edge feature
Adds a --delete-limit flag to control memory usage during pruning, with
intelligent defaults based on the build configuration:

- Edge builds (--features edge): defaults to 100,000 entries per run
  to prevent OOM during large batch deletions with static files
- Legacy builds: defaults to unlimited (no OOM risk with MDBX-only)
- Users can always override with --delete-limit <N> or --delete-limit 0

Changes:
- Use #[cfg(feature = "edge")] for compile-time default selection
- Loop until pruning completes when batching is enabled
- Add stuck-loop guard (bail after 2 consecutive zero-prune runs)

This ensures normal MDBX full node pruning is unaffected while providing
OOM protection for edge/static-file configurations.

Amp-Thread-ID: https://ampcode.com/threads/T-019c0cbd-d62c-7074-936b-3ac5d97ec6a3
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 03:49:12 +00:00
4 changed files with 81 additions and 8 deletions

View File

@@ -1,4 +1,4 @@
//! Command that runs pruning without any limits.
//! Command that runs pruning with a configurable delete limit.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
@@ -14,9 +14,19 @@ use reth_node_metrics::{
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
use tracing::info;
use tracing::{info, warn};
/// Prunes according to the configuration without any limits
/// Default delete limit per prune run to prevent OOM when static files are involved.
#[cfg(feature = "edge")]
const DEFAULT_DELETE_LIMIT: usize = 100_000;
/// Prunes according to the configuration with a configurable delete limit.
///
/// For edge builds (compiled with `--features edge`), this command defaults to limiting
/// deletions to 100,000 entries per pruner run to prevent OOM with static file operations.
/// For legacy builds, no limit is applied by default.
///
/// Use `--delete-limit <N>` to override the default, or `--delete-limit 0` for unlimited.
#[derive(Debug, Parser)]
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
@@ -25,6 +35,14 @@ pub struct PruneCommand<C: ChainSpecParser> {
/// Prometheus metrics configuration.
#[command(flatten)]
metrics: MetricArgs,
/// Maximum number of entries to delete per pruner run (across all segments).
///
/// For edge builds, defaults to 100,000 to prevent OOM.
/// For legacy builds, defaults to unlimited.
/// Set to 0 for unlimited deletions per run.
#[arg(long)]
delete_limit: Option<usize>,
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
@@ -58,6 +76,19 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
MetricServer::new(config).serve().await?;
}
// Resolve delete limit based on build configuration:
// - If user specified a limit, use it (0 means unlimited)
// - Edge builds default to batched deletions to prevent OOM with static files
// - Legacy builds default to unlimited (no OOM risk with MDBX-only)
let delete_limit = match self.delete_limit {
Some(0) => usize::MAX,
Some(limit) => limit,
#[cfg(feature = "edge")]
None => DEFAULT_DELETE_LIMIT,
#[cfg(not(feature = "edge"))]
None => usize::MAX,
};
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");
let static_file_producer =
@@ -68,14 +99,45 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
info!(target: "reth::cli", ?prune_tip, ?config, ?delete_limit, "Pruning data from database...");
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.delete_limit(delete_limit)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;
info!(target: "reth::cli", "Pruned data from database");
// Loop until pruning is complete, respecting delete_limit per iteration
let mut total_pruned = 0usize;
let mut runs = 0usize;
let mut consecutive_zero_runs = 0usize;
loop {
runs += 1;
let output = pruner.run(prune_tip)?;
let pruned_this_run: usize =
output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned += pruned_this_run;
if output.progress.is_finished() {
info!(target: "reth::cli", runs, total_pruned, "Pruned data from database");
break;
}
// Stuck-loop guard: bail after consecutive runs with no progress
if pruned_this_run == 0 {
consecutive_zero_runs += 1;
if consecutive_zero_runs >= 2 {
warn!(target: "reth::cli", runs, total_pruned, consecutive_zero_runs, "Pruner returned HasMoreData but made no progress");
eyre::bail!(
"Pruner returned HasMoreData but made no progress after \
{consecutive_zero_runs} consecutive runs. \
Try increasing --delete-limit or use --delete-limit 0 (unlimited)."
);
}
} else {
consecutive_zero_runs = 0;
}
info!(target: "reth::cli", runs, pruned_this_run, total_pruned, "Pruner has more data, continuing...");
}
}
Ok(())

View File

@@ -177,6 +177,7 @@ where
for segment in &self.segments {
if limiter.is_limit_reached() {
output.progress = limiter.progress(false);
break
}

View File

@@ -181,6 +181,11 @@ RocksDB:
[default: false]
[possible values: true, false]
--delete-limit <DELETE_LIMIT>
Maximum number of entries to delete per pruner run (across all segments).
For edge builds, defaults to 100,000 to prevent OOM. For legacy builds, defaults to unlimited. Set to 0 for unlimited deletions per run.
Metrics:
--metrics <PROMETHEUS>
Enable Prometheus metrics.

View File

@@ -181,6 +181,11 @@ RocksDB:
[default: false]
[possible values: true, false]
--delete-limit <DELETE_LIMIT>
Maximum number of entries to delete per pruner run (across all segments).
For edge builds, defaults to 100,000 to prevent OOM. For legacy builds, defaults to unlimited. Set to 0 for unlimited deletions per run.
Metrics:
--metrics <PROMETHEUS>
Enable Prometheus metrics.