Compare commits

...

1 Commits

Author SHA1 Message Date
Dan Cline
078e8d2162 feat(prune): allow headers to be pruned 2026-04-07 17:40:47 -04:00
15 changed files with 355 additions and 28 deletions

View File

@@ -78,6 +78,7 @@ pub enum SegmentArg {
ContractLogs,
AccountHistory,
StorageHistory,
Headers,
Bodies,
}
@@ -90,6 +91,7 @@ impl From<SegmentArg> for PruneSegment {
SegmentArg::ContractLogs => Self::ContractLogs,
SegmentArg::AccountHistory => Self::AccountHistory,
SegmentArg::StorageHistory => Self::StorageHistory,
SegmentArg::Headers => Self::Headers,
SegmentArg::Bodies => Self::Bodies,
}
}

View File

@@ -56,16 +56,17 @@ where
let segments = &config.prune.segments;
// Collect (segment, mode) pairs for all configured prune segments
let checkpoints: Vec<(PruneSegment, PruneMode)> = [
(PruneSegment::SenderRecovery, segments.sender_recovery),
(PruneSegment::TransactionLookup, segments.transaction_lookup),
(PruneSegment::Receipts, segments.receipts),
(PruneSegment::AccountHistory, segments.account_history),
(PruneSegment::StorageHistory, segments.storage_history),
(PruneSegment::Bodies, segments.bodies_history),
let checkpoints: Vec<(PruneSegment, PruneMode, bool)> = [
(PruneSegment::SenderRecovery, segments.sender_recovery, true),
(PruneSegment::TransactionLookup, segments.transaction_lookup, true),
(PruneSegment::Receipts, segments.receipts, true),
(PruneSegment::AccountHistory, segments.account_history, true),
(PruneSegment::StorageHistory, segments.storage_history, true),
(PruneSegment::Headers, segments.headers, false),
(PruneSegment::Bodies, segments.bodies_history, true),
]
.into_iter()
.filter_map(|(segment, mode)| mode.map(|m| (segment, m)))
.filter_map(|(segment, mode, uses_tx_number)| mode.map(|m| (segment, m, uses_tx_number)))
.collect();
if checkpoints.is_empty() {
@@ -76,10 +77,10 @@ where
let tx_number =
tx.get::<tables::BlockBodyIndices>(snapshot_block)?.map(|indices| indices.last_tx_num());
for (segment, prune_mode) in &checkpoints {
for (segment, prune_mode, uses_tx_number) in &checkpoints {
let checkpoint = PruneCheckpoint {
block_number: Some(snapshot_block),
tx_number,
tx_number: uses_tx_number.then_some(tx_number).flatten(),
prune_mode: *prune_mode,
};
@@ -259,6 +260,7 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
[
("sender_recovery", segments.sender_recovery),
("transaction_lookup", segments.transaction_lookup),
("headers", segments.headers),
("bodies_history", segments.bodies_history),
("receipts", segments.receipts),
("account_history", segments.account_history),

View File

@@ -587,6 +587,7 @@ impl PruneConfig {
receipts,
account_history,
storage_history,
headers,
bodies_history,
receipts_log_filter,
},
@@ -609,6 +610,7 @@ impl PruneConfig {
self.segments.receipts = self.segments.receipts.or(receipts);
self.segments.account_history = self.segments.account_history.or(account_history);
self.segments.storage_history = self.segments.storage_history.or(storage_history);
self.segments.headers = self.segments.headers.or(headers);
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
@@ -1121,6 +1123,7 @@ receipts = { distance = 16384 }
receipts: Some(PruneMode::Distance(1000)),
account_history: None,
storage_history: Some(PruneMode::Before(5000)),
headers: None,
bodies_history: None,
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
@@ -1138,6 +1141,7 @@ receipts = { distance = 16384 }
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Distance(2000)),
storage_history: Some(PruneMode::Distance(3000)),
headers: Some(PruneMode::Before(10_000)),
bodies_history: None,
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
@@ -1157,6 +1161,7 @@ receipts = { distance = 16384 }
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.headers, Some(PruneMode::Before(10_000)));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}

View File

@@ -1308,6 +1308,9 @@ mod tests {
storage_history_full: false,
storage_history_distance: None,
storage_history_before: None,
headers_full: false,
headers_distance: None,
headers_before: None,
bodies_pre_merge: false,
bodies_distance: None,
receipts_log_filter: None,

View File

@@ -73,6 +73,7 @@ impl Default for DefaultPruningValues {
receipts: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
headers: None,
// This field is ignored when full_bodies_history_use_pre_merge is true
bodies_history: None,
receipts_log_filter: Default::default(),
@@ -84,6 +85,7 @@ impl Default for DefaultPruningValues {
receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
headers: None,
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
receipts_log_filter: Default::default(),
},
@@ -184,6 +186,19 @@ pub struct PruningArgs {
#[arg(long = "prune.storage-history.before", alias = "prune.storagehistory.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["storage_history_full", "storage_history_distance"])]
pub storage_history_before: Option<BlockNumber>,
// Headers
/// Prunes header history data.
#[arg(long = "prune.headers.full", conflicts_with_all = &["headers_distance", "headers_before"])]
pub headers_full: bool,
/// Prune header history before the `head-N` block number. In other words, keep last N + 1
/// blocks.
#[arg(long = "prune.headers.distance", value_name = "BLOCKS", conflicts_with_all = &["headers_full", "headers_before"])]
pub headers_distance: Option<u64>,
/// Prune header history before the specified block number. The specified block number is not
/// pruned.
#[arg(long = "prune.headers.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["headers_full", "headers_distance"])]
pub headers_before: Option<BlockNumber>,
// Bodies
/// Prune bodies before the merge block.
#[arg(long = "prune.bodies.pre-merge", value_name = "BLOCKS", conflicts_with_all = &["bodies_distance", "bodies_before"])]
@@ -260,6 +275,9 @@ impl PruningArgs {
if let Some(mode) = self.account_history_prune_mode() {
config.segments.account_history = Some(mode);
}
if let Some(mode) = self.headers_prune_mode() {
config.segments.headers = Some(mode);
}
if let Some(mode) = self.bodies_prune_mode(chain_spec) {
config.segments.bodies_history = Some(mode);
}
@@ -348,6 +366,18 @@ impl PruningArgs {
}
}
const fn headers_prune_mode(&self) -> Option<PruneMode> {
if self.headers_full {
Some(PruneMode::Full)
} else if let Some(distance) = self.headers_distance {
Some(PruneMode::Distance(distance))
} else if let Some(block_number) = self.headers_before {
Some(PruneMode::Before(block_number))
} else {
None
}
}
const fn storage_history_prune_mode(&self) -> Option<PruneMode> {
if self.storage_history_full {
Some(PruneMode::Full)
@@ -409,6 +439,7 @@ mod tests {
use super::*;
use alloy_primitives::address;
use clap::Parser;
use reth_chainspec::MAINNET;
/// A helper type to parse Args more easily
#[derive(Parser)]
@@ -440,6 +471,22 @@ mod tests {
assert_eq!(args, default_args);
}
#[test]
fn parse_headers_pruning_flags() {
let args = CommandParser::<PruningArgs>::parse_from(["reth", "--prune.headers.full"]).args;
assert!(args.headers_full);
let config = args.prune_config(MAINNET.as_ref()).expect("headers prune config");
assert_eq!(config.segments.headers, Some(PruneMode::Full));
let args =
CommandParser::<PruningArgs>::parse_from(["reth", "--prune.headers.before", "900000"])
.args;
let config = args.prune_config(MAINNET.as_ref()).expect("headers prune config");
assert_eq!(config.segments.headers, Some(PruneMode::Before(900_000)));
}
#[test]
fn test_parse_receipts_log_filter() {
let filter1 = "0x0000000000000000000000000000000000000001:full";

View File

@@ -17,7 +17,7 @@ pub use set::SegmentSet;
use std::{fmt::Debug, ops::RangeInclusive};
use tracing::error;
pub use user::{
AccountHistory, Bodies, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery,
AccountHistory, Bodies, Headers, Receipts as UserReceipts, ReceiptsByLogs, SenderRecovery,
StorageHistory, TransactionLookup,
};
@@ -71,6 +71,52 @@ where
})
}
/// Prunes block-based static files for a given segment.
///
/// This is used by header pruning where deleted rows are tracked by block, not transaction.
pub(crate) fn prune_block_based_static_files<Provider>(
provider: &Provider,
input: PruneInput,
segment: StaticFileSegment,
) -> Result<SegmentOutput, PrunerError>
where
Provider: StaticFileProviderFactory,
{
let deleted_headers =
provider.static_file_provider().delete_segment_below_block(segment, input.to_block + 1)?;
if deleted_headers.is_empty() {
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned: 0,
checkpoint: input
.previous_checkpoint
.map(SegmentOutputCheckpoint::from_prune_checkpoint),
})
}
let pruned = deleted_headers
.iter()
.filter_map(|header| header.block_range())
.map(|range| range.len())
.sum::<u64>() as usize;
let checkpoint_block = deleted_headers
.iter()
.filter_map(|header| header.block_range())
.map(|range| range.end())
.max();
Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: checkpoint_block,
tx_number: None,
}),
})
}
/// Deletes ALL static file jars for a given segment.
///
/// This is used for `PruneMode::Full` where all data should be removed, including the highest jar.

View File

@@ -1,5 +1,5 @@
use crate::segments::{
user::ReceiptsByLogs, AccountHistory, Bodies, Segment, SenderRecovery, StorageHistory,
user::ReceiptsByLogs, AccountHistory, Bodies, Headers, Segment, SenderRecovery, StorageHistory,
TransactionLookup, UserReceipts,
};
use alloy_eips::eip2718::Encodable2718;
@@ -71,11 +71,14 @@ where
receipts,
account_history,
storage_history,
headers,
bodies_history,
receipts_log_filter,
} = prune_modes;
Self::default()
// Headers
.segment_opt(headers.map(Headers::new))
// Transaction lookup must run before bodies because it needs to read transaction
// data from static files before bodies deletes them.
.segment_opt(transaction_lookup.map(TransactionLookup::new))

View File

@@ -0,0 +1,204 @@
use crate::{
segments::{self, PruneInput, Segment},
PrunerError,
};
use reth_provider::StaticFileProviderFactory;
use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutput};
use reth_static_file_types::StaticFileSegment;
use tracing::instrument;
/// Segment responsible for pruning headers in static files.
#[derive(Debug)]
pub struct Headers {
mode: PruneMode,
}
impl Headers {
/// Creates a new [`Headers`] segment with the given prune mode.
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
}
}
impl<Provider> Segment<Provider> for Headers
where
Provider: StaticFileProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::Headers
}
fn mode(&self) -> Option<PruneMode> {
Some(self.mode)
}
fn purpose(&self) -> PrunePurpose {
PrunePurpose::User
}
#[instrument(
name = "Headers::prune",
target = "pruner",
skip(self, provider),
ret(level = "trace")
)]
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
segments::prune_block_based_static_files(provider, input, StaticFileSegment::Headers)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{segments::PruneLimiter, SegmentOutput};
use reth_provider::{
test_utils::{create_test_provider_factory, MockNodeTypesWithDB},
ProviderFactory, StaticFileWriter,
};
use reth_prune_types::{PruneMode, PruneProgress};
use reth_static_file_types::{
SegmentHeader, SegmentRangeInclusive, DEFAULT_BLOCKS_PER_STATIC_FILE,
};
fn setup_header_static_file_jars<P: StaticFileProviderFactory>(provider: &P, tip_block: u64) {
let num_jars = (tip_block + 1) / DEFAULT_BLOCKS_PER_STATIC_FILE;
let static_file_provider = provider.static_file_provider();
let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap();
for jar_idx in 0..num_jars {
let block_start = jar_idx * DEFAULT_BLOCKS_PER_STATIC_FILE;
let block_end = ((jar_idx + 1) * DEFAULT_BLOCKS_PER_STATIC_FILE - 1).min(tip_block);
*writer.user_header_mut() = SegmentHeader::new(
SegmentRangeInclusive::new(block_start, block_end),
Some(SegmentRangeInclusive::new(block_start, block_end)),
None,
StaticFileSegment::Headers,
);
writer.inner().set_dirty();
writer.commit().expect("commit empty header jar");
if jar_idx < num_jars - 1 {
writer.increment_block(block_end + 1).expect("increment header block");
}
}
static_file_provider.initialize_index().expect("initialize index");
}
fn setup_transaction_static_file_jars<P: StaticFileProviderFactory>(
provider: &P,
tip_block: u64,
) {
let num_jars = (tip_block + 1) / DEFAULT_BLOCKS_PER_STATIC_FILE;
let static_file_provider = provider.static_file_provider();
let mut writer =
static_file_provider.latest_writer(StaticFileSegment::Transactions).unwrap();
for jar_idx in 0..num_jars {
let block_start = jar_idx * DEFAULT_BLOCKS_PER_STATIC_FILE;
let block_end = ((jar_idx + 1) * DEFAULT_BLOCKS_PER_STATIC_FILE - 1).min(tip_block);
*writer.user_header_mut() = SegmentHeader::new(
SegmentRangeInclusive::new(block_start, block_end),
Some(SegmentRangeInclusive::new(block_start, block_end)),
Some(SegmentRangeInclusive::new(jar_idx, jar_idx)),
StaticFileSegment::Transactions,
);
writer.inner().set_dirty();
writer.commit().expect("commit empty transaction jar");
if jar_idx < num_jars - 1 {
writer.increment_block(block_end + 1).expect("increment transaction block");
}
}
static_file_provider.initialize_index().expect("initialize index");
}
fn prune_headers(
factory: &ProviderFactory<MockNodeTypesWithDB>,
mode: PruneMode,
tip: u64,
) -> SegmentOutput {
let (to_block, _) = mode
.prune_target_block(tip, PruneSegment::Headers, PrunePurpose::User)
.unwrap()
.expect("headers should have data to prune");
Headers::new(mode)
.prune(
factory,
PruneInput {
previous_checkpoint: None,
to_block,
limiter: PruneLimiter::default(),
},
)
.expect("headers prune should succeed")
}
#[test]
fn prune_headers_before_deletes_whole_jars_and_tracks_blocks() {
let factory = create_test_provider_factory();
let tip = 2_499_999;
setup_header_static_file_jars(&factory, tip);
let output = prune_headers(&factory, PruneMode::Before(900_000), tip);
assert_eq!(output.progress, PruneProgress::Finished);
assert_eq!(output.pruned, DEFAULT_BLOCKS_PER_STATIC_FILE as usize);
assert_eq!(output.checkpoint.unwrap().block_number, Some(499_999));
assert_eq!(
factory.static_file_provider().get_lowest_range_start(StaticFileSegment::Headers),
Some(500_000)
);
assert_eq!(
factory
.static_file_provider()
.get_highest_static_file_block(StaticFileSegment::Headers),
Some(tip)
);
}
#[test]
fn prune_headers_full_keeps_recent_history_jar() {
let factory = create_test_provider_factory();
let tip = 2_499_999;
setup_header_static_file_jars(&factory, tip);
let output = prune_headers(&factory, PruneMode::Full, tip);
assert_eq!(output.progress, PruneProgress::Finished);
assert_eq!(output.pruned, (DEFAULT_BLOCKS_PER_STATIC_FILE * 4) as usize);
assert_eq!(output.checkpoint.unwrap().block_number, Some(1_999_999));
assert_eq!(
factory.static_file_provider().get_lowest_range_start(StaticFileSegment::Headers),
Some(2_000_000)
);
assert_eq!(
factory
.static_file_provider()
.get_highest_static_file_block(StaticFileSegment::Headers),
Some(tip)
);
}
#[test]
fn pruning_headers_updates_earliest_history_height() {
let factory = create_test_provider_factory();
let tip = 999_999;
setup_header_static_file_jars(&factory, tip);
setup_transaction_static_file_jars(&factory, tip);
let output = prune_headers(&factory, PruneMode::Before(900_000), tip);
assert_eq!(output.checkpoint.unwrap().block_number, Some(499_999));
assert_eq!(factory.static_file_provider().earliest_history_height(), 500_000);
}
}

View File

@@ -1,5 +1,6 @@
mod account_history;
mod bodies;
mod headers;
mod history;
mod receipts;
mod receipts_by_logs;
@@ -9,6 +10,7 @@ mod transaction_lookup;
pub use account_history::AccountHistory;
pub use bodies::Bodies;
pub use headers::Headers;
pub use receipts::Receipts;
pub use receipts_by_logs::ReceiptsByLogs;
pub use sender_recovery::SenderRecovery;

View File

@@ -28,8 +28,6 @@ pub enum PruneSegment {
AccountHistory,
/// Prunes storage changesets (static files/MDBX) and `StoragesHistory`.
StorageHistory,
#[deprecated = "Variant indexes cannot be changed"]
#[strum(disabled)]
/// Prune segment responsible for the `CanonicalHeaders`, `Headers` tables.
Headers,
#[deprecated = "Variant indexes cannot be changed"]
@@ -66,13 +64,13 @@ impl PruneSegment {
pub const fn min_blocks(&self) -> u64 {
match self {
Self::SenderRecovery | Self::TransactionLookup => 0,
Self::Receipts | Self::Bodies => MINIMUM_DISTANCE,
Self::Receipts | Self::Headers | Self::Bodies => MINIMUM_DISTANCE,
Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => {
MINIMUM_UNWIND_SAFE_DISTANCE
}
#[expect(deprecated)]
#[expect(clippy::match_same_arms)]
Self::Headers | Self::Transactions | Self::MerkleChangeSets => 0,
Self::Transactions | Self::MerkleChangeSets => 0,
}
}
@@ -121,13 +119,14 @@ mod tests {
use super::*;
#[test]
fn test_prune_segment_iter_excludes_deprecated() {
fn test_prune_segment_iter_includes_headers_and_excludes_deprecated() {
let segments: Vec<PruneSegment> = PruneSegment::variants().collect();
assert!(segments.contains(&PruneSegment::Headers));
// Verify deprecated variants are not included derived iter
#[expect(deprecated)]
{
assert!(!segments.contains(&PruneSegment::Headers));
assert!(!segments.contains(&PruneSegment::Transactions));
assert!(!segments.contains(&PruneSegment::MerkleChangeSets));
}

View File

@@ -74,6 +74,9 @@ pub struct PruneModes {
)
)]
pub storage_history: Option<PruneMode>,
/// Headers pruning configuration.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub headers: Option<PruneMode>,
/// Bodies History pruning configuration.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub bodies_history: Option<PruneMode>,
@@ -98,6 +101,7 @@ impl PruneModes {
receipts: Some(PruneMode::Full),
account_history: Some(PruneMode::Full),
storage_history: Some(PruneMode::Full),
headers: Some(PruneMode::Full),
bodies_history: Some(PruneMode::Full),
receipts_log_filter: Default::default(),
}

View File

@@ -1216,14 +1216,15 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
// If this is a re-initialization, we need to clear this as well
self.map.clear();
// initialize the expired history height to the lowest static file block
if let Some(lowest_range) =
indexes.get(StaticFileSegment::Transactions).and_then(|index| index.min_block_range)
{
// the earliest height is the lowest available block number
self.earliest_history_height
.store(lowest_range.start(), std::sync::atomic::Ordering::Relaxed);
}
let earliest_history_height = [StaticFileSegment::Headers, StaticFileSegment::Transactions]
.into_iter()
.filter_map(|segment| indexes.get(segment).and_then(|index| index.min_block_range))
.map(|range| range.start())
.max()
.unwrap_or_default();
self.earliest_history_height
.store(earliest_history_height, std::sync::atomic::Ordering::Relaxed);
Ok(())
}

View File

@@ -12,7 +12,7 @@ Options:
--segment <SEGMENT>
Specific segment to query. If omitted, shows all segments
[possible values: sender-recovery, transaction-lookup, receipts, contract-logs, account-history, storage-history, bodies]
[possible values: sender-recovery, transaction-lookup, receipts, contract-logs, account-history, storage-history, headers, bodies]
-h, --help
Print help (see a summary with '-h')

View File

@@ -12,7 +12,7 @@ Options:
--segment <SEGMENT>
The prune segment to update
[possible values: sender-recovery, transaction-lookup, receipts, contract-logs, account-history, storage-history, bodies]
[possible values: sender-recovery, transaction-lookup, receipts, contract-logs, account-history, storage-history, headers, bodies]
--block-number <BLOCK_NUMBER>
Highest pruned block number

View File

@@ -905,6 +905,15 @@ Pruning:
--prune.storage-history.before <BLOCK_NUMBER>
Prune storage history before the specified block number. The specified block number is not pruned
--prune.headers.full
Prunes header history data
--prune.headers.distance <BLOCKS>
Prune header history before the `head-N` block number. In other words, keep last N + 1 blocks
--prune.headers.before <BLOCK_NUMBER>
Prune header history before the specified block number. The specified block number is not pruned
--prune.bodies.pre-merge
Prune bodies before the merge block