chore: remove total difficulty from HeaderProvider (#19151)

crates/era-utils/src/export.rs

@@ -1,6 +1,7 @@
 //! Logic to export from database era1 block history
 //! and injecting them into era1 files with `Era1Writer`.
 
+use crate::calculate_td_by_number;
 use alloy_consensus::BlockHeader;
 use alloy_primitives::{BlockNumber, B256, U256};
 use eyre::{eyre, Result};
@@ -114,9 +115,7 @@ where
 
     let mut total_difficulty = if config.first_block_number > 0 {
         let prev_block_number = config.first_block_number - 1;
-        provider
-            .header_td_by_number(prev_block_number)?
-            .ok_or_else(|| eyre!("Total difficulty not found for block {prev_block_number}"))?
+        calculate_td_by_number(provider, prev_block_number)?
     } else {
         U256::ZERO
     };

crates/era-utils/src/history.rs

@@ -1,3 +1,4 @@
+use alloy_consensus::BlockHeader;
 use alloy_primitives::{BlockHash, BlockNumber, U256};
 use futures_util::{Stream, StreamExt};
 use reth_db_api::{
@@ -19,15 +20,15 @@ use reth_etl::Collector;
 use reth_fs_util as fs;
 use reth_primitives_traits::{Block, FullBlockBody, FullBlockHeader, NodePrimitives};
 use reth_provider::{
-    providers::StaticFileProviderRWRefMut, BlockWriter, ProviderError, StaticFileProviderFactory,
+    providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory,
     StaticFileSegment, StaticFileWriter,
 };
 use reth_stages_types::{
     CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId,
 };
 use reth_storage_api::{
-    errors::ProviderResult, DBProvider, DatabaseProviderFactory, HeaderProvider,
-    NodePrimitivesProvider, StageCheckpointWriter,
+    errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider,
+    StageCheckpointWriter,
 };
 use std::{
     collections::Bound,
@@ -82,11 +83,6 @@ where
         .get_highest_static_file_block(StaticFileSegment::Headers)
         .unwrap_or_default();
 
-    // Find the latest total difficulty
-    let mut td = static_file_provider
-        .header_td_by_number(height)?
-        .ok_or(ProviderError::TotalDifficultyNotFound(height))?;
-
     while let Some(meta) = rx.recv()? {
         let from = height;
         let provider = provider_factory.database_provider_rw()?;
@@ -96,7 +92,6 @@ where
             &mut static_file_provider.latest_writer(StaticFileSegment::Headers)?,
             &provider,
             hash_collector,
-            &mut td,
             height..,
         )?;
 
@@ -146,7 +141,7 @@ where
 
 /// Extracts block headers and bodies from `meta` and appends them using `writer` and `provider`.
 ///
-/// Adds on to `total_difficulty` and collects hash to height using `hash_collector`.
+/// Collects hash to height using `hash_collector`.
 ///
 /// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the
 /// [`end_bound`] or the end of the file.
@@ -160,7 +155,6 @@ pub fn process<Era, P, B, BB, BH>(
     writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
     provider: &P,
     hash_collector: &mut Collector<BlockHash, BlockNumber>,
-    total_difficulty: &mut U256,
     block_numbers: impl RangeBounds<BlockNumber>,
 ) -> eyre::Result<BlockNumber>
 where
@@ -182,7 +176,7 @@ where
                 as Box<dyn Fn(Result<BlockTuple, E2sError>) -> eyre::Result<(BH, BB)>>);
     let iter = ProcessIter { iter, era: meta };
 
-    process_iter(iter, writer, provider, hash_collector, total_difficulty, block_numbers)
+    process_iter(iter, writer, provider, hash_collector, block_numbers)
 }
 
 type ProcessInnerIter<R, BH, BB> =
@@ -271,7 +265,6 @@ pub fn process_iter<P, B, BB, BH>(
     writer: &mut StaticFileProviderRWRefMut<'_, <P as NodePrimitivesProvider>::Primitives>,
     provider: &P,
     hash_collector: &mut Collector<BlockHash, BlockNumber>,
-    total_difficulty: &mut U256,
     block_numbers: impl RangeBounds<BlockNumber>,
 ) -> eyre::Result<BlockNumber>
 where
@@ -311,11 +304,8 @@ where
         let hash = header.hash_slow();
         last_header_number = number;
 
-        // Increase total difficulty
-        *total_difficulty += header.difficulty();
-
         // Append to Headers segment
-        writer.append_header(&header, *total_difficulty, &hash)?;
+        writer.append_header(&header, &hash)?;
 
         // Write bodies to database.
         provider.append_block_bodies(vec![(header.number(), Some(body))])?;
@@ -382,3 +372,28 @@ where
 
     Ok(())
 }
+
+/// Calculates the total difficulty for a given block number by summing the difficulty
+/// of all blocks from genesis to the given block.
+///
+/// Very expensive - iterates through all blocks in batches of 1000.
+///
+/// Returns an error if any block is missing.
+pub fn calculate_td_by_number<P>(provider: &P, num: BlockNumber) -> eyre::Result<U256>
+where
+    P: BlockReader,
+{
+    let mut total_difficulty = U256::ZERO;
+    let mut start = 0;
+
+    while start <= num {
+        let end = (start + 1000 - 1).min(num);
+
+        total_difficulty +=
+            provider.headers_range(start..=end)?.iter().map(|h| h.difficulty()).sum::<U256>();
+
+        start = end + 1;
+    }
+
+    Ok(total_difficulty)
+}

crates/era-utils/src/lib.rs

@@ -14,5 +14,6 @@ pub use export::{export, ExportConfig};
 
 /// Imports history from ERA files.
 pub use history::{
-    build_index, decode, import, open, process, process_iter, save_stage_checkpoints, ProcessIter,
+    build_index, calculate_td_by_number, decode, import, open, process, process_iter,
+    save_stage_checkpoints, ProcessIter,
 };