Compare commits

...

1 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
b22a039341 perf(trie): change reveal_nodes to take &mut slice for buffer reuse
Changes reveal_nodes signature from Vec<ProofTrieNode> to &mut [ProofTrieNode]
to enable caller-side buffer reuse and ensure expensive drops happen after
lock release.

Key changes:
- Update SparseTrie trait: reveal_nodes now takes &mut [ProofTrieNode]
- Update SerialSparseTrie impl to use core::mem::replace for node extraction
- Update ParallelSparseTrie impl to work with slice reference
- Add proof_nodes_buf field to SparseStateTrie for buffer reuse
- Add DeferredDrops struct to hold buffers for deferred deallocation
- Update into_trie_for_reuse and into_cleared_trie to return (trie, DeferredDrops)
- Ensure preserved_sparse_trie lock is released before DeferredDrops is dropped

Co-authored-by: Amp <amp@ampcode.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019c23b9-0835-7254-8579-ad1bc1d5d5e3
2026-02-03 13:40:56 +00:00
6 changed files with 100 additions and 60 deletions

View File

@@ -582,11 +582,14 @@ where
target: "engine::tree::payload_processor",
"State root receiver dropped, clearing trie"
);
let trie = task.into_cleared_trie(
let (trie, deferred) = task.into_cleared_trie(
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
guard.store(PreservedSparseTrie::cleared(trie));
// Drop guard before deferred to release lock before expensive deallocations
drop(guard);
drop(deferred);
return;
}
@@ -594,9 +597,9 @@ where
// A failed computation may have left the trie in a partially updated state.
let _enter =
debug_span!(target: "engine::tree::payload_processor", "preserve").entered();
if let Some(state_root) = computed_state_root {
let deferred = if let Some(state_root) = computed_state_root {
let start = std::time::Instant::now();
let trie = task.into_trie_for_reuse(
let (trie, deferred) = task.into_trie_for_reuse(
prune_depth,
max_storage_tries,
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
@@ -606,17 +609,22 @@ where
.into_trie_for_reuse_duration_histogram
.record(start.elapsed().as_secs_f64());
guard.store(PreservedSparseTrie::anchored(trie, state_root));
deferred
} else {
debug!(
target: "engine::tree::payload_processor",
"State root computation failed, clearing trie"
);
let trie = task.into_cleared_trie(
let (trie, deferred) = task.into_cleared_trie(
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
guard.store(PreservedSparseTrie::cleared(trie));
}
deferred
};
// Drop guard before deferred to release lock before expensive deallocations
drop(guard);
drop(deferred);
});
}

View File

@@ -28,7 +28,7 @@ use reth_trie_parallel::{
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
DeferredDrops, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
};
use revm_primitives::{hash_map::Entry, B256Map};
use smallvec::SmallVec;
@@ -72,7 +72,7 @@ where
max_storage_tries: usize,
max_nodes_capacity: usize,
max_values_capacity: usize,
) -> SparseStateTrie<A, S> {
) -> (SparseStateTrie<A, S>, DeferredDrops) {
match self {
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
Self::Cached(task) => task.into_trie_for_reuse(
@@ -88,7 +88,7 @@ where
self,
max_nodes_capacity: usize,
max_values_capacity: usize,
) -> SparseStateTrie<A, S> {
) -> (SparseStateTrie<A, S>, DeferredDrops) {
match self {
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
Self::Cached(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
@@ -198,10 +198,11 @@ where
mut self,
max_nodes_capacity: usize,
max_values_capacity: usize,
) -> SparseStateTrie<A, S> {
) -> (SparseStateTrie<A, S>, DeferredDrops) {
self.trie.clear();
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
self.trie
let deferred = self.trie.take_deferred_drops();
(self.trie, deferred)
}
}
@@ -311,10 +312,11 @@ where
max_storage_tries: usize,
max_nodes_capacity: usize,
max_values_capacity: usize,
) -> SparseStateTrie<A, S> {
) -> (SparseStateTrie<A, S>, DeferredDrops) {
self.trie.prune(prune_depth, max_storage_tries);
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
self.trie
let deferred = self.trie.take_deferred_drops();
(self.trie, deferred)
}
/// Clears and shrinks the trie, discarding all state.
@@ -325,10 +327,11 @@ where
mut self,
max_nodes_capacity: usize,
max_values_capacity: usize,
) -> SparseStateTrie<A, S> {
) -> (SparseStateTrie<A, S>, DeferredDrops) {
self.trie.clear();
self.trie.shrink_to(max_nodes_capacity, max_values_capacity);
self.trie
let deferred = self.trie.take_deferred_drops();
(self.trie, deferred)
}
/// Runs the sparse trie task to completion.

View File

@@ -173,7 +173,7 @@ impl SparseTrie for ParallelSparseTrie {
self.updates = retain_updates.then(Default::default);
}
fn reveal_nodes(&mut self, mut nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNode]) -> SparseTrieResult<()> {
if nodes.is_empty() {
return Ok(())
}
@@ -189,7 +189,7 @@ impl SparseTrie for ParallelSparseTrie {
);
// Update the top-level branch node masks. This is simple and can't be done in parallel.
for ProofTrieNode { path, masks, .. } in &nodes {
for ProofTrieNode { path, masks, .. } in nodes.iter() {
if let Some(branch_masks) = masks {
self.branch_node_masks.insert(*path, *branch_masks);
}
@@ -4097,7 +4097,7 @@ mod tests {
let node = create_leaf_node([0x2, 0x3], 42);
let masks = None;
trie.reveal_nodes(vec![ProofTrieNode { path, node, masks }]).unwrap();
trie.reveal_nodes(&mut [ProofTrieNode { path, node, masks }]).unwrap();
assert_matches!(
trie.upper_subtrie.nodes.get(&path),
@@ -4118,7 +4118,7 @@ mod tests {
let node = create_leaf_node([0x3, 0x4], 42);
let masks = None;
trie.reveal_nodes(vec![ProofTrieNode { path, node, masks }]).unwrap();
trie.reveal_nodes(&mut [ProofTrieNode { path, node, masks }]).unwrap();
// Check that the lower subtrie was created
let idx = path_subtrie_index_unchecked(&path);
@@ -4142,7 +4142,7 @@ mod tests {
let node = create_leaf_node([0x4, 0x5], 42);
let masks = None;
trie.reveal_nodes(vec![ProofTrieNode { path, node, masks }]).unwrap();
trie.reveal_nodes(&mut [ProofTrieNode { path, node, masks }]).unwrap();
// Check that the lower subtrie's path hasn't changed
let idx = path_subtrie_index_unchecked(&path);
@@ -4203,7 +4203,7 @@ mod tests {
let node = create_extension_node([0x2], child_hash);
let masks = None;
trie.reveal_nodes(vec![ProofTrieNode { path, node, masks }]).unwrap();
trie.reveal_nodes(&mut [ProofTrieNode { path, node, masks }]).unwrap();
// Extension node should be in upper trie
assert_matches!(
@@ -4265,7 +4265,7 @@ mod tests {
let node = create_branch_node_with_children(&[0x0, 0x7, 0xf], child_hashes.clone());
let masks = None;
trie.reveal_nodes(vec![ProofTrieNode { path, node, masks }]).unwrap();
trie.reveal_nodes(&mut [ProofTrieNode { path, node, masks }]).unwrap();
// Branch node should be in upper trie
assert_matches!(
@@ -4321,7 +4321,7 @@ mod tests {
let branch_node = create_branch_node_with_children(&[0x0, 0x1, 0x2], child_hashes);
// Reveal nodes using reveal_nodes
trie.reveal_nodes(vec![
trie.reveal_nodes(&mut [
ProofTrieNode { path: branch_path, node: branch_node, masks: None },
ProofTrieNode { path: leaf_1_path, node: leaf_1, masks: None },
ProofTrieNode { path: leaf_2_path, node: leaf_2, masks: None },
@@ -5024,7 +5024,7 @@ mod tests {
let removed_branch_path = Nibbles::from_nibbles([0x4, 0xf, 0x8, 0x8, 0x0, 0x7, 0x2]);
// Convert the logs into reveal_nodes call on a fresh ParallelSparseTrie
let nodes = vec![
let mut nodes = vec![
// Branch at 0x4f8807
ProofTrieNode {
path: branch_path,
@@ -5155,7 +5155,7 @@ mod tests {
.unwrap();
// Call reveal_nodes
trie.reveal_nodes(nodes).unwrap();
trie.reveal_nodes(&mut nodes).unwrap();
// Remove the leaf at "0x4f88072c077f86613088dfcae648abe831fadca55ad43ab165d1680dd567b5d6"
let leaf_key = Nibbles::from_nibbles([
@@ -5241,7 +5241,7 @@ mod tests {
// Step 2: Reveal nodes in the trie
let mut trie = ParallelSparseTrie::from_root(extension, None, true).unwrap();
trie.reveal_nodes(vec![
trie.reveal_nodes(&mut [
ProofTrieNode { path: branch_path, node: branch, masks: None },
ProofTrieNode { path: leaf_1_path, node: leaf_1, masks: None },
ProofTrieNode { path: leaf_2_path, node: leaf_2, masks: None },
@@ -5781,7 +5781,7 @@ mod tests {
// ├── 0 -> Hash (Path = 0)
// └── 1 -> Leaf (Path = 1)
sparse
.reveal_nodes(vec![
.reveal_nodes(&mut [
ProofTrieNode {
path: Nibbles::default(),
node: branch,
@@ -5836,7 +5836,7 @@ mod tests {
// ├── 0 -> Hash (Path = 0)
// └── 1 -> Leaf (Path = 1)
sparse
.reveal_nodes(vec![
.reveal_nodes(&mut [
ProofTrieNode {
path: Nibbles::default(),
node: branch,
@@ -6202,7 +6202,7 @@ mod tests {
Default::default(),
[key1()],
);
let revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
let mut revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
.nodes_sorted()
.into_iter()
.map(|(path, node)| {
@@ -6212,7 +6212,7 @@ mod tests {
ProofTrieNode { path, node: TrieNode::decode(&mut &node[..]).unwrap(), masks }
})
.collect();
sparse.reveal_nodes(revealed_nodes).unwrap();
sparse.reveal_nodes(&mut revealed_nodes).unwrap();
// Check that the branch node exists with only two nibbles set
assert_eq!(
@@ -6237,7 +6237,7 @@ mod tests {
Default::default(),
[key3()],
);
let revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
let mut revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
.nodes_sorted()
.into_iter()
.map(|(path, node)| {
@@ -6247,7 +6247,7 @@ mod tests {
ProofTrieNode { path, node: TrieNode::decode(&mut &node[..]).unwrap(), masks }
})
.collect();
sparse.reveal_nodes(revealed_nodes).unwrap();
sparse.reveal_nodes(&mut revealed_nodes).unwrap();
// Check that nothing changed in the branch node
assert_eq!(
@@ -6323,7 +6323,7 @@ mod tests {
Default::default(),
[key1(), Nibbles::from_nibbles_unchecked([0x01])],
);
let revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
let mut revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
.nodes_sorted()
.into_iter()
.map(|(path, node)| {
@@ -6333,7 +6333,7 @@ mod tests {
ProofTrieNode { path, node: TrieNode::decode(&mut &node[..]).unwrap(), masks }
})
.collect();
sparse.reveal_nodes(revealed_nodes).unwrap();
sparse.reveal_nodes(&mut revealed_nodes).unwrap();
// Check that the branch node exists
assert_eq!(
@@ -6358,7 +6358,7 @@ mod tests {
Default::default(),
[key2()],
);
let revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
let mut revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
.nodes_sorted()
.into_iter()
.map(|(path, node)| {
@@ -6368,7 +6368,7 @@ mod tests {
ProofTrieNode { path, node: TrieNode::decode(&mut &node[..]).unwrap(), masks }
})
.collect();
sparse.reveal_nodes(revealed_nodes).unwrap();
sparse.reveal_nodes(&mut revealed_nodes).unwrap();
// Check that nothing changed in the extension node
assert_eq!(
@@ -6450,7 +6450,7 @@ mod tests {
Default::default(),
[key1()],
);
let revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
let mut revealed_nodes: Vec<ProofTrieNode> = hash_builder_proof_nodes
.nodes_sorted()
.into_iter()
.map(|(path, node)| {
@@ -6460,7 +6460,7 @@ mod tests {
ProofTrieNode { path, node: TrieNode::decode(&mut &node[..]).unwrap(), masks }
})
.collect();
sparse.reveal_nodes(revealed_nodes).unwrap();
sparse.reveal_nodes(&mut revealed_nodes).unwrap();
// Check that the branch node wasn't overwritten by the extension node in the proof
assert_matches!(
@@ -7424,7 +7424,7 @@ mod tests {
let leaf_node = LeafNode::new(leaf_key, leaf_value);
let leaf_masks = None;
trie.reveal_nodes(vec![
trie.reveal_nodes(&mut [
ProofTrieNode {
path: Nibbles::from_nibbles([0x3]),
node: TrieNode::Branch(branch_0x3_node),
@@ -7734,7 +7734,7 @@ mod tests {
);
// Reveal the trie structure using ProofTrieNode
let proof_nodes = vec![
let mut proof_nodes = vec![
ProofTrieNode {
path: Nibbles::from_nibbles([0x3]),
node: branch_0x3_node,
@@ -7765,7 +7765,7 @@ mod tests {
)
.expect("root revealed");
trie.reveal_nodes(proof_nodes).unwrap();
trie.reveal_nodes(&mut proof_nodes).unwrap();
// Update the leaf in order to reveal it in the trie
trie.update_leaf(leaf_nibbles, leaf_value, &provider).unwrap();
@@ -7813,7 +7813,7 @@ mod tests {
let leaf_node = create_leaf_node(leaf_key.to_vec(), 42);
// Reveal the leaf node
trie.reveal_nodes(vec![ProofTrieNode { path: leaf_path, node: leaf_node, masks: None }])
trie.reveal_nodes(&mut [ProofTrieNode { path: leaf_path, node: leaf_node, masks: None }])
.unwrap();
// The full path is leaf_path + leaf_key

View File

@@ -23,6 +23,16 @@ use reth_trie_common::{
use tracing::debug;
use tracing::{instrument, trace};
/// Holds data that should be dropped after any locks are released.
///
/// This is used to defer expensive deallocations (like proof node buffers)
/// until after the `preserved_sparse_trie` lock is released.
#[derive(Debug, Default)]
pub struct DeferredDrops {
/// Proof nodes buffer that can be dropped after lock release.
pub proof_nodes_buf: Vec<ProofTrieNode>,
}
#[derive(Debug)]
/// Sparse state trie representing lazy-loaded Ethereum state trie.
pub struct SparseStateTrie<
@@ -39,6 +49,8 @@ pub struct SparseStateTrie<
retain_updates: bool,
/// Reusable buffer for RLP encoding of trie accounts.
account_rlp_buf: Vec<u8>,
/// Reusable buffer for proof nodes, to avoid allocations across payload runs.
proof_nodes_buf: Vec<ProofTrieNode>,
/// Metrics for the sparse state trie.
#[cfg(feature = "metrics")]
metrics: crate::metrics::SparseStateTrieMetrics,
@@ -56,6 +68,7 @@ where
storage: Default::default(),
retain_updates: false,
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
proof_nodes_buf: Vec::new(),
#[cfg(feature = "metrics")]
metrics: Default::default(),
}
@@ -105,6 +118,14 @@ impl<A, S> SparseStateTrie<A, S> {
self.set_default_storage_trie(trie);
self
}
/// Takes the proof nodes buffer for deferred dropping.
///
/// This allows the caller to drop the buffer after releasing any locks,
/// avoiding expensive deallocations while holding locks.
pub fn take_deferred_drops(&mut self) -> DeferredDrops {
DeferredDrops { proof_nodes_buf: core::mem::take(&mut self.proof_nodes_buf) }
}
}
impl<A, S> SparseStateTrie<A, S>
@@ -420,12 +441,16 @@ where
account_subtree: DecodedProofNodes,
branch_node_masks: BranchNodeMasksMap,
) -> SparseStateTrieResult<()> {
let FilterMappedProofNodes { root_node, nodes, new_nodes, metric_values: _metric_values } =
filter_map_revealed_nodes(
account_subtree,
&mut self.revealed_account_paths,
&branch_node_masks,
)?;
let FilterMappedProofNodes {
root_node,
mut nodes,
new_nodes,
metric_values: _metric_values,
} = filter_map_revealed_nodes(
account_subtree,
&mut self.revealed_account_paths,
&branch_node_masks,
)?;
#[cfg(feature = "metrics")]
{
self.metrics.increment_total_account_nodes(_metric_values.total_nodes as u64);
@@ -443,7 +468,7 @@ where
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", total_nodes = ?nodes.len(), "Revealing account nodes");
trie.reveal_nodes(nodes)?;
trie.reveal_nodes(&mut nodes)?;
}
Ok(())
@@ -457,7 +482,7 @@ where
&mut self,
nodes: Vec<ProofTrieNode>,
) -> SparseStateTrieResult<()> {
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values: _metric_values } =
let FilteredV2ProofNodes { root_node, mut nodes, new_nodes, metric_values: _metric_values } =
filter_revealed_v2_proof_nodes(nodes, &mut self.revealed_account_paths)?;
#[cfg(feature = "metrics")]
@@ -476,7 +501,7 @@ where
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", total_nodes = ?nodes.len(), "Revealing account nodes from V2 proof");
trie.reveal_nodes(nodes)?;
trie.reveal_nodes(&mut nodes)?;
Ok(())
}
@@ -517,7 +542,7 @@ where
trie: &mut RevealableSparseTrie<S>,
retain_updates: bool,
) -> SparseStateTrieResult<ProofNodesMetricValues> {
let FilteredV2ProofNodes { root_node, nodes, new_nodes, metric_values } =
let FilteredV2ProofNodes { root_node, mut nodes, new_nodes, metric_values } =
filter_revealed_v2_proof_nodes(nodes, revealed_nodes)?;
let trie = if let Some(root_node) = root_node {
@@ -530,7 +555,7 @@ where
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", ?account, total_nodes = ?nodes.len(), "Revealing storage nodes from V2 proof");
trie.reveal_nodes(nodes)?;
trie.reveal_nodes(&mut nodes)?;
Ok(metric_values)
}
@@ -579,7 +604,7 @@ where
trie: &mut RevealableSparseTrie<S>,
retain_updates: bool,
) -> SparseStateTrieResult<ProofNodesMetricValues> {
let FilterMappedProofNodes { root_node, nodes, new_nodes, metric_values } =
let FilterMappedProofNodes { root_node, mut nodes, new_nodes, metric_values } =
filter_map_revealed_nodes(
storage_subtree.subtree,
revealed_nodes,
@@ -596,7 +621,7 @@ where
trie.reserve_nodes(new_nodes);
trace!(target: "trie::sparse", ?account, total_nodes = ?nodes.len(), "Revealing storage nodes");
trie.reveal_nodes(nodes)?;
trie.reveal_nodes(&mut nodes)?;
}
Ok(metric_values)

View File

@@ -2,7 +2,7 @@
use core::fmt::Debug;
use alloc::{borrow::Cow, vec, vec::Vec};
use alloc::{borrow::Cow, vec::Vec};
use alloy_primitives::{
map::{B256Map, HashMap, HashSet},
B256,
@@ -102,7 +102,7 @@ pub trait SparseTrie: Sized + Debug + Send + Sync {
node: TrieNode,
masks: Option<BranchNodeMasks>,
) -> SparseTrieResult<()> {
self.reveal_nodes(vec![ProofTrieNode { path, node, masks }])
self.reveal_nodes(&mut [ProofTrieNode { path, node, masks }])
}
/// Reveals one or more trie nodes if they have not been revealed before.
@@ -119,7 +119,7 @@ pub trait SparseTrie: Sized + Debug + Send + Sync {
/// # Returns
///
/// `Ok(())` if successful, or an error if any of the nodes was not revealed.
fn reveal_nodes(&mut self, nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()>;
fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNode]) -> SparseTrieResult<()>;
/// Updates the value of a leaf node at the specified path.
///

View File

@@ -623,10 +623,14 @@ impl SparseTrieTrait for SerialSparseTrie {
Ok(())
}
fn reveal_nodes(&mut self, mut nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNode]) -> SparseTrieResult<()> {
nodes.sort_unstable_by_key(|node| node.path);
for node in nodes {
self.reveal_node(node.path, node.node, node.masks)?;
for node in nodes.iter_mut() {
self.reveal_node(
node.path,
core::mem::replace(&mut node.node, TrieNode::EmptyRoot),
node.masks.take(),
)?;
}
Ok(())
}