perf(trie): parallelize merge_ancestors_into_overlay (#21202)

2026-02-19 03:04:27 -05:00 · 2026-01-21 21:08:03 +01:00
parent ec50fd40b3
commit 7609deddda
5 changed files with 273 additions and 3 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -14636,3 +14636,138 @@ dependencies = [
 "cc",
 "pkg-config",
 ]
+
+[[patch.unused]]
+name = "alloy-consensus"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-contract"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-eips"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-genesis"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-json-rpc"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-network"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-network-primitives"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-provider"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-pubsub"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-client"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-admin"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-anvil"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-beacon"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-debug"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-engine"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-eth"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-mev"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-trace"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-rpc-types-txpool"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-serde"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-signer"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-signer-local"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-transport"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-transport-http"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-transport-ipc"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
+
+[[patch.unused]]
+name = "alloy-transport-ws"
+version = "1.5.1"
+source = "git+https://github.com/alloy-rs/alloy?branch=main#05fd66e6f05399b71dfc9c802e6ee182b19e8575"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -797,3 +797,32 @@ ipnet = "2.11"

 # alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
 # alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
+
+# Patched by patch-alloy.sh
+alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
+alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
--- a/crates/chain-state/src/deferred_trie.rs
+++ b/crates/chain-state/src/deferred_trie.rs
@@ -243,8 +243,53 @@ impl DeferredTrieData {
    /// In normal operation, the parent always has a cached overlay and this
    /// function is never called.
    ///
-    /// Iterates ancestors oldest -> newest, then extends with current block's data,
-    /// so later state takes precedence.
+    /// When the `rayon` feature is enabled, uses parallel collection and merge:
+    /// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
+    /// 2. Merges hashed state and trie updates in parallel with each other
+    /// 3. Uses tree reduction within each merge for O(log n) depth
+    #[cfg(feature = "rayon")]
+    fn merge_ancestors_into_overlay(
+        ancestors: &[Self],
+        sorted_hashed_state: &HashedPostStateSorted,
+        sorted_trie_updates: &TrieUpdatesSorted,
+    ) -> TrieInputSorted {
+        // Early exit: no ancestors means just wrap current block's data
+        if ancestors.is_empty() {
+            return TrieInputSorted::new(
+                Arc::new(sorted_trie_updates.clone()),
+                Arc::new(sorted_hashed_state.clone()),
+                Default::default(),
+            );
+        }
+
+        // Collect ancestor data, unzipping states and updates into Arc slices
+        let (states, updates): (Vec<_>, Vec<_>) = ancestors
+            .iter()
+            .map(|a| {
+                let data = a.wait_cloned();
+                (data.hashed_state, data.trie_updates)
+            })
+            .unzip();
+
+        // Merge state and nodes in parallel with each other using tree reduction
+        let (state, nodes) = rayon::join(
+            || {
+                let mut merged = HashedPostStateSorted::merge_parallel(&states);
+                merged.extend_ref_and_sort(sorted_hashed_state);
+                merged
+            },
+            || {
+                let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
+                merged.extend_ref_and_sort(sorted_trie_updates);
+                merged
+            },
+        );
+
+        TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
+    }
+
+    /// Merge all ancestors and current block's data into a single overlay (sequential fallback).
+    #[cfg(not(feature = "rayon"))]
    fn merge_ancestors_into_overlay(
        ancestors: &[Self],
        sorted_hashed_state: &HashedPostStateSorted,
--- a/crates/trie/common/src/hashed_state.rs
+++ b/crates/trie/common/src/hashed_state.rs
@@ -6,7 +6,7 @@ use crate::{
    utils::{extend_sorted_vec, kway_merge_sorted},
    KeyHasher, MultiProofTargets, Nibbles,
 };
-use alloc::{borrow::Cow, vec::Vec};
+use alloc::{borrow::Cow, sync::Arc, vec::Vec};
 use alloy_primitives::{
    keccak256,
    map::{hash_map, B256Map, HashMap, HashSet},
@@ -710,6 +710,36 @@ impl HashedPostStateSorted {
        self.accounts.clear();
        self.storages.clear();
    }
+
+    /// Parallel batch-merge sorted hashed post states. Slice is **oldest to newest**.
+    ///
+    /// This is more efficient than sequential `extend_ref` calls when merging many states,
+    /// as it processes all states in parallel with tree reduction using divide-and-conquer.
+    #[cfg(feature = "rayon")]
+    pub fn merge_parallel(states: &[Arc<Self>]) -> Self {
+        fn parallel_merge_tree(states: &[Arc<HashedPostStateSorted>]) -> HashedPostStateSorted {
+            match states.len() {
+                0 => HashedPostStateSorted::default(),
+                1 => states[0].as_ref().clone(),
+                2 => {
+                    let mut acc = states[0].as_ref().clone();
+                    acc.extend_ref_and_sort(&states[1]);
+                    acc
+                }
+                n => {
+                    let mid = n / 2;
+                    let (mut left, right) = rayon::join(
+                        || parallel_merge_tree(&states[..mid]),
+                        || parallel_merge_tree(&states[mid..]),
+                    );
+                    left.extend_ref_and_sort(&right);
+                    left
+                }
+            }
+        }
+
+        parallel_merge_tree(states)
+    }
 }

 impl AsRef<Self> for HashedPostStateSorted {
--- a/crates/trie/common/src/updates.rs
+++ b/crates/trie/common/src/updates.rs
@@ -4,6 +4,7 @@ use crate::{
 };
 use alloc::{
    collections::{btree_map::BTreeMap, btree_set::BTreeSet},
+    sync::Arc,
    vec::Vec,
 };
 use alloy_primitives::{
@@ -697,6 +698,36 @@ impl TrieUpdatesSorted {

        Self { account_nodes, storage_tries }.into()
    }
+
+    /// Parallel batch-merge sorted trie updates. Slice is **oldest to newest**.
+    ///
+    /// This is more efficient than sequential `extend_ref` calls when merging many updates,
+    /// as it processes all updates in parallel with tree reduction using divide-and-conquer.
+    #[cfg(feature = "rayon")]
+    pub fn merge_parallel(updates: &[Arc<Self>]) -> Self {
+        fn parallel_merge_tree(updates: &[Arc<TrieUpdatesSorted>]) -> TrieUpdatesSorted {
+            match updates.len() {
+                0 => TrieUpdatesSorted::default(),
+                1 => updates[0].as_ref().clone(),
+                2 => {
+                    let mut acc = updates[0].as_ref().clone();
+                    acc.extend_ref_and_sort(&updates[1]);
+                    acc
+                }
+                n => {
+                    let mid = n / 2;
+                    let (mut left, right) = rayon::join(
+                        || parallel_merge_tree(&updates[..mid]),
+                        || parallel_merge_tree(&updates[mid..]),
+                    );
+                    left.extend_ref_and_sort(&right);
+                    left
+                }
+            }
+        }
+
+        parallel_merge_tree(updates)
+    }
 }

 impl AsRef<Self> for TrieUpdatesSorted {