diff --git a/Cargo.lock b/Cargo.lock index d8c2fc5bb4..836aea5f87 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5149,6 +5149,7 @@ dependencies = [ "tiny-keccak", "tokio", "tokio-stream", + "tracing", "triehash", "url", ] diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index e76268fc49..386d22a1c3 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -36,6 +36,9 @@ secp256k1 = { version = "0.27.0", default-features = false, features = [ # used for forkid crc = "3" +# tracing +tracing = "0.1" + # tokio tokio = { version = "1", default-features = false, features = ["sync"] } tokio-stream = "0.1" diff --git a/crates/primitives/src/checkpoints.rs b/crates/primitives/src/checkpoints.rs index dd0f8e93c5..ee9baad7c0 100644 --- a/crates/primitives/src/checkpoints.rs +++ b/crates/primitives/src/checkpoints.rs @@ -1,5 +1,5 @@ use crate::{ - trie::{HashBuilderState, StoredSubNode}, + trie::{hash_builder::HashBuilderState, StoredSubNode}, Address, H256, }; use bytes::Buf; diff --git a/crates/trie/src/hash_builder.rs b/crates/primitives/src/trie/hash_builder/mod.rs similarity index 98% rename from crates/trie/src/hash_builder.rs rename to crates/primitives/src/trie/hash_builder/mod.rs index 0e071323b3..34b3b702fd 100644 --- a/crates/trie/src/hash_builder.rs +++ b/crates/primitives/src/trie/hash_builder/mod.rs @@ -1,15 +1,16 @@ -use crate::{ +use super::{ nodes::{rlp_hash, BranchNode, ExtensionNode, LeafNode}, - Nibbles, -}; -use reth_primitives::{ - keccak256, - proofs::EMPTY_ROOT, - trie::{BranchNodeCompact, HashBuilderState, HashBuilderValue, TrieMask}, - H256, + BranchNodeCompact, Nibbles, TrieMask, }; +use crate::{keccak256, proofs::EMPTY_ROOT, H256}; use std::{collections::HashMap, fmt::Debug}; +mod state; +pub use state::HashBuilderState; + +mod value; +pub use value::HashBuilderValue; + /// A component used to construct the root hash of the trie. The primary purpose of a Hash Builder /// is to build the Merkle proof that is essential for verifying the integrity and authenticity of /// the trie's contents. It achieves this by constructing the root hash from the hashes of child @@ -414,8 +415,8 @@ impl HashBuilder { #[cfg(test)] mod tests { use super::*; + use crate::{hex_literal::hex, proofs::KeccakHasher, H256, U256}; use proptest::prelude::*; - use reth_primitives::{hex_literal::hex, proofs::KeccakHasher, H256, U256}; use std::collections::{BTreeMap, HashMap}; fn trie_root(iter: I) -> H256 @@ -588,7 +589,7 @@ mod tests { // Manually create the branch node that should be there after the first 2 leaves are added. // Skip the 0th element given in this example they have a common prefix and will // collapse to a Branch node. - use reth_primitives::bytes::BytesMut; + use crate::bytes::BytesMut; use reth_rlp::Encodable; let leaf1 = LeafNode::new(&Nibbles::unpack(&raw_input[0].0[1..]), input[0].1); let leaf2 = LeafNode::new(&Nibbles::unpack(&raw_input[1].0[1..]), input[1].1); diff --git a/crates/primitives/src/trie/hash_builder.rs b/crates/primitives/src/trie/hash_builder/state.rs similarity index 67% rename from crates/primitives/src/trie/hash_builder.rs rename to crates/primitives/src/trie/hash_builder/state.rs index 6944b2a788..fef54726bb 100644 --- a/crates/primitives/src/trie/hash_builder.rs +++ b/crates/primitives/src/trie/hash_builder/state.rs @@ -1,5 +1,4 @@ -use super::TrieMask; -use crate::H256; +use super::{super::TrieMask, HashBuilderValue}; use bytes::Buf; use reth_codecs::{derive_arbitrary, Compact}; use serde::{Deserialize, Serialize}; @@ -114,84 +113,6 @@ impl Compact for HashBuilderState { } } -/// The current value of the hash builder. -#[derive_arbitrary(compact)] -#[derive(Clone, PartialEq, Serialize, Deserialize)] -pub enum HashBuilderValue { - /// Value of the leaf node. - Hash(H256), - /// Hash of adjacent nodes. - Bytes(Vec), -} - -impl Compact for HashBuilderValue { - fn to_compact(self, buf: &mut B) -> usize - where - B: bytes::BufMut + AsMut<[u8]>, - { - match self { - Self::Hash(hash) => { - buf.put_u8(0); - 1 + hash.to_compact(buf) - } - Self::Bytes(bytes) => { - buf.put_u8(1); - 1 + bytes.to_compact(buf) - } - } - } - - fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) - where - Self: Sized, - { - match buf[0] { - 0 => { - let (hash, buf) = H256::from_compact(&buf[1..], 32); - (Self::Hash(hash), buf) - } - 1 => { - let (bytes, buf) = Vec::from_compact(&buf[1..], 0); - (Self::Bytes(bytes), buf) - } - _ => panic!("Invalid hash builder value"), - } - } -} - -impl std::fmt::Debug for HashBuilderValue { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Bytes(bytes) => write!(f, "Bytes({:?})", hex::encode(bytes)), - Self::Hash(hash) => write!(f, "Hash({:?})", hash), - } - } -} - -impl From> for HashBuilderValue { - fn from(value: Vec) -> Self { - Self::Bytes(value) - } -} - -impl From<&[u8]> for HashBuilderValue { - fn from(value: &[u8]) -> Self { - Self::Bytes(value.to_vec()) - } -} - -impl From for HashBuilderValue { - fn from(value: H256) -> Self { - Self::Hash(value) - } -} - -impl Default for HashBuilderValue { - fn default() -> Self { - Self::Bytes(vec![]) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/crates/primitives/src/trie/hash_builder/value.rs b/crates/primitives/src/trie/hash_builder/value.rs new file mode 100644 index 0000000000..ae6e83f3e5 --- /dev/null +++ b/crates/primitives/src/trie/hash_builder/value.rs @@ -0,0 +1,81 @@ +use crate::H256; +use reth_codecs::{derive_arbitrary, Compact}; +use serde::{Deserialize, Serialize}; + +/// The current value of the hash builder. +#[derive_arbitrary(compact)] +#[derive(Clone, PartialEq, Serialize, Deserialize)] +pub enum HashBuilderValue { + /// Value of the leaf node. + Hash(H256), + /// Hash of adjacent nodes. + Bytes(Vec), +} + +impl Compact for HashBuilderValue { + fn to_compact(self, buf: &mut B) -> usize + where + B: bytes::BufMut + AsMut<[u8]>, + { + match self { + Self::Hash(hash) => { + buf.put_u8(0); + 1 + hash.to_compact(buf) + } + Self::Bytes(bytes) => { + buf.put_u8(1); + 1 + bytes.to_compact(buf) + } + } + } + + fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) + where + Self: Sized, + { + match buf[0] { + 0 => { + let (hash, buf) = H256::from_compact(&buf[1..], 32); + (Self::Hash(hash), buf) + } + 1 => { + let (bytes, buf) = Vec::from_compact(&buf[1..], 0); + (Self::Bytes(bytes), buf) + } + _ => panic!("Invalid hash builder value"), + } + } +} + +impl std::fmt::Debug for HashBuilderValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bytes(bytes) => write!(f, "Bytes({:?})", hex::encode(bytes)), + Self::Hash(hash) => write!(f, "Hash({:?})", hash), + } + } +} + +impl From> for HashBuilderValue { + fn from(value: Vec) -> Self { + Self::Bytes(value) + } +} + +impl From<&[u8]> for HashBuilderValue { + fn from(value: &[u8]) -> Self { + Self::Bytes(value.to_vec()) + } +} + +impl From for HashBuilderValue { + fn from(value: H256) -> Self { + Self::Hash(value) + } +} + +impl Default for HashBuilderValue { + fn default() -> Self { + Self::Bytes(vec![]) + } +} diff --git a/crates/primitives/src/trie/mod.rs b/crates/primitives/src/trie/mod.rs index 34a92c8824..4252aa39b4 100644 --- a/crates/primitives/src/trie/mod.rs +++ b/crates/primitives/src/trie/mod.rs @@ -1,17 +1,21 @@ //! Collection of trie related types. -mod branch_node; -mod hash_builder; +/// Various branch nodes produced by the hash builder. +pub mod nodes; +pub use nodes::BranchNodeCompact; + +/// The implementation of hash builder. +pub mod hash_builder; +pub use hash_builder::HashBuilder; + mod mask; mod nibbles; mod storage; mod subnode; pub use self::{ - branch_node::BranchNodeCompact, - hash_builder::{HashBuilderState, HashBuilderValue}, mask::TrieMask, - nibbles::{StoredNibbles, StoredNibblesSubKey}, + nibbles::{Nibbles, StoredNibbles, StoredNibblesSubKey}, storage::StorageTrieEntry, subnode::StoredSubNode, }; diff --git a/crates/primitives/src/trie/nibbles.rs b/crates/primitives/src/trie/nibbles.rs index 65bbf2df5e..9842b7f980 100644 --- a/crates/primitives/src/trie/nibbles.rs +++ b/crates/primitives/src/trie/nibbles.rs @@ -1,6 +1,7 @@ use crate::Bytes; -use derive_more::Deref; +use derive_more::{Deref, DerefMut, From, Index}; use reth_codecs::{main_codec, Compact}; +use reth_rlp::RlpEncodableWrapper; use serde::{Deserialize, Serialize}; /// The nibbles are the keys for the AccountsTrie and the subkeys for the StorageTrie. @@ -49,3 +50,298 @@ impl Compact for StoredNibblesSubKey { (Self(StoredNibbles { inner }), &buf[65..]) } } + +/// Structure representing a sequence of nibbles. +/// +/// A nibble is a 4-bit value, and this structure is used to store +/// the nibble sequence representing the keys in a Merkle Patricia Trie (MPT). +/// Using nibbles simplifies trie operations and enables consistent key +/// representation in the MPT. +/// +/// The `hex_data` field is a `Vec` that stores the nibbles, with each +/// `u8` value containing a single nibble. This means that each byte in +/// `hex_data` has its upper 4 bits set to zero and the lower 4 bits +/// representing the nibble value. +#[derive( + Default, + Clone, + Eq, + PartialEq, + RlpEncodableWrapper, + PartialOrd, + Ord, + Hash, + Index, + From, + Deref, + DerefMut, +)] +pub struct Nibbles { + /// The inner representation of the nibble sequence. + pub hex_data: Vec, +} + +impl From<&[u8]> for Nibbles { + fn from(slice: &[u8]) -> Self { + Nibbles::from_hex(slice.to_vec()) + } +} + +impl From<&[u8; N]> for Nibbles { + fn from(arr: &[u8; N]) -> Self { + Nibbles::from_hex(arr.to_vec()) + } +} + +impl std::fmt::Debug for Nibbles { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Nibbles").field("hex_data", &hex::encode(&self.hex_data)).finish() + } +} + +impl Nibbles { + /// Creates a new [Nibbles] instance from bytes. + pub fn from_hex(hex: Vec) -> Self { + Nibbles { hex_data: hex } + } + + /// Take a byte array (slice or vector) as input and convert it into a [Nibbles] struct + /// containing the nibbles (half-bytes or 4 bits) that make up the input byte data. + pub fn unpack>(data: T) -> Self { + Nibbles { hex_data: data.as_ref().iter().flat_map(|item| [item / 16, item % 16]).collect() } + } + + /// Packs the nibbles stored in the struct into a byte vector. + /// + /// This method combines each pair of consecutive nibbles into a single byte, + /// effectively reducing the size of the data by a factor of two. + /// If the number of nibbles is odd, the last nibble is shifted left by 4 bits and + /// added to the packed byte vector. + pub fn pack(&self) -> Vec { + let length = (self.len() + 1) / 2; + if length == 0 { + Vec::new() + } else { + self.iter() + .enumerate() + .filter_map(|(index, nibble)| { + if index % 2 == 0 { + let next_nibble = self.get(index + 1).unwrap_or(&0); + Some((*nibble << 4) + *next_nibble) + } else { + None + } + }) + .collect() + } + } + + /// Encodes a given path leaf as a compact array of bytes, where each byte represents two + /// "nibbles" (half-bytes or 4 bits) of the original hex data, along with additional information + /// about the leaf itself. + /// + /// The method takes the following input: + /// `is_leaf`: A boolean value indicating whether the current node is a leaf node or not. + /// + /// The first byte of the encoded vector is set based on the `is_leaf` flag and the parity of + /// the hex data length (even or odd number of nibbles). + /// - If the node is an extension with even length, the header byte is `0x00`. + /// - If the node is an extension with odd length, the header byte is `0x10 + `. + /// - If the node is a leaf with even length, the header byte is `0x20`. + /// - If the node is a leaf with odd length, the header byte is `0x30 + `. + /// + /// If there is an odd number of nibbles, store the first nibble in the lower 4 bits of the + /// first byte of encoded. + /// + /// # Returns + /// + /// A `Vec` containing the compact byte representation of the nibble sequence, including the + /// header byte. + /// + /// # Example + /// + /// ``` + /// # use reth_primitives::trie::Nibbles; + /// + /// // Extension node with an even path length: + /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]); + /// assert_eq!(nibbles.encode_path_leaf(false), vec![0x00, 0xAB, 0xCD]); + /// + /// // Extension node with an odd path length: + /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]); + /// assert_eq!(nibbles.encode_path_leaf(false), vec![0x1A, 0xBC]); + /// + /// // Leaf node with an even path length: + /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]); + /// assert_eq!(nibbles.encode_path_leaf(true), vec![0x20, 0xAB, 0xCD]); + /// + /// // Leaf node with an odd path length: + /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]); + /// assert_eq!(nibbles.encode_path_leaf(true), vec![0x3A, 0xBC]); + /// ``` + pub fn encode_path_leaf(&self, is_leaf: bool) -> Vec { + let mut encoded = vec![0u8; self.len() / 2 + 1]; + let odd_nibbles = self.len() % 2 != 0; + + // Set the first byte of the encoded vector. + encoded[0] = match (is_leaf, odd_nibbles) { + (true, true) => 0x30 | self[0], + (true, false) => 0x20, + (false, true) => 0x10 | self[0], + (false, false) => 0x00, + }; + + let mut nibble_idx = if odd_nibbles { 1 } else { 0 }; + for byte in encoded.iter_mut().skip(1) { + *byte = (self[nibble_idx] << 4) + self[nibble_idx + 1]; + nibble_idx += 2; + } + + encoded + } + + /// Increments the nibble sequence by one. + pub fn increment(&self) -> Option { + let mut incremented = self.hex_data.clone(); + + for nibble in incremented.iter_mut().rev() { + assert!(*nibble < 0x10); + if *nibble < 0xf { + *nibble += 1; + return Some(Nibbles::from(incremented)) + } else { + *nibble = 0; + } + } + + None + } + + /// The last element of the hex vector is used to determine whether the nibble sequence + /// represents a leaf or an extension node. If the last element is 0x10 (16), then it's a leaf. + pub fn is_leaf(&self) -> bool { + self.hex_data[self.hex_data.len() - 1] == 16 + } + + /// Returns `true` if the current nibble sequence starts with the given prefix. + pub fn has_prefix(&self, other: &Self) -> bool { + self.starts_with(other) + } + + /// Returns the nibble at the given index. + pub fn at(&self, i: usize) -> usize { + self.hex_data[i] as usize + } + + /// Returns the last nibble of the current nibble sequence. + pub fn last(&self) -> Option { + self.hex_data.last().copied() + } + + /// Returns the length of the common prefix between the current nibble sequence and the given. + pub fn common_prefix_length(&self, other: &Nibbles) -> usize { + let len = std::cmp::min(self.len(), other.len()); + for i in 0..len { + if self[i] != other[i] { + return i + } + } + len + } + + /// Slice the current nibbles from the given start index to the end. + pub fn slice_from(&self, index: usize) -> Nibbles { + self.slice(index, self.hex_data.len()) + } + + /// Slice the current nibbles within the provided index range. + pub fn slice(&self, start: usize, end: usize) -> Nibbles { + Nibbles::from_hex(self.hex_data[start..end].to_vec()) + } + + /// Join two nibbles together. + pub fn join(&self, b: &Nibbles) -> Nibbles { + let mut hex_data = Vec::with_capacity(self.len() + b.len()); + hex_data.extend_from_slice(self); + hex_data.extend_from_slice(b); + Nibbles::from_hex(hex_data) + } + + /// Extend the current nibbles with another nibbles. + pub fn extend(&mut self, b: impl AsRef<[u8]>) { + self.hex_data.extend_from_slice(b.as_ref()); + } + + /// Truncate the current nibbles to the given length. + pub fn truncate(&mut self, len: usize) { + self.hex_data.truncate(len) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use proptest::prelude::*; + + #[test] + fn hashed_regression() { + let nibbles = hex::decode("05010406040a040203030f010805020b050c04070003070e0909070f010b0a0805020301070c0a0902040b0f000f0006040a04050f020b090701000a0a040b").unwrap(); + let nibbles = Nibbles::from(nibbles); + let path = nibbles.encode_path_leaf(true); + let expected = + hex::decode("351464a4233f1852b5c47037e997f1ba852317ca924bf0f064a45f2b9710aa4b") + .unwrap(); + assert_eq!(path, expected); + } + + #[test] + fn pack_nibbles() { + for (input, expected) in [ + (vec![], vec![]), + (vec![0xa], vec![0xa0]), + (vec![0xa, 0xb], vec![0xab]), + (vec![0xa, 0xb, 0x2], vec![0xab, 0x20]), + (vec![0xa, 0xb, 0x2, 0x0], vec![0xab, 0x20]), + (vec![0xa, 0xb, 0x2, 0x7], vec![0xab, 0x27]), + ] { + let nibbles = Nibbles::from(input); + let encoded = nibbles.pack(); + assert_eq!(encoded, expected); + } + } + + proptest! { + #[test] + fn pack_unpack_roundtrip(input in any::>()) { + let nibbles = Nibbles::unpack(&input); + let packed = nibbles.pack(); + prop_assert_eq!(packed, input); + } + + #[test] + fn encode_path_first_byte(input in any::>()) { + prop_assume!(!input.is_empty()); + let input = Nibbles::unpack(input); + let input_is_odd = input.len() % 2 == 1; + + let compact_leaf = input.encode_path_leaf(true); + let leaf_flag = compact_leaf[0]; + // Check flag + assert_ne!(leaf_flag & 0x20, 0); + assert_eq!(input_is_odd, (leaf_flag & 0x10) != 0); + if input_is_odd { + assert_eq!(leaf_flag & 0x0f, *input.first().unwrap()); + } + + + let compact_extension = input.encode_path_leaf(false); + let extension_flag = compact_extension[0]; + // Check first byte + assert_eq!(extension_flag & 0x20, 0); + assert_eq!(input_is_odd, (extension_flag & 0x10) != 0); + if input_is_odd { + assert_eq!(extension_flag & 0x0f, *input.first().unwrap()); + } + } + } +} diff --git a/crates/primitives/src/trie/branch_node.rs b/crates/primitives/src/trie/nodes/branch.rs similarity index 69% rename from crates/primitives/src/trie/branch_node.rs rename to crates/primitives/src/trie/nodes/branch.rs index b6c30caf17..36ee35eb98 100644 --- a/crates/primitives/src/trie/branch_node.rs +++ b/crates/primitives/src/trie/nodes/branch.rs @@ -1,9 +1,83 @@ -use super::TrieMask; +use super::{super::TrieMask, rlp_node, CHILD_INDEX_RANGE}; use crate::H256; use bytes::Buf; use reth_codecs::Compact; +use reth_rlp::{BufMut, EMPTY_STRING_CODE}; use serde::{Deserialize, Serialize}; +/// A Branch node is only a pointer to the stack of nodes and is used to +/// create the RLP encoding of the node using masks which filter from +/// the stack of nodes. +#[derive(Clone, Debug)] +pub struct BranchNode<'a> { + /// Rlp encoded children + pub stack: &'a [Vec], +} + +impl<'a> BranchNode<'a> { + /// Create a new branch node from the stack of nodes. + pub fn new(stack: &'a [Vec]) -> Self { + Self { stack } + } + + /// Given the hash and state mask of children present, return an iterator over the stack items + /// that match the mask. + pub fn children( + &self, + state_mask: TrieMask, + hash_mask: TrieMask, + ) -> impl Iterator + '_ { + let mut index = self.stack.len() - state_mask.count_ones() as usize; + CHILD_INDEX_RANGE.filter_map(move |digit| { + let mut child = None; + if state_mask.is_bit_set(digit) { + if hash_mask.is_bit_set(digit) { + child = Some(&self.stack[index]); + } + index += 1; + } + child.map(|child| H256::from_slice(&child[1..])) + }) + } + + /// Returns the RLP encoding of the branch node given the state mask of children present. + pub fn rlp(&self, state_mask: TrieMask, buf: &mut Vec) -> Vec { + let first_child_idx = self.stack.len() - state_mask.count_ones() as usize; + + // Create the RLP header from the mask elements present. + let mut i = first_child_idx; + let header = CHILD_INDEX_RANGE.fold( + reth_rlp::Header { list: true, payload_length: 1 }, + |mut header, digit| { + if state_mask.is_bit_set(digit) { + header.payload_length += self.stack[i].len(); + i += 1; + } else { + header.payload_length += 1; + } + header + }, + ); + header.encode(buf); + + // Extend the RLP buffer with the present children + let mut i = first_child_idx; + CHILD_INDEX_RANGE.for_each(|idx| { + if state_mask.is_bit_set(idx) { + buf.extend_from_slice(&self.stack[i]); + i += 1; + } else { + buf.put_u8(EMPTY_STRING_CODE) + } + }); + + // Is this needed? + buf.put_u8(EMPTY_STRING_CODE); + + rlp_node(buf) + } +} + /// A struct representing a branch node in an Ethereum trie. /// /// A branch node can have up to 16 children, each corresponding to one of the possible nibble diff --git a/crates/trie/src/nodes/extension.rs b/crates/primitives/src/trie/nodes/extension.rs similarity index 97% rename from crates/trie/src/nodes/extension.rs rename to crates/primitives/src/trie/nodes/extension.rs index 73f8467617..d124084bde 100644 --- a/crates/trie/src/nodes/extension.rs +++ b/crates/primitives/src/trie/nodes/extension.rs @@ -1,5 +1,4 @@ -use super::rlp_node; -use crate::Nibbles; +use super::{super::Nibbles, rlp_node}; use reth_rlp::{BufMut, Encodable}; /// An intermediate node that exists solely to compress the trie's paths. It contains a path segment diff --git a/crates/trie/src/nodes/leaf.rs b/crates/primitives/src/trie/nodes/leaf.rs similarity index 96% rename from crates/trie/src/nodes/leaf.rs rename to crates/primitives/src/trie/nodes/leaf.rs index 9a7559f513..cbf3ecd551 100644 --- a/crates/trie/src/nodes/leaf.rs +++ b/crates/primitives/src/trie/nodes/leaf.rs @@ -1,5 +1,4 @@ -use super::rlp_node; -use crate::Nibbles; +use super::{super::Nibbles, rlp_node}; use reth_rlp::{BufMut, Encodable}; /// A leaf node represents the endpoint or terminal node in the trie. In other words, a leaf node is @@ -55,7 +54,7 @@ impl std::fmt::Debug for LeafNode<'_> { #[cfg(test)] mod tests { use super::*; - use reth_primitives::hex_literal::hex; + use crate::hex_literal::hex; // From manual regression test #[test] diff --git a/crates/trie/src/nodes/mod.rs b/crates/primitives/src/trie/nodes/mod.rs similarity index 80% rename from crates/trie/src/nodes/mod.rs rename to crates/primitives/src/trie/nodes/mod.rs index 5cac694d50..df38b58faa 100644 --- a/crates/trie/src/nodes/mod.rs +++ b/crates/primitives/src/trie/nodes/mod.rs @@ -1,4 +1,4 @@ -use reth_primitives::{keccak256, H256}; +use crate::{keccak256, H256}; use reth_rlp::EMPTY_STRING_CODE; use std::ops::Range; @@ -6,7 +6,11 @@ mod branch; mod extension; mod leaf; -pub use self::{branch::BranchNode, extension::ExtensionNode, leaf::LeafNode}; +pub use self::{ + branch::{BranchNode, BranchNodeCompact}, + extension::ExtensionNode, + leaf::LeafNode, +}; /// The range of valid child indexes. pub const CHILD_INDEX_RANGE: Range = 0..16; diff --git a/crates/trie/benches/prefix_set.rs b/crates/trie/benches/prefix_set.rs index a79f3e9f13..13e9c7f797 100644 --- a/crates/trie/benches/prefix_set.rs +++ b/crates/trie/benches/prefix_set.rs @@ -6,7 +6,8 @@ use proptest::{ strategy::{Strategy, ValueTree}, test_runner::{basic_result_cache, TestRunner}, }; -use reth_trie::{prefix_set::PrefixSet, Nibbles}; +use reth_primitives::trie::Nibbles; +use reth_trie::prefix_set::PrefixSet; use std::collections::BTreeSet; pub trait PrefixSetAbstraction: Default { diff --git a/crates/trie/src/hashed_cursor/post_state.rs b/crates/trie/src/hashed_cursor/post_state.rs index affdf0b1f7..f43ab20a1c 100644 --- a/crates/trie/src/hashed_cursor/post_state.rs +++ b/crates/trie/src/hashed_cursor/post_state.rs @@ -1,12 +1,11 @@ -use crate::{prefix_set::PrefixSet, Nibbles}; - use super::{HashedAccountCursor, HashedCursorFactory, HashedStorageCursor}; +use crate::prefix_set::PrefixSet; use reth_db::{ cursor::{DbCursorRO, DbDupCursorRO}, tables, transaction::{DbTx, DbTxGAT}, }; -use reth_primitives::{Account, StorageEntry, H256, U256}; +use reth_primitives::{trie::Nibbles, Account, StorageEntry, H256, U256}; use std::collections::{BTreeMap, HashMap}; /// The post state account storage with hashed slots. diff --git a/crates/trie/src/lib.rs b/crates/trie/src/lib.rs index 8bff7984f2..ea45b5fcad 100644 --- a/crates/trie/src/lib.rs +++ b/crates/trie/src/lib.rs @@ -9,18 +9,9 @@ //! authenticated radix trie that is used to store key-value bindings. //! -mod nibbles; -pub use nibbles::Nibbles; - /// The Ethereum account as represented in the trie. pub mod account; -/// Various branch nodes produced by the hash builder. -pub mod nodes; - -/// The implementation of hash builder. -pub mod hash_builder; - /// The implementation of a container for storing intermediate changes to a trie. /// The container indicates when the trie has been modified. pub mod prefix_set; diff --git a/crates/trie/src/nibbles.rs b/crates/trie/src/nibbles.rs deleted file mode 100644 index fd3660ecb7..0000000000 --- a/crates/trie/src/nibbles.rs +++ /dev/null @@ -1,297 +0,0 @@ -use derive_more::{Deref, DerefMut, From, Index}; -use reth_rlp::RlpEncodableWrapper; - -/// Structure representing a sequence of nibbles. -/// -/// A nibble is a 4-bit value, and this structure is used to store -/// the nibble sequence representing the keys in a Merkle Patricia Trie (MPT). -/// Using nibbles simplifies trie operations and enables consistent key -/// representation in the MPT. -/// -/// The `hex_data` field is a `Vec` that stores the nibbles, with each -/// `u8` value containing a single nibble. This means that each byte in -/// `hex_data` has its upper 4 bits set to zero and the lower 4 bits -/// representing the nibble value. -#[derive( - Default, - Clone, - Eq, - PartialEq, - RlpEncodableWrapper, - PartialOrd, - Ord, - Hash, - Index, - From, - Deref, - DerefMut, -)] -pub struct Nibbles { - /// The inner representation of the nibble sequence. - pub hex_data: Vec, -} - -impl From<&[u8]> for Nibbles { - fn from(slice: &[u8]) -> Self { - Nibbles::from_hex(slice.to_vec()) - } -} - -impl From<&[u8; N]> for Nibbles { - fn from(arr: &[u8; N]) -> Self { - Nibbles::from_hex(arr.to_vec()) - } -} - -impl std::fmt::Debug for Nibbles { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Nibbles").field("hex_data", &hex::encode(&self.hex_data)).finish() - } -} - -impl Nibbles { - /// Creates a new [Nibbles] instance from bytes. - pub fn from_hex(hex: Vec) -> Self { - Nibbles { hex_data: hex } - } - - /// Take a byte array (slice or vector) as input and convert it into a [Nibbles] struct - /// containing the nibbles (half-bytes or 4 bits) that make up the input byte data. - pub fn unpack>(data: T) -> Self { - Nibbles { hex_data: data.as_ref().iter().flat_map(|item| [item / 16, item % 16]).collect() } - } - - /// Packs the nibbles stored in the struct into a byte vector. - /// - /// This method combines each pair of consecutive nibbles into a single byte, - /// effectively reducing the size of the data by a factor of two. - /// If the number of nibbles is odd, the last nibble is shifted left by 4 bits and - /// added to the packed byte vector. - pub fn pack(&self) -> Vec { - let length = (self.len() + 1) / 2; - if length == 0 { - Vec::new() - } else { - self.iter() - .enumerate() - .filter_map(|(index, nibble)| { - if index % 2 == 0 { - let next_nibble = self.get(index + 1).unwrap_or(&0); - Some((*nibble << 4) + *next_nibble) - } else { - None - } - }) - .collect() - } - } - - /// Encodes a given path leaf as a compact array of bytes, where each byte represents two - /// "nibbles" (half-bytes or 4 bits) of the original hex data, along with additional information - /// about the leaf itself. - /// - /// The method takes the following input: - /// `is_leaf`: A boolean value indicating whether the current node is a leaf node or not. - /// - /// The first byte of the encoded vector is set based on the `is_leaf` flag and the parity of - /// the hex data length (even or odd number of nibbles). - /// - If the node is an extension with even length, the header byte is `0x00`. - /// - If the node is an extension with odd length, the header byte is `0x10 + `. - /// - If the node is a leaf with even length, the header byte is `0x20`. - /// - If the node is a leaf with odd length, the header byte is `0x30 + `. - /// - /// If there is an odd number of nibbles, store the first nibble in the lower 4 bits of the - /// first byte of encoded. - /// - /// # Returns - /// - /// A `Vec` containing the compact byte representation of the nibble sequence, including the - /// header byte. - /// - /// # Example - /// - /// ``` - /// # use reth_trie::Nibbles; - /// - /// // Extension node with an even path length: - /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]); - /// assert_eq!(nibbles.encode_path_leaf(false), vec![0x00, 0xAB, 0xCD]); - /// - /// // Extension node with an odd path length: - /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]); - /// assert_eq!(nibbles.encode_path_leaf(false), vec![0x1A, 0xBC]); - /// - /// // Leaf node with an even path length: - /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]); - /// assert_eq!(nibbles.encode_path_leaf(true), vec![0x20, 0xAB, 0xCD]); - /// - /// // Leaf node with an odd path length: - /// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]); - /// assert_eq!(nibbles.encode_path_leaf(true), vec![0x3A, 0xBC]); - /// ``` - pub fn encode_path_leaf(&self, is_leaf: bool) -> Vec { - let mut encoded = vec![0u8; self.len() / 2 + 1]; - let odd_nibbles = self.len() % 2 != 0; - - // Set the first byte of the encoded vector. - encoded[0] = match (is_leaf, odd_nibbles) { - (true, true) => 0x30 | self[0], - (true, false) => 0x20, - (false, true) => 0x10 | self[0], - (false, false) => 0x00, - }; - - let mut nibble_idx = if odd_nibbles { 1 } else { 0 }; - for byte in encoded.iter_mut().skip(1) { - *byte = (self[nibble_idx] << 4) + self[nibble_idx + 1]; - nibble_idx += 2; - } - - encoded - } - - /// Increments the nibble sequence by one. - pub fn increment(&self) -> Option { - let mut incremented = self.hex_data.clone(); - - for nibble in incremented.iter_mut().rev() { - assert!(*nibble < 0x10); - if *nibble < 0xf { - *nibble += 1; - return Some(Nibbles::from(incremented)) - } else { - *nibble = 0; - } - } - - None - } - - /// The last element of the hex vector is used to determine whether the nibble sequence - /// represents a leaf or an extension node. If the last element is 0x10 (16), then it's a leaf. - pub fn is_leaf(&self) -> bool { - self.hex_data[self.hex_data.len() - 1] == 16 - } - - /// Returns `true` if the current nibble sequence starts with the given prefix. - pub fn has_prefix(&self, other: &Self) -> bool { - self.starts_with(other) - } - - /// Returns the nibble at the given index. - pub fn at(&self, i: usize) -> usize { - self.hex_data[i] as usize - } - - /// Returns the last nibble of the current nibble sequence. - pub fn last(&self) -> Option { - self.hex_data.last().copied() - } - - /// Returns the length of the common prefix between the current nibble sequence and the given. - pub fn common_prefix_length(&self, other: &Nibbles) -> usize { - let len = std::cmp::min(self.len(), other.len()); - for i in 0..len { - if self[i] != other[i] { - return i - } - } - len - } - - /// Slice the current nibbles from the given start index to the end. - pub fn slice_from(&self, index: usize) -> Nibbles { - self.slice(index, self.hex_data.len()) - } - - /// Slice the current nibbles within the provided index range. - pub fn slice(&self, start: usize, end: usize) -> Nibbles { - Nibbles::from_hex(self.hex_data[start..end].to_vec()) - } - - /// Join two nibbles together. - pub fn join(&self, b: &Nibbles) -> Nibbles { - let mut hex_data = Vec::with_capacity(self.len() + b.len()); - hex_data.extend_from_slice(self); - hex_data.extend_from_slice(b); - Nibbles::from_hex(hex_data) - } - - /// Extend the current nibbles with another nibbles. - pub fn extend(&mut self, b: impl AsRef<[u8]>) { - self.hex_data.extend_from_slice(b.as_ref()); - } - - /// Truncate the current nibbles to the given length. - pub fn truncate(&mut self, len: usize) { - self.hex_data.truncate(len) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use proptest::prelude::*; - - #[test] - fn hashed_regression() { - let nibbles = hex::decode("05010406040a040203030f010805020b050c04070003070e0909070f010b0a0805020301070c0a0902040b0f000f0006040a04050f020b090701000a0a040b").unwrap(); - let nibbles = Nibbles::from(nibbles); - let path = nibbles.encode_path_leaf(true); - let expected = - hex::decode("351464a4233f1852b5c47037e997f1ba852317ca924bf0f064a45f2b9710aa4b") - .unwrap(); - assert_eq!(path, expected); - } - - #[test] - fn pack_nibbles() { - for (input, expected) in [ - (vec![], vec![]), - (vec![0xa], vec![0xa0]), - (vec![0xa, 0xb], vec![0xab]), - (vec![0xa, 0xb, 0x2], vec![0xab, 0x20]), - (vec![0xa, 0xb, 0x2, 0x0], vec![0xab, 0x20]), - (vec![0xa, 0xb, 0x2, 0x7], vec![0xab, 0x27]), - ] { - let nibbles = Nibbles::from(input); - let encoded = nibbles.pack(); - assert_eq!(encoded, expected); - } - } - - proptest! { - #[test] - fn pack_unpack_roundtrip(input in any::>()) { - let nibbles = Nibbles::unpack(&input); - let packed = nibbles.pack(); - prop_assert_eq!(packed, input); - } - - #[test] - fn encode_path_first_byte(input in any::>()) { - prop_assume!(!input.is_empty()); - let input = Nibbles::unpack(input); - let input_is_odd = input.len() % 2 == 1; - - let compact_leaf = input.encode_path_leaf(true); - let leaf_flag = compact_leaf[0]; - // Check flag - assert_ne!(leaf_flag & 0x20, 0); - assert_eq!(input_is_odd, (leaf_flag & 0x10) != 0); - if input_is_odd { - assert_eq!(leaf_flag & 0x0f, *input.first().unwrap()); - } - - - let compact_extension = input.encode_path_leaf(false); - let extension_flag = compact_extension[0]; - // Check first byte - assert_eq!(extension_flag & 0x20, 0); - assert_eq!(input_is_odd, (extension_flag & 0x10) != 0); - if input_is_odd { - assert_eq!(extension_flag & 0x0f, *input.first().unwrap()); - } - } - } -} diff --git a/crates/trie/src/nodes/branch.rs b/crates/trie/src/nodes/branch.rs deleted file mode 100644 index 899b165f45..0000000000 --- a/crates/trie/src/nodes/branch.rs +++ /dev/null @@ -1,76 +0,0 @@ -use super::{rlp_node, CHILD_INDEX_RANGE}; -use reth_primitives::{trie::TrieMask, H256}; -use reth_rlp::{BufMut, EMPTY_STRING_CODE}; - -/// A Branch node is only a pointer to the stack of nodes and is used to -/// create the RLP encoding of the node using masks which filter from -/// the stack of nodes. -#[derive(Clone, Debug)] -pub struct BranchNode<'a> { - /// Rlp encoded children - pub stack: &'a [Vec], -} - -impl<'a> BranchNode<'a> { - /// Create a new branch node from the stack of nodes. - pub fn new(stack: &'a [Vec]) -> Self { - Self { stack } - } - - /// Given the hash and state mask of children present, return an iterator over the stack items - /// that match the mask. - pub fn children( - &self, - state_mask: TrieMask, - hash_mask: TrieMask, - ) -> impl Iterator + '_ { - let mut index = self.stack.len() - state_mask.count_ones() as usize; - CHILD_INDEX_RANGE.filter_map(move |digit| { - let mut child = None; - if state_mask.is_bit_set(digit) { - if hash_mask.is_bit_set(digit) { - child = Some(&self.stack[index]); - } - index += 1; - } - child.map(|child| H256::from_slice(&child[1..])) - }) - } - - /// Returns the RLP encoding of the branch node given the state mask of children present. - pub fn rlp(&self, state_mask: TrieMask, buf: &mut Vec) -> Vec { - let first_child_idx = self.stack.len() - state_mask.count_ones() as usize; - - // Create the RLP header from the mask elements present. - let mut i = first_child_idx; - let header = CHILD_INDEX_RANGE.fold( - reth_rlp::Header { list: true, payload_length: 1 }, - |mut header, digit| { - if state_mask.is_bit_set(digit) { - header.payload_length += self.stack[i].len(); - i += 1; - } else { - header.payload_length += 1; - } - header - }, - ); - header.encode(buf); - - // Extend the RLP buffer with the present children - let mut i = first_child_idx; - CHILD_INDEX_RANGE.for_each(|idx| { - if state_mask.is_bit_set(idx) { - buf.extend_from_slice(&self.stack[i]); - i += 1; - } else { - buf.put_u8(EMPTY_STRING_CODE) - } - }); - - // Is this needed? - buf.put_u8(EMPTY_STRING_CODE); - - rlp_node(buf) - } -} diff --git a/crates/trie/src/prefix_set/loader.rs b/crates/trie/src/prefix_set/loader.rs index 33a8569aa3..6d46e871f6 100644 --- a/crates/trie/src/prefix_set/loader.rs +++ b/crates/trie/src/prefix_set/loader.rs @@ -1,5 +1,4 @@ use super::PrefixSet; -use crate::Nibbles; use derive_more::Deref; use reth_db::{ cursor::DbCursorRO, @@ -8,7 +7,7 @@ use reth_db::{ transaction::DbTx, Error, }; -use reth_primitives::{keccak256, BlockNumber, StorageEntry, H256}; +use reth_primitives::{keccak256, trie::Nibbles, BlockNumber, StorageEntry, H256}; use std::{collections::HashMap, ops::RangeInclusive}; /// A wrapper around a database transaction that loads prefix sets within a given block range. diff --git a/crates/trie/src/prefix_set/mod.rs b/crates/trie/src/prefix_set/mod.rs index 02e7dfe4ed..e0101db129 100644 --- a/crates/trie/src/prefix_set/mod.rs +++ b/crates/trie/src/prefix_set/mod.rs @@ -1,4 +1,4 @@ -use crate::Nibbles; +use reth_primitives::trie::Nibbles; mod loader; pub use loader::PrefixSetLoader; diff --git a/crates/trie/src/progress.rs b/crates/trie/src/progress.rs index 20175bf9b5..0164d60178 100644 --- a/crates/trie/src/progress.rs +++ b/crates/trie/src/progress.rs @@ -1,5 +1,8 @@ -use crate::{hash_builder::HashBuilder, trie_cursor::CursorSubNode, updates::TrieUpdates, Nibbles}; -use reth_primitives::{trie::StoredSubNode, MerkleCheckpoint, H256}; +use crate::{trie_cursor::CursorSubNode, updates::TrieUpdates}; +use reth_primitives::{ + trie::{hash_builder::HashBuilder, Nibbles, StoredSubNode}, + MerkleCheckpoint, H256, +}; /// The progress of the state root computation. #[derive(Debug)] diff --git a/crates/trie/src/trie.rs b/crates/trie/src/trie.rs index d55c3bf40c..8656686ffb 100644 --- a/crates/trie/src/trie.rs +++ b/crates/trie/src/trie.rs @@ -1,8 +1,6 @@ use crate::{ account::EthAccount, - hash_builder::HashBuilder, hashed_cursor::{HashedAccountCursor, HashedCursorFactory, HashedStorageCursor}, - nibbles::Nibbles, prefix_set::{PrefixSet, PrefixSetLoader}, progress::{IntermediateStateRootState, StateRootProgress}, trie_cursor::{AccountTrieCursor, StorageTrieCursor}, @@ -11,7 +9,12 @@ use crate::{ StateRootError, StorageRootError, }; use reth_db::{tables, transaction::DbTx}; -use reth_primitives::{keccak256, proofs::EMPTY_ROOT, Address, BlockNumber, StorageEntry, H256}; +use reth_primitives::{ + keccak256, + proofs::EMPTY_ROOT, + trie::{HashBuilder, Nibbles}, + Address, BlockNumber, StorageEntry, H256, +}; use reth_rlp::Encodable; use std::{collections::HashMap, ops::RangeInclusive}; diff --git a/crates/trie/src/trie_cursor/subnode.rs b/crates/trie/src/trie_cursor/subnode.rs index 31a404a677..368af22865 100644 --- a/crates/trie/src/trie_cursor/subnode.rs +++ b/crates/trie/src/trie_cursor/subnode.rs @@ -1,6 +1,5 @@ -use crate::{nodes::CHILD_INDEX_RANGE, Nibbles}; use reth_primitives::{ - trie::{BranchNodeCompact, StoredSubNode}, + trie::{nodes::CHILD_INDEX_RANGE, BranchNodeCompact, Nibbles, StoredSubNode}, H256, }; diff --git a/crates/trie/src/updates.rs b/crates/trie/src/updates.rs index 847fd4d19c..0b666915c4 100644 --- a/crates/trie/src/updates.rs +++ b/crates/trie/src/updates.rs @@ -1,4 +1,3 @@ -use crate::Nibbles; use derive_more::Deref; use reth_db::{ cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW}, @@ -6,7 +5,7 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, }; use reth_primitives::{ - trie::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}, + trie::{BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey}, H256, }; use std::collections::{hash_map::IntoIter, HashMap}; diff --git a/crates/trie/src/walker.rs b/crates/trie/src/walker.rs index af1d267cee..a8125da1ec 100644 --- a/crates/trie/src/walker.rs +++ b/crates/trie/src/walker.rs @@ -2,10 +2,12 @@ use crate::{ prefix_set::PrefixSet, trie_cursor::{CursorSubNode, TrieCursor}, updates::TrieUpdates, - Nibbles, }; use reth_db::{table::Key, Error}; -use reth_primitives::{trie::BranchNodeCompact, H256}; +use reth_primitives::{ + trie::{BranchNodeCompact, Nibbles}, + H256, +}; use std::marker::PhantomData; /// `TrieWalker` is a structure that enables traversal of a Merkle trie.