github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-01-29 09:08:05 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(trie): move hash builder to primitives (#2493)

Browse Source
This commit is contained in:
Roman Krasiuk
2023-05-01 20:47:35 +03:00
committed by GitHub
parent 39427fad4d
commit cafb31aa53
24 changed files with 511 additions and 505 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@@ -5149,6 +5149,7 @@ dependencies = [
"tiny-keccak",
"tokio",
"tokio-stream",
"tracing",
"triehash",
"url",
]

3
crates/primitives/Cargo.toml
View File

@@ -36,6 +36,9 @@ secp256k1 = { version = "0.27.0", default-features = false, features = [
# used for forkid
crc = "3"
# tracing
tracing = "0.1"
# tokio
tokio = { version = "1", default-features = false, features = ["sync"] }
tokio-stream = "0.1"

2
crates/primitives/src/checkpoints.rs
View File

@@ -1,5 +1,5 @@
use crate::{
trie::{HashBuilderState, StoredSubNode},
trie::{hash_builder::HashBuilderState, StoredSubNode},
Address, H256,
};
use bytes::Buf;

21
crates/trie/src/hash_builder.rs → crates/primitives/src/trie/hash_builder/mod.rs
View File

@@ -1,15 +1,16 @@
use crate::{
use super::{
nodes::{rlp_hash, BranchNode, ExtensionNode, LeafNode},
Nibbles,
};
use reth_primitives::{
keccak256,
proofs::EMPTY_ROOT,
trie::{BranchNodeCompact, HashBuilderState, HashBuilderValue, TrieMask},
H256,
BranchNodeCompact, Nibbles, TrieMask,
};
use crate::{keccak256, proofs::EMPTY_ROOT, H256};
use std::{collections::HashMap, fmt::Debug};
mod state;
pub use state::HashBuilderState;
mod value;
pub use value::HashBuilderValue;
/// A component used to construct the root hash of the trie. The primary purpose of a Hash Builder
/// is to build the Merkle proof that is essential for verifying the integrity and authenticity of
/// the trie's contents. It achieves this by constructing the root hash from the hashes of child
@@ -414,8 +415,8 @@ impl HashBuilder {
#[cfg(test)]
mod tests {
use super::*;
use crate::{hex_literal::hex, proofs::KeccakHasher, H256, U256};
use proptest::prelude::*;
use reth_primitives::{hex_literal::hex, proofs::KeccakHasher, H256, U256};
use std::collections::{BTreeMap, HashMap};
fn trie_root<I, K, V>(iter: I) -> H256
@@ -588,7 +589,7 @@ mod tests {
// Manually create the branch node that should be there after the first 2 leaves are added.
// Skip the 0th element given in this example they have a common prefix and will
// collapse to a Branch node.
use reth_primitives::bytes::BytesMut;
use crate::bytes::BytesMut;
use reth_rlp::Encodable;
let leaf1 = LeafNode::new(&Nibbles::unpack(&raw_input[0].0[1..]), input[0].1);
let leaf2 = LeafNode::new(&Nibbles::unpack(&raw_input[1].0[1..]), input[1].1);

81
crates/primitives/src/trie/hash_builder.rs → crates/primitives/src/trie/hash_builder/state.rs
View File

@@ -1,5 +1,4 @@
use super::TrieMask;
use crate::H256;
use super::{super::TrieMask, HashBuilderValue};
use bytes::Buf;
use reth_codecs::{derive_arbitrary, Compact};
use serde::{Deserialize, Serialize};
@@ -114,84 +113,6 @@ impl Compact for HashBuilderState {
}
}
/// The current value of the hash builder.
#[derive_arbitrary(compact)]
#[derive(Clone, PartialEq, Serialize, Deserialize)]
pub enum HashBuilderValue {
/// Value of the leaf node.
Hash(H256),
/// Hash of adjacent nodes.
Bytes(Vec<u8>),
}
impl Compact for HashBuilderValue {
fn to_compact<B>(self, buf: &mut B) -> usize
where
B: bytes::BufMut + AsMut<[u8]>,
{
match self {
Self::Hash(hash) => {
buf.put_u8(0);
1 + hash.to_compact(buf)
}
Self::Bytes(bytes) => {
buf.put_u8(1);
1 + bytes.to_compact(buf)
}
}
}
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8])
where
Self: Sized,
{
match buf[0] {
0 => {
let (hash, buf) = H256::from_compact(&buf[1..], 32);
(Self::Hash(hash), buf)
}
1 => {
let (bytes, buf) = Vec::from_compact(&buf[1..], 0);
(Self::Bytes(bytes), buf)
}
_ => panic!("Invalid hash builder value"),
}
}
}
impl std::fmt::Debug for HashBuilderValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Bytes(bytes) => write!(f, "Bytes({:?})", hex::encode(bytes)),
Self::Hash(hash) => write!(f, "Hash({:?})", hash),
}
}
}
impl From<Vec<u8>> for HashBuilderValue {
fn from(value: Vec<u8>) -> Self {
Self::Bytes(value)
}
}
impl From<&[u8]> for HashBuilderValue {
fn from(value: &[u8]) -> Self {
Self::Bytes(value.to_vec())
}
}
impl From<H256> for HashBuilderValue {
fn from(value: H256) -> Self {
Self::Hash(value)
}
}
impl Default for HashBuilderValue {
fn default() -> Self {
Self::Bytes(vec![])
}
}
#[cfg(test)]
mod tests {
use super::*;

81
crates/primitives/src/trie/hash_builder/value.rs Normal file
View File

@@ -0,0 +1,81 @@
use crate::H256;
use reth_codecs::{derive_arbitrary, Compact};
use serde::{Deserialize, Serialize};
/// The current value of the hash builder.
#[derive_arbitrary(compact)]
#[derive(Clone, PartialEq, Serialize, Deserialize)]
pub enum HashBuilderValue {
/// Value of the leaf node.
Hash(H256),
/// Hash of adjacent nodes.
Bytes(Vec<u8>),
}
impl Compact for HashBuilderValue {
fn to_compact<B>(self, buf: &mut B) -> usize
where
B: bytes::BufMut + AsMut<[u8]>,
{
match self {
Self::Hash(hash) => {
buf.put_u8(0);
1 + hash.to_compact(buf)
}
Self::Bytes(bytes) => {
buf.put_u8(1);
1 + bytes.to_compact(buf)
}
}
}
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8])
where
Self: Sized,
{
match buf[0] {
0 => {
let (hash, buf) = H256::from_compact(&buf[1..], 32);
(Self::Hash(hash), buf)
}
1 => {
let (bytes, buf) = Vec::from_compact(&buf[1..], 0);
(Self::Bytes(bytes), buf)
}
_ => panic!("Invalid hash builder value"),
}
}
}
impl std::fmt::Debug for HashBuilderValue {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Bytes(bytes) => write!(f, "Bytes({:?})", hex::encode(bytes)),
Self::Hash(hash) => write!(f, "Hash({:?})", hash),
}
}
}
impl From<Vec<u8>> for HashBuilderValue {
fn from(value: Vec<u8>) -> Self {
Self::Bytes(value)
}
}
impl From<&[u8]> for HashBuilderValue {
fn from(value: &[u8]) -> Self {
Self::Bytes(value.to_vec())
}
}
impl From<H256> for HashBuilderValue {
fn from(value: H256) -> Self {
Self::Hash(value)
}
}
impl Default for HashBuilderValue {
fn default() -> Self {
Self::Bytes(vec![])
}
}

14
crates/primitives/src/trie/mod.rs
View File

@@ -1,17 +1,21 @@
//! Collection of trie related types.
mod branch_node;
mod hash_builder;
/// Various branch nodes produced by the hash builder.
pub mod nodes;
pub use nodes::BranchNodeCompact;
/// The implementation of hash builder.
pub mod hash_builder;
pub use hash_builder::HashBuilder;
mod mask;
mod nibbles;
mod storage;
mod subnode;
pub use self::{
branch_node::BranchNodeCompact,
hash_builder::{HashBuilderState, HashBuilderValue},
mask::TrieMask,
nibbles::{StoredNibbles, StoredNibblesSubKey},
nibbles::{Nibbles, StoredNibbles, StoredNibblesSubKey},
storage::StorageTrieEntry,
subnode::StoredSubNode,
};

298
crates/primitives/src/trie/nibbles.rs
View File

@@ -1,6 +1,7 @@
use crate::Bytes;
use derive_more::Deref;
use derive_more::{Deref, DerefMut, From, Index};
use reth_codecs::{main_codec, Compact};
use reth_rlp::RlpEncodableWrapper;
use serde::{Deserialize, Serialize};
/// The nibbles are the keys for the AccountsTrie and the subkeys for the StorageTrie.
@@ -49,3 +50,298 @@ impl Compact for StoredNibblesSubKey {
(Self(StoredNibbles { inner }), &buf[65..])
}
}
/// Structure representing a sequence of nibbles.
///
/// A nibble is a 4-bit value, and this structure is used to store
/// the nibble sequence representing the keys in a Merkle Patricia Trie (MPT).
/// Using nibbles simplifies trie operations and enables consistent key
/// representation in the MPT.
///
/// The `hex_data` field is a `Vec<u8>` that stores the nibbles, with each
/// `u8` value containing a single nibble. This means that each byte in
/// `hex_data` has its upper 4 bits set to zero and the lower 4 bits
/// representing the nibble value.
#[derive(
Default,
Clone,
Eq,
PartialEq,
RlpEncodableWrapper,
PartialOrd,
Ord,
Hash,
Index,
From,
Deref,
DerefMut,
)]
pub struct Nibbles {
/// The inner representation of the nibble sequence.
pub hex_data: Vec<u8>,
}
impl From<&[u8]> for Nibbles {
fn from(slice: &[u8]) -> Self {
Nibbles::from_hex(slice.to_vec())
}
}
impl<const N: usize> From<&[u8; N]> for Nibbles {
fn from(arr: &[u8; N]) -> Self {
Nibbles::from_hex(arr.to_vec())
}
}
impl std::fmt::Debug for Nibbles {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Nibbles").field("hex_data", &hex::encode(&self.hex_data)).finish()
}
}
impl Nibbles {
/// Creates a new [Nibbles] instance from bytes.
pub fn from_hex(hex: Vec<u8>) -> Self {
Nibbles { hex_data: hex }
}
/// Take a byte array (slice or vector) as input and convert it into a [Nibbles] struct
/// containing the nibbles (half-bytes or 4 bits) that make up the input byte data.
pub fn unpack<T: AsRef<[u8]>>(data: T) -> Self {
Nibbles { hex_data: data.as_ref().iter().flat_map(|item| [item / 16, item % 16]).collect() }
}
/// Packs the nibbles stored in the struct into a byte vector.
///
/// This method combines each pair of consecutive nibbles into a single byte,
/// effectively reducing the size of the data by a factor of two.
/// If the number of nibbles is odd, the last nibble is shifted left by 4 bits and
/// added to the packed byte vector.
pub fn pack(&self) -> Vec<u8> {
let length = (self.len() + 1) / 2;
if length == 0 {
Vec::new()
} else {
self.iter()
.enumerate()
.filter_map(|(index, nibble)| {
if index % 2 == 0 {
let next_nibble = self.get(index + 1).unwrap_or(&0);
Some((*nibble << 4) + *next_nibble)
} else {
None
}
})
.collect()
}
}
/// Encodes a given path leaf as a compact array of bytes, where each byte represents two
/// "nibbles" (half-bytes or 4 bits) of the original hex data, along with additional information
/// about the leaf itself.
///
/// The method takes the following input:
/// `is_leaf`: A boolean value indicating whether the current node is a leaf node or not.
///
/// The first byte of the encoded vector is set based on the `is_leaf` flag and the parity of
/// the hex data length (even or odd number of nibbles).
/// - If the node is an extension with even length, the header byte is `0x00`.
/// - If the node is an extension with odd length, the header byte is `0x10 + <first nibble>`.
/// - If the node is a leaf with even length, the header byte is `0x20`.
/// - If the node is a leaf with odd length, the header byte is `0x30 + <first nibble>`.
///
/// If there is an odd number of nibbles, store the first nibble in the lower 4 bits of the
/// first byte of encoded.
///
/// # Returns
///
/// A `Vec<u8>` containing the compact byte representation of the nibble sequence, including the
/// header byte.
///
/// # Example
///
/// ```
/// # use reth_primitives::trie::Nibbles;
///
/// // Extension node with an even path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]);
/// assert_eq!(nibbles.encode_path_leaf(false), vec![0x00, 0xAB, 0xCD]);
///
/// // Extension node with an odd path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]);
/// assert_eq!(nibbles.encode_path_leaf(false), vec![0x1A, 0xBC]);
///
/// // Leaf node with an even path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]);
/// assert_eq!(nibbles.encode_path_leaf(true), vec![0x20, 0xAB, 0xCD]);
///
/// // Leaf node with an odd path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]);
/// assert_eq!(nibbles.encode_path_leaf(true), vec![0x3A, 0xBC]);
/// ```
pub fn encode_path_leaf(&self, is_leaf: bool) -> Vec<u8> {
let mut encoded = vec![0u8; self.len() / 2 + 1];
let odd_nibbles = self.len() % 2 != 0;
// Set the first byte of the encoded vector.
encoded[0] = match (is_leaf, odd_nibbles) {
(true, true) => 0x30 | self[0],
(true, false) => 0x20,
(false, true) => 0x10 | self[0],
(false, false) => 0x00,
};
let mut nibble_idx = if odd_nibbles { 1 } else { 0 };
for byte in encoded.iter_mut().skip(1) {
*byte = (self[nibble_idx] << 4) + self[nibble_idx + 1];
nibble_idx += 2;
}
encoded
}
/// Increments the nibble sequence by one.
pub fn increment(&self) -> Option<Nibbles> {
let mut incremented = self.hex_data.clone();
for nibble in incremented.iter_mut().rev() {
assert!(*nibble < 0x10);
if *nibble < 0xf {
*nibble += 1;
return Some(Nibbles::from(incremented))
} else {
*nibble = 0;
}
}
None
}
/// The last element of the hex vector is used to determine whether the nibble sequence
/// represents a leaf or an extension node. If the last element is 0x10 (16), then it's a leaf.
pub fn is_leaf(&self) -> bool {
self.hex_data[self.hex_data.len() - 1] == 16
}
/// Returns `true` if the current nibble sequence starts with the given prefix.
pub fn has_prefix(&self, other: &Self) -> bool {
self.starts_with(other)
}
/// Returns the nibble at the given index.
pub fn at(&self, i: usize) -> usize {
self.hex_data[i] as usize
}
/// Returns the last nibble of the current nibble sequence.
pub fn last(&self) -> Option<u8> {
self.hex_data.last().copied()
}
/// Returns the length of the common prefix between the current nibble sequence and the given.
pub fn common_prefix_length(&self, other: &Nibbles) -> usize {
let len = std::cmp::min(self.len(), other.len());
for i in 0..len {
if self[i] != other[i] {
return i
}
}
len
}
/// Slice the current nibbles from the given start index to the end.
pub fn slice_from(&self, index: usize) -> Nibbles {
self.slice(index, self.hex_data.len())
}
/// Slice the current nibbles within the provided index range.
pub fn slice(&self, start: usize, end: usize) -> Nibbles {
Nibbles::from_hex(self.hex_data[start..end].to_vec())
}
/// Join two nibbles together.
pub fn join(&self, b: &Nibbles) -> Nibbles {
let mut hex_data = Vec::with_capacity(self.len() + b.len());
hex_data.extend_from_slice(self);
hex_data.extend_from_slice(b);
Nibbles::from_hex(hex_data)
}
/// Extend the current nibbles with another nibbles.
pub fn extend(&mut self, b: impl AsRef<[u8]>) {
self.hex_data.extend_from_slice(b.as_ref());
}
/// Truncate the current nibbles to the given length.
pub fn truncate(&mut self, len: usize) {
self.hex_data.truncate(len)
}
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
#[test]
fn hashed_regression() {
let nibbles = hex::decode("05010406040a040203030f010805020b050c04070003070e0909070f010b0a0805020301070c0a0902040b0f000f0006040a04050f020b090701000a0a040b").unwrap();
let nibbles = Nibbles::from(nibbles);
let path = nibbles.encode_path_leaf(true);
let expected =
hex::decode("351464a4233f1852b5c47037e997f1ba852317ca924bf0f064a45f2b9710aa4b")
.unwrap();
assert_eq!(path, expected);
}
#[test]
fn pack_nibbles() {
for (input, expected) in [
(vec![], vec![]),
(vec![0xa], vec![0xa0]),
(vec![0xa, 0xb], vec![0xab]),
(vec![0xa, 0xb, 0x2], vec![0xab, 0x20]),
(vec![0xa, 0xb, 0x2, 0x0], vec![0xab, 0x20]),
(vec![0xa, 0xb, 0x2, 0x7], vec![0xab, 0x27]),
] {
let nibbles = Nibbles::from(input);
let encoded = nibbles.pack();
assert_eq!(encoded, expected);
}
}
proptest! {
#[test]
fn pack_unpack_roundtrip(input in any::<Vec<u8>>()) {
let nibbles = Nibbles::unpack(&input);
let packed = nibbles.pack();
prop_assert_eq!(packed, input);
}
#[test]
fn encode_path_first_byte(input in any::<Vec<u8>>()) {
prop_assume!(!input.is_empty());
let input = Nibbles::unpack(input);
let input_is_odd = input.len() % 2 == 1;
let compact_leaf = input.encode_path_leaf(true);
let leaf_flag = compact_leaf[0];
// Check flag
assert_ne!(leaf_flag & 0x20, 0);
assert_eq!(input_is_odd, (leaf_flag & 0x10) != 0);
if input_is_odd {
assert_eq!(leaf_flag & 0x0f, *input.first().unwrap());
}
let compact_extension = input.encode_path_leaf(false);
let extension_flag = compact_extension[0];
// Check first byte
assert_eq!(extension_flag & 0x20, 0);
assert_eq!(input_is_odd, (extension_flag & 0x10) != 0);
if input_is_odd {
assert_eq!(extension_flag & 0x0f, *input.first().unwrap());
}
}
}
}

76
crates/primitives/src/trie/branch_node.rs → crates/primitives/src/trie/nodes/branch.rs
View File

@@ -1,9 +1,83 @@
use super::TrieMask;
use super::{super::TrieMask, rlp_node, CHILD_INDEX_RANGE};
use crate::H256;
use bytes::Buf;
use reth_codecs::Compact;
use reth_rlp::{BufMut, EMPTY_STRING_CODE};
use serde::{Deserialize, Serialize};
/// A Branch node is only a pointer to the stack of nodes and is used to
/// create the RLP encoding of the node using masks which filter from
/// the stack of nodes.
#[derive(Clone, Debug)]
pub struct BranchNode<'a> {
/// Rlp encoded children
pub stack: &'a [Vec<u8>],
}
impl<'a> BranchNode<'a> {
/// Create a new branch node from the stack of nodes.
pub fn new(stack: &'a [Vec<u8>]) -> Self {
Self { stack }
}
/// Given the hash and state mask of children present, return an iterator over the stack items
/// that match the mask.
pub fn children(
&self,
state_mask: TrieMask,
hash_mask: TrieMask,
) -> impl Iterator<Item = H256> + '_ {
let mut index = self.stack.len() - state_mask.count_ones() as usize;
CHILD_INDEX_RANGE.filter_map(move |digit| {
let mut child = None;
if state_mask.is_bit_set(digit) {
if hash_mask.is_bit_set(digit) {
child = Some(&self.stack[index]);
}
index += 1;
}
child.map(|child| H256::from_slice(&child[1..]))
})
}
/// Returns the RLP encoding of the branch node given the state mask of children present.
pub fn rlp(&self, state_mask: TrieMask, buf: &mut Vec<u8>) -> Vec<u8> {
let first_child_idx = self.stack.len() - state_mask.count_ones() as usize;
// Create the RLP header from the mask elements present.
let mut i = first_child_idx;
let header = CHILD_INDEX_RANGE.fold(
reth_rlp::Header { list: true, payload_length: 1 },
|mut header, digit| {
if state_mask.is_bit_set(digit) {
header.payload_length += self.stack[i].len();
i += 1;
} else {
header.payload_length += 1;
}
header
},
);
header.encode(buf);
// Extend the RLP buffer with the present children
let mut i = first_child_idx;
CHILD_INDEX_RANGE.for_each(|idx| {
if state_mask.is_bit_set(idx) {
buf.extend_from_slice(&self.stack[i]);
i += 1;
} else {
buf.put_u8(EMPTY_STRING_CODE)
}
});
// Is this needed?
buf.put_u8(EMPTY_STRING_CODE);
rlp_node(buf)
}
}
/// A struct representing a branch node in an Ethereum trie.
///
/// A branch node can have up to 16 children, each corresponding to one of the possible nibble

3
crates/trie/src/nodes/extension.rs → crates/primitives/src/trie/nodes/extension.rs
View File

@@ -1,5 +1,4 @@
use super::rlp_node;
use crate::Nibbles;
use super::{super::Nibbles, rlp_node};
use reth_rlp::{BufMut, Encodable};
/// An intermediate node that exists solely to compress the trie's paths. It contains a path segment

5
crates/trie/src/nodes/leaf.rs → crates/primitives/src/trie/nodes/leaf.rs
View File

@@ -1,5 +1,4 @@
use super::rlp_node;
use crate::Nibbles;
use super::{super::Nibbles, rlp_node};
use reth_rlp::{BufMut, Encodable};
/// A leaf node represents the endpoint or terminal node in the trie. In other words, a leaf node is
@@ -55,7 +54,7 @@ impl std::fmt::Debug for LeafNode<'_> {
#[cfg(test)]
mod tests {
use super::*;
use reth_primitives::hex_literal::hex;
use crate::hex_literal::hex;
// From manual regression test
#[test]

8
crates/trie/src/nodes/mod.rs → crates/primitives/src/trie/nodes/mod.rs
View File

@@ -1,4 +1,4 @@
use reth_primitives::{keccak256, H256};
use crate::{keccak256, H256};
use reth_rlp::EMPTY_STRING_CODE;
use std::ops::Range;
@@ -6,7 +6,11 @@ mod branch;
mod extension;
mod leaf;
pub use self::{branch::BranchNode, extension::ExtensionNode, leaf::LeafNode};
pub use self::{
branch::{BranchNode, BranchNodeCompact},
extension::ExtensionNode,
leaf::LeafNode,
};
/// The range of valid child indexes.
pub const CHILD_INDEX_RANGE: Range<u8> = 0..16;

3
crates/trie/benches/prefix_set.rs
View File

@@ -6,7 +6,8 @@ use proptest::{
strategy::{Strategy, ValueTree},
test_runner::{basic_result_cache, TestRunner},
};
use reth_trie::{prefix_set::PrefixSet, Nibbles};
use reth_primitives::trie::Nibbles;
use reth_trie::prefix_set::PrefixSet;
use std::collections::BTreeSet;
pub trait PrefixSetAbstraction: Default {

5
crates/trie/src/hashed_cursor/post_state.rs
View File

@@ -1,12 +1,11 @@
use crate::{prefix_set::PrefixSet, Nibbles};
use super::{HashedAccountCursor, HashedCursorFactory, HashedStorageCursor};
use crate::prefix_set::PrefixSet;
use reth_db::{
cursor::{DbCursorRO, DbDupCursorRO},
tables,
transaction::{DbTx, DbTxGAT},
};
use reth_primitives::{Account, StorageEntry, H256, U256};
use reth_primitives::{trie::Nibbles, Account, StorageEntry, H256, U256};
use std::collections::{BTreeMap, HashMap};
/// The post state account storage with hashed slots.

9
crates/trie/src/lib.rs
View File

@@ -9,18 +9,9 @@
//! authenticated radix trie that is used to store key-value bindings.
//! <https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/>
mod nibbles;
pub use nibbles::Nibbles;
/// The Ethereum account as represented in the trie.
pub mod account;
/// Various branch nodes produced by the hash builder.
pub mod nodes;
/// The implementation of hash builder.
pub mod hash_builder;
/// The implementation of a container for storing intermediate changes to a trie.
/// The container indicates when the trie has been modified.
pub mod prefix_set;

297
crates/trie/src/nibbles.rs
View File

@@ -1,297 +0,0 @@
use derive_more::{Deref, DerefMut, From, Index};
use reth_rlp::RlpEncodableWrapper;
/// Structure representing a sequence of nibbles.
///
/// A nibble is a 4-bit value, and this structure is used to store
/// the nibble sequence representing the keys in a Merkle Patricia Trie (MPT).
/// Using nibbles simplifies trie operations and enables consistent key
/// representation in the MPT.
///
/// The `hex_data` field is a `Vec<u8>` that stores the nibbles, with each
/// `u8` value containing a single nibble. This means that each byte in
/// `hex_data` has its upper 4 bits set to zero and the lower 4 bits
/// representing the nibble value.
#[derive(
Default,
Clone,
Eq,
PartialEq,
RlpEncodableWrapper,
PartialOrd,
Ord,
Hash,
Index,
From,
Deref,
DerefMut,
)]
pub struct Nibbles {
/// The inner representation of the nibble sequence.
pub hex_data: Vec<u8>,
}
impl From<&[u8]> for Nibbles {
fn from(slice: &[u8]) -> Self {
Nibbles::from_hex(slice.to_vec())
}
}
impl<const N: usize> From<&[u8; N]> for Nibbles {
fn from(arr: &[u8; N]) -> Self {
Nibbles::from_hex(arr.to_vec())
}
}
impl std::fmt::Debug for Nibbles {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Nibbles").field("hex_data", &hex::encode(&self.hex_data)).finish()
}
}
impl Nibbles {
/// Creates a new [Nibbles] instance from bytes.
pub fn from_hex(hex: Vec<u8>) -> Self {
Nibbles { hex_data: hex }
}
/// Take a byte array (slice or vector) as input and convert it into a [Nibbles] struct
/// containing the nibbles (half-bytes or 4 bits) that make up the input byte data.
pub fn unpack<T: AsRef<[u8]>>(data: T) -> Self {
Nibbles { hex_data: data.as_ref().iter().flat_map(|item| [item / 16, item % 16]).collect() }
}
/// Packs the nibbles stored in the struct into a byte vector.
///
/// This method combines each pair of consecutive nibbles into a single byte,
/// effectively reducing the size of the data by a factor of two.
/// If the number of nibbles is odd, the last nibble is shifted left by 4 bits and
/// added to the packed byte vector.
pub fn pack(&self) -> Vec<u8> {
let length = (self.len() + 1) / 2;
if length == 0 {
Vec::new()
} else {
self.iter()
.enumerate()
.filter_map(|(index, nibble)| {
if index % 2 == 0 {
let next_nibble = self.get(index + 1).unwrap_or(&0);
Some((*nibble << 4) + *next_nibble)
} else {
None
}
})
.collect()
}
}
/// Encodes a given path leaf as a compact array of bytes, where each byte represents two
/// "nibbles" (half-bytes or 4 bits) of the original hex data, along with additional information
/// about the leaf itself.
///
/// The method takes the following input:
/// `is_leaf`: A boolean value indicating whether the current node is a leaf node or not.
///
/// The first byte of the encoded vector is set based on the `is_leaf` flag and the parity of
/// the hex data length (even or odd number of nibbles).
/// - If the node is an extension with even length, the header byte is `0x00`.
/// - If the node is an extension with odd length, the header byte is `0x10 + <first nibble>`.
/// - If the node is a leaf with even length, the header byte is `0x20`.
/// - If the node is a leaf with odd length, the header byte is `0x30 + <first nibble>`.
///
/// If there is an odd number of nibbles, store the first nibble in the lower 4 bits of the
/// first byte of encoded.
///
/// # Returns
///
/// A `Vec<u8>` containing the compact byte representation of the nibble sequence, including the
/// header byte.
///
/// # Example
///
/// ```
/// # use reth_trie::Nibbles;
///
/// // Extension node with an even path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]);
/// assert_eq!(nibbles.encode_path_leaf(false), vec![0x00, 0xAB, 0xCD]);
///
/// // Extension node with an odd path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]);
/// assert_eq!(nibbles.encode_path_leaf(false), vec![0x1A, 0xBC]);
///
/// // Leaf node with an even path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C, 0x0D]);
/// assert_eq!(nibbles.encode_path_leaf(true), vec![0x20, 0xAB, 0xCD]);
///
/// // Leaf node with an odd path length:
/// let nibbles = Nibbles::from_hex(vec![0x0A, 0x0B, 0x0C]);
/// assert_eq!(nibbles.encode_path_leaf(true), vec![0x3A, 0xBC]);
/// ```
pub fn encode_path_leaf(&self, is_leaf: bool) -> Vec<u8> {
let mut encoded = vec![0u8; self.len() / 2 + 1];
let odd_nibbles = self.len() % 2 != 0;
// Set the first byte of the encoded vector.
encoded[0] = match (is_leaf, odd_nibbles) {
(true, true) => 0x30 | self[0],
(true, false) => 0x20,
(false, true) => 0x10 | self[0],
(false, false) => 0x00,
};
let mut nibble_idx = if odd_nibbles { 1 } else { 0 };
for byte in encoded.iter_mut().skip(1) {
*byte = (self[nibble_idx] << 4) + self[nibble_idx + 1];
nibble_idx += 2;
}
encoded
}
/// Increments the nibble sequence by one.
pub fn increment(&self) -> Option<Nibbles> {
let mut incremented = self.hex_data.clone();
for nibble in incremented.iter_mut().rev() {
assert!(*nibble < 0x10);
if *nibble < 0xf {
*nibble += 1;
return Some(Nibbles::from(incremented))
} else {
*nibble = 0;
}
}
None
}
/// The last element of the hex vector is used to determine whether the nibble sequence
/// represents a leaf or an extension node. If the last element is 0x10 (16), then it's a leaf.
pub fn is_leaf(&self) -> bool {
self.hex_data[self.hex_data.len() - 1] == 16
}
/// Returns `true` if the current nibble sequence starts with the given prefix.
pub fn has_prefix(&self, other: &Self) -> bool {
self.starts_with(other)
}
/// Returns the nibble at the given index.
pub fn at(&self, i: usize) -> usize {
self.hex_data[i] as usize
}
/// Returns the last nibble of the current nibble sequence.
pub fn last(&self) -> Option<u8> {
self.hex_data.last().copied()
}
/// Returns the length of the common prefix between the current nibble sequence and the given.
pub fn common_prefix_length(&self, other: &Nibbles) -> usize {
let len = std::cmp::min(self.len(), other.len());
for i in 0..len {
if self[i] != other[i] {
return i
}
}
len
}
/// Slice the current nibbles from the given start index to the end.
pub fn slice_from(&self, index: usize) -> Nibbles {
self.slice(index, self.hex_data.len())
}
/// Slice the current nibbles within the provided index range.
pub fn slice(&self, start: usize, end: usize) -> Nibbles {
Nibbles::from_hex(self.hex_data[start..end].to_vec())
}
/// Join two nibbles together.
pub fn join(&self, b: &Nibbles) -> Nibbles {
let mut hex_data = Vec::with_capacity(self.len() + b.len());
hex_data.extend_from_slice(self);
hex_data.extend_from_slice(b);
Nibbles::from_hex(hex_data)
}
/// Extend the current nibbles with another nibbles.
pub fn extend(&mut self, b: impl AsRef<[u8]>) {
self.hex_data.extend_from_slice(b.as_ref());
}
/// Truncate the current nibbles to the given length.
pub fn truncate(&mut self, len: usize) {
self.hex_data.truncate(len)
}
}
#[cfg(test)]
mod tests {
use super::*;
use proptest::prelude::*;
#[test]
fn hashed_regression() {
let nibbles = hex::decode("05010406040a040203030f010805020b050c04070003070e0909070f010b0a0805020301070c0a0902040b0f000f0006040a04050f020b090701000a0a040b").unwrap();
let nibbles = Nibbles::from(nibbles);
let path = nibbles.encode_path_leaf(true);
let expected =
hex::decode("351464a4233f1852b5c47037e997f1ba852317ca924bf0f064a45f2b9710aa4b")
.unwrap();
assert_eq!(path, expected);
}
#[test]
fn pack_nibbles() {
for (input, expected) in [
(vec![], vec![]),
(vec![0xa], vec![0xa0]),
(vec![0xa, 0xb], vec![0xab]),
(vec![0xa, 0xb, 0x2], vec![0xab, 0x20]),
(vec![0xa, 0xb, 0x2, 0x0], vec![0xab, 0x20]),
(vec![0xa, 0xb, 0x2, 0x7], vec![0xab, 0x27]),
] {
let nibbles = Nibbles::from(input);
let encoded = nibbles.pack();
assert_eq!(encoded, expected);
}
}
proptest! {
#[test]
fn pack_unpack_roundtrip(input in any::<Vec<u8>>()) {
let nibbles = Nibbles::unpack(&input);
let packed = nibbles.pack();
prop_assert_eq!(packed, input);
}
#[test]
fn encode_path_first_byte(input in any::<Vec<u8>>()) {
prop_assume!(!input.is_empty());
let input = Nibbles::unpack(input);
let input_is_odd = input.len() % 2 == 1;
let compact_leaf = input.encode_path_leaf(true);
let leaf_flag = compact_leaf[0];
// Check flag
assert_ne!(leaf_flag & 0x20, 0);
assert_eq!(input_is_odd, (leaf_flag & 0x10) != 0);
if input_is_odd {
assert_eq!(leaf_flag & 0x0f, *input.first().unwrap());
}
let compact_extension = input.encode_path_leaf(false);
let extension_flag = compact_extension[0];
// Check first byte
assert_eq!(extension_flag & 0x20, 0);
assert_eq!(input_is_odd, (extension_flag & 0x10) != 0);
if input_is_odd {
assert_eq!(extension_flag & 0x0f, *input.first().unwrap());
}
}
}
}

76
crates/trie/src/nodes/branch.rs
View File

@@ -1,76 +0,0 @@
use super::{rlp_node, CHILD_INDEX_RANGE};
use reth_primitives::{trie::TrieMask, H256};
use reth_rlp::{BufMut, EMPTY_STRING_CODE};
/// A Branch node is only a pointer to the stack of nodes and is used to
/// create the RLP encoding of the node using masks which filter from
/// the stack of nodes.
#[derive(Clone, Debug)]
pub struct BranchNode<'a> {
/// Rlp encoded children
pub stack: &'a [Vec<u8>],
}
impl<'a> BranchNode<'a> {
/// Create a new branch node from the stack of nodes.
pub fn new(stack: &'a [Vec<u8>]) -> Self {
Self { stack }
}
/// Given the hash and state mask of children present, return an iterator over the stack items
/// that match the mask.
pub fn children(
&self,
state_mask: TrieMask,
hash_mask: TrieMask,
) -> impl Iterator<Item = H256> + '_ {
let mut index = self.stack.len() - state_mask.count_ones() as usize;
CHILD_INDEX_RANGE.filter_map(move |digit| {
let mut child = None;
if state_mask.is_bit_set(digit) {
if hash_mask.is_bit_set(digit) {
child = Some(&self.stack[index]);
}
index += 1;
}
child.map(|child| H256::from_slice(&child[1..]))
})
}
/// Returns the RLP encoding of the branch node given the state mask of children present.
pub fn rlp(&self, state_mask: TrieMask, buf: &mut Vec<u8>) -> Vec<u8> {
let first_child_idx = self.stack.len() - state_mask.count_ones() as usize;
// Create the RLP header from the mask elements present.
let mut i = first_child_idx;
let header = CHILD_INDEX_RANGE.fold(
reth_rlp::Header { list: true, payload_length: 1 },
|mut header, digit| {
if state_mask.is_bit_set(digit) {
header.payload_length += self.stack[i].len();
i += 1;
} else {
header.payload_length += 1;
}
header
},
);
header.encode(buf);
// Extend the RLP buffer with the present children
let mut i = first_child_idx;
CHILD_INDEX_RANGE.for_each(|idx| {
if state_mask.is_bit_set(idx) {
buf.extend_from_slice(&self.stack[i]);
i += 1;
} else {
buf.put_u8(EMPTY_STRING_CODE)
}
});
// Is this needed?
buf.put_u8(EMPTY_STRING_CODE);
rlp_node(buf)
}
}

3
crates/trie/src/prefix_set/loader.rs
View File

@@ -1,5 +1,4 @@
use super::PrefixSet;
use crate::Nibbles;
use derive_more::Deref;
use reth_db::{
cursor::DbCursorRO,
@@ -8,7 +7,7 @@ use reth_db::{
transaction::DbTx,
Error,
};
use reth_primitives::{keccak256, BlockNumber, StorageEntry, H256};
use reth_primitives::{keccak256, trie::Nibbles, BlockNumber, StorageEntry, H256};
use std::{collections::HashMap, ops::RangeInclusive};
/// A wrapper around a database transaction that loads prefix sets within a given block range.

2
crates/trie/src/prefix_set/mod.rs
View File

@@ -1,4 +1,4 @@
use crate::Nibbles;
use reth_primitives::trie::Nibbles;
mod loader;
pub use loader::PrefixSetLoader;

7
crates/trie/src/progress.rs
View File

@@ -1,5 +1,8 @@
use crate::{hash_builder::HashBuilder, trie_cursor::CursorSubNode, updates::TrieUpdates, Nibbles};
use reth_primitives::{trie::StoredSubNode, MerkleCheckpoint, H256};
use crate::{trie_cursor::CursorSubNode, updates::TrieUpdates};
use reth_primitives::{
trie::{hash_builder::HashBuilder, Nibbles, StoredSubNode},
MerkleCheckpoint, H256,
};
/// The progress of the state root computation.
#[derive(Debug)]

9
crates/trie/src/trie.rs
View File

@@ -1,8 +1,6 @@
use crate::{
account::EthAccount,
hash_builder::HashBuilder,
hashed_cursor::{HashedAccountCursor, HashedCursorFactory, HashedStorageCursor},
nibbles::Nibbles,
prefix_set::{PrefixSet, PrefixSetLoader},
progress::{IntermediateStateRootState, StateRootProgress},
trie_cursor::{AccountTrieCursor, StorageTrieCursor},
@@ -11,7 +9,12 @@ use crate::{
StateRootError, StorageRootError,
};
use reth_db::{tables, transaction::DbTx};
use reth_primitives::{keccak256, proofs::EMPTY_ROOT, Address, BlockNumber, StorageEntry, H256};
use reth_primitives::{
keccak256,
proofs::EMPTY_ROOT,
trie::{HashBuilder, Nibbles},
Address, BlockNumber, StorageEntry, H256,
};
use reth_rlp::Encodable;
use std::{collections::HashMap, ops::RangeInclusive};

3
crates/trie/src/trie_cursor/subnode.rs
View File

@@ -1,6 +1,5 @@
use crate::{nodes::CHILD_INDEX_RANGE, Nibbles};
use reth_primitives::{
trie::{BranchNodeCompact, StoredSubNode},
trie::{nodes::CHILD_INDEX_RANGE, BranchNodeCompact, Nibbles, StoredSubNode},
H256,
};

3
crates/trie/src/updates.rs
View File

@@ -1,4 +1,3 @@
use crate::Nibbles;
use derive_more::Deref;
use reth_db::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO, DbDupCursorRW},
@@ -6,7 +5,7 @@ use reth_db::{
transaction::{DbTx, DbTxMut},
};
use reth_primitives::{
trie::{BranchNodeCompact, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey},
trie::{BranchNodeCompact, Nibbles, StorageTrieEntry, StoredNibbles, StoredNibblesSubKey},
H256,
};
use std::collections::{hash_map::IntoIter, HashMap};

6
crates/trie/src/walker.rs
View File

@@ -2,10 +2,12 @@ use crate::{
prefix_set::PrefixSet,
trie_cursor::{CursorSubNode, TrieCursor},
updates::TrieUpdates,
Nibbles,
};
use reth_db::{table::Key, Error};
use reth_primitives::{trie::BranchNodeCompact, H256};
use reth_primitives::{
trie::{BranchNodeCompact, Nibbles},
H256,
};
use std::marker::PhantomData;
/// `TrieWalker` is a structure that enables traversal of a Merkle trie.