github/reth
1
1
Fork 0
mirror of https://github.com/paradigmxyz/reth.git synced 2026-04-30 03:01:58 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat: BlockchainTree (#1212)

Browse Source 
Co-authored-by: Dragan Rakita <draganrakita@192.168.1.4>
This commit is contained in:
rakita
2023-03-14 19:17:14 +01:00
committed by GitHub
parent 06db495d96
commit 237fd5ce6e
54 changed files with 3409 additions and 367 deletions
Download Patch File Download Diff File Expand all files Collapse all files

312
crates/executor/src/blockchain_tree/block_indices.rs Normal file
View File

@@ -0,0 +1,312 @@
//! Implementation of [`BlockIndices`] related to [`super::BlockchainTree`]
use super::chain::{BlockChainId, Chain, ForkBlock};
use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders};
use std::collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap, HashSet};
/// Internal indices of the blocks and chains. This is main connection
/// between blocks, chains and canonical chain.
///
/// It contains list of canonical block hashes, forks to childs blocks
/// and block hash to chain id.
pub struct BlockIndices {
/// Last finalized block.
last_finalized_block: BlockNumber,
/// For EVM's "BLOCKHASH" opcode we require last 256 block hashes. So we need to specify
/// at least `additional_canonical_block_hashes`+`max_reorg_depth`, for eth that would be
/// 256+64.
num_of_additional_canonical_block_hashes: u64,
/// Canonical chain. Contains N number (depends on `finalization_depth`) of blocks.
/// These blocks are found in fork_to_child but not inside `blocks_to_chain` or
/// `number_to_block` as those are chain specific indices.
canonical_chain: BTreeMap<BlockNumber, BlockHash>,
/// Index needed when discarding the chain, so we can remove connected chains from tree.
/// NOTE: It contains just a blocks that are forks as a key and not all blocks.
fork_to_child: HashMap<BlockHash, HashSet<BlockHash>>,
/// Block hashes and side chain they belong
blocks_to_chain: HashMap<BlockHash, BlockChainId>,
/// Utility index. Block number to block hash. Can be used for
/// RPC to fetch all pending block in chain by its number.
index_number_to_block: HashMap<BlockNumber, HashSet<BlockHash>>,
}
impl BlockIndices {
/// Create new block indices structure
pub fn new(
last_finalized_block: BlockNumber,
num_of_additional_canonical_block_hashes: u64,
canonical_chain: BTreeMap<BlockNumber, BlockHash>,
) -> Self {
Self {
last_finalized_block,
num_of_additional_canonical_block_hashes,
fork_to_child: Default::default(),
canonical_chain,
blocks_to_chain: Default::default(),
index_number_to_block: Default::default(),
}
}
/// Return number of additional canonical block hashes that we need
/// to have to be able to have enought information for EVM execution.
pub fn num_of_additional_canonical_block_hashes(&self) -> u64 {
self.num_of_additional_canonical_block_hashes
}
/// Return fork to child indices
pub fn fork_to_child(&self) -> &HashMap<BlockHash, HashSet<BlockHash>> {
&self.fork_to_child
}
/// Return block to chain id
pub fn blocks_to_chain(&self) -> &HashMap<BlockHash, BlockChainId> {
&self.blocks_to_chain
}
/// Returns `true` if the Tree knowns the block hash.
pub fn contains_pending_block_hash(&self, block_hash: BlockHash) -> bool {
self.blocks_to_chain.contains_key(&block_hash)
}
/// Check if block hash belongs to canonical chain.
pub fn is_block_hash_canonical(&self, block_hash: &BlockHash) -> bool {
self.canonical_chain.range(self.last_finalized_block..).any(|(_, &h)| h == *block_hash)
}
/// Last finalized block
pub fn last_finalized_block(&self) -> BlockNumber {
self.last_finalized_block
}
/// Insert non fork block.
pub fn insert_non_fork_block(
&mut self,
block_number: BlockNumber,
block_hash: BlockHash,
chain_id: BlockChainId,
) {
self.index_number_to_block.entry(block_number).or_default().insert(block_hash);
self.blocks_to_chain.insert(block_hash, chain_id);
}
/// Insert block to chain and fork child indices of the new chain
pub fn insert_chain(&mut self, chain_id: BlockChainId, chain: &Chain) {
for (number, block) in chain.blocks().iter() {
// add block -> chain_id index
self.blocks_to_chain.insert(block.hash(), chain_id);
// add number -> block
self.index_number_to_block.entry(*number).or_default().insert(block.hash());
}
let first = chain.first();
// add parent block -> block index
self.fork_to_child.entry(first.parent_hash).or_default().insert(first.hash());
}
/// Get the chain ID the block belongs to
pub fn get_blocks_chain_id(&self, block: &BlockHash) -> Option<BlockChainId> {
self.blocks_to_chain.get(block).cloned()
}
/// Update all block hashes. iterate over present and new list of canonical hashes and compare
/// them. Remove all missmatches, disconnect them and return all chains that needs to be
/// removed.
pub fn update_block_hashes(
&mut self,
hashes: BTreeMap<u64, BlockHash>,
) -> BTreeSet<BlockChainId> {
let mut new_hashes = hashes.iter();
let mut old_hashes = self.canonical_chain().clone().into_iter();
let mut remove = Vec::new();
let mut new_hash = new_hashes.next();
let mut old_hash = old_hashes.next();
loop {
let Some(old_block_value) = old_hash else {
// end of old_hashes canonical chain. New chain has more block then old chain.
break
};
let Some(new_block_value) = new_hash else {
// Old canonical chain had more block than new chain.
// remove all present block.
// this is mostly not going to happen as reorg should make new chain in Tree.
while let Some(rem) = old_hash {
remove.push(rem);
old_hash = old_hashes.next();
}
break;
};
// compare old and new canonical block number
match new_block_value.0.cmp(&old_block_value.0) {
std::cmp::Ordering::Less => {
// new chain has more past blocks than old chain
new_hash = new_hashes.next();
}
std::cmp::Ordering::Equal => {
if *new_block_value.1 != old_block_value.1 {
// remove block hash as it is different
remove.push(old_block_value);
}
new_hash = new_hashes.next();
old_hash = old_hashes.next();
}
std::cmp::Ordering::Greater => {
// old chain has more past blocks that new chain
remove.push(old_block_value);
old_hash = old_hashes.next()
}
}
}
self.canonical_chain = hashes;
remove.into_iter().fold(BTreeSet::new(), |mut fold, (number, hash)| {
fold.extend(self.remove_block(number, hash));
fold
})
}
/// Remove chain from indices and return dependent chains that needs to be removed.
/// Does the cleaning of the tree and removing blocks from the chain.
pub fn remove_chain(&mut self, chain: &Chain) -> BTreeSet<BlockChainId> {
let mut lose_chains = BTreeSet::new();
for (block_number, block) in chain.blocks().iter() {
let block_hash = block.hash();
lose_chains.extend(self.remove_block(*block_number, block_hash))
}
lose_chains
}
/// Remove Blocks from indices.
fn remove_block(
&mut self,
block_number: BlockNumber,
block_hash: BlockHash,
) -> BTreeSet<BlockChainId> {
// rm number -> block
if let Entry::Occupied(mut entry) = self.index_number_to_block.entry(block_number) {
let set = entry.get_mut();
set.remove(&block_hash);
// remove set if empty
if set.is_empty() {
entry.remove();
}
}
// rm block -> chain_id
self.blocks_to_chain.remove(&block_hash);
// rm fork -> child
let removed_fork = self.fork_to_child.remove(&block_hash);
removed_fork
.map(|fork_blocks| {
fork_blocks
.into_iter()
.filter_map(|fork_child| self.blocks_to_chain.remove(&fork_child))
.collect()
})
.unwrap_or_default()
}
/// Remove all blocks from canonical list and insert new blocks to it.
///
/// It is assumed that blocks are interconnected and that they connect to canonical chain
pub fn canonicalize_blocks(&mut self, blocks: &BTreeMap<BlockNumber, SealedBlockWithSenders>) {
if blocks.is_empty() {
return
}
// Remove all blocks from canonical chain
let first_number = *blocks.first_key_value().unwrap().0;
// this will remove all blocks numbers that are going to be replaced.
self.canonical_chain.retain(|num, _| *num < first_number);
// remove them from block to chain_id index
blocks.iter().map(|(_, b)| (b.number, b.hash(), b.parent_hash)).for_each(
|(number, hash, parent_hash)| {
// rm block -> chain_id
self.blocks_to_chain.remove(&hash);
// rm number -> block
if let Entry::Occupied(mut entry) = self.index_number_to_block.entry(number) {
let set = entry.get_mut();
set.remove(&hash);
// remove set if empty
if set.is_empty() {
entry.remove();
}
}
// rm fork block -> hash
if let Entry::Occupied(mut entry) = self.fork_to_child.entry(parent_hash) {
let set = entry.get_mut();
set.remove(&hash);
// remove set if empty
if set.is_empty() {
entry.remove();
}
}
},
);
// insert new canonical
self.canonical_chain.extend(blocks.iter().map(|(number, block)| (*number, block.hash())))
}
/// Used for finalization of block.
/// Return list of chains for removal that depend on finalized canonical chain.
pub fn finalize_canonical_blocks(
&mut self,
finalized_block: BlockNumber,
) -> BTreeSet<BlockChainId> {
// get finalized chains. blocks between [self.last_finalized,finalized_block).
// Dont remove finalized_block, as sidechain can point to it.
let finalized_blocks: Vec<BlockHash> = self
.canonical_chain
.iter()
.filter(|(&number, _)| number >= self.last_finalized_block && number < finalized_block)
.map(|(_, hash)| *hash)
.collect();
// remove unneeded canonical hashes.
let remove_until =
finalized_block.saturating_sub(self.num_of_additional_canonical_block_hashes);
self.canonical_chain.retain(|&number, _| number >= remove_until);
let mut lose_chains = BTreeSet::new();
for block_hash in finalized_blocks.into_iter() {
// there is a fork block.
if let Some(fork_blocks) = self.fork_to_child.remove(&block_hash) {
lose_chains = fork_blocks.into_iter().fold(lose_chains, |mut fold, fork_child| {
if let Some(lose_chain) = self.blocks_to_chain.remove(&fork_child) {
fold.insert(lose_chain);
}
fold
});
}
}
// set last finalized block.
self.last_finalized_block = finalized_block;
lose_chains
}
/// get canonical hash
pub fn canonical_hash(&self, block_number: &BlockNumber) -> Option<BlockHash> {
self.canonical_chain.get(block_number).cloned()
}
/// get canonical tip
pub fn canonical_tip(&self) -> ForkBlock {
let (&number, &hash) =
self.canonical_chain.last_key_value().expect("There is always the canonical chain");
ForkBlock { number, hash }
}
/// Canonical chain needs for execution of EVM. It should contains last 256 block hashes.
pub fn canonical_chain(&self) -> &BTreeMap<BlockNumber, BlockHash> {
&self.canonical_chain
}
}

441
crates/executor/src/blockchain_tree/chain.rs Normal file
View File

@@ -0,0 +1,441 @@
//! Handles substate and list of blocks.
//! have functions to split, branch and append the chain.
use crate::{
execution_result::ExecutionResult,
substate::{SubStateData, SubStateWithProvider},
};
use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error};
use reth_primitives::{BlockHash, BlockNumber, SealedBlockWithSenders, SealedHeader, U256};
use reth_provider::{BlockExecutor, ExecutorFactory, StateProvider};
use std::collections::BTreeMap;
/// Internal to BlockchainTree chain identification.
pub(crate) type BlockChainId = u64;
/// Side chain that contain it state and connect to block found in canonical chain.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Chain {
/// Chain substate. Updated state after execution all blocks in chain.
substate: SubStateData,
/// Changesets for block and transaction. Will be used to update tables in database.
changesets: Vec<ExecutionResult>,
/// Blocks in this chain
blocks: BTreeMap<BlockNumber, SealedBlockWithSenders>,
}
/// Contains fork block and hash.
#[derive(Clone, Copy)]
pub struct ForkBlock {
/// Block number of block that chains branches from
pub number: u64,
/// Block hash of block that chains branches from
pub hash: BlockHash,
}
impl ForkBlock {
/// Return the number hash tuple.
pub fn num_hash(&self) -> (BlockNumber, BlockHash) {
(self.number, self.hash)
}
}
impl Chain {
/// Return blocks found in chain
pub fn blocks(&self) -> &BTreeMap<BlockNumber, SealedBlockWithSenders> {
&self.blocks
}
/// Into inner components
pub fn into_inner(
self,
) -> (BTreeMap<BlockNumber, SealedBlockWithSenders>, Vec<ExecutionResult>, SubStateData) {
(self.blocks, self.changesets, self.substate)
}
/// Return execution results of blocks
pub fn changesets(&self) -> &Vec<ExecutionResult> {
&self.changesets
}
/// Return fork block number and hash.
pub fn fork_block(&self) -> ForkBlock {
let tip = self.first();
ForkBlock { number: tip.number.saturating_sub(1), hash: tip.parent_hash }
}
/// Block fork number
pub fn fork_block_number(&self) -> BlockNumber {
self.first().number.saturating_sub(1)
}
/// Block fork hash
pub fn fork_block_hash(&self) -> BlockHash {
self.first().parent_hash
}
/// First block in chain.
pub fn first(&self) -> &SealedBlockWithSenders {
self.blocks.first_key_value().expect("Chain has at least one block for first").1
}
/// Return tip of the chain. Chain always have at least one block inside
pub fn tip(&self) -> &SealedBlockWithSenders {
self.last()
}
/// Return tip of the chain. Chain always have at least one block inside
pub fn last(&self) -> &SealedBlockWithSenders {
self.blocks.last_key_value().expect("Chain has at least one block for last").1
}
/// Create new chain with given blocks and execution result.
pub fn new(blocks: Vec<(SealedBlockWithSenders, ExecutionResult)>) -> Self {
let (blocks, changesets): (Vec<_>, Vec<_>) = blocks.into_iter().unzip();
let blocks = blocks.into_iter().map(|b| (b.number, b)).collect::<BTreeMap<_, _>>();
let mut substate = SubStateData::default();
substate.apply(&changesets);
Self { substate, changesets, blocks }
}
/// Create new chain that joins canonical block
/// If parent block is the tip mark chain fork.
pub fn new_canonical_fork<SP: StateProvider, C: Consensus, EF: ExecutorFactory>(
block: &SealedBlockWithSenders,
parent_header: &SealedHeader,
canonical_block_hashes: &BTreeMap<BlockNumber, BlockHash>,
provider: &SP,
consensus: &C,
factory: &EF,
) -> Result<Self, Error> {
// substate
let substate = SubStateData::default();
let empty = BTreeMap::new();
let substate_with_sp =
SubStateWithProvider::new(&substate, provider, &empty, canonical_block_hashes);
let changeset = Self::validate_and_execute(
block.clone(),
parent_header,
substate_with_sp,
consensus,
factory,
)?;
Ok(Self::new(vec![(block.clone(), changeset)]))
}
/// Create new chain that branches out from existing side chain.
pub fn new_chain_fork<SP: StateProvider, C: Consensus, EF: ExecutorFactory>(
&self,
block: SealedBlockWithSenders,
side_chain_block_hashes: BTreeMap<BlockNumber, BlockHash>,
canonical_block_hashes: &BTreeMap<BlockNumber, BlockHash>,
provider: &SP,
consensus: &C,
factory: &EF,
) -> Result<Self, Error> {
let parent_number = block.number - 1;
let parent = self
.blocks
.get(&parent_number)
.ok_or(ExecError::BlockNumberNotFoundInChain { block_number: parent_number })?;
// revert changesets
let revert_from = self.changesets.len() - (self.tip().number - parent.number) as usize;
let mut substate = self.substate.clone();
// Revert changesets to get the state of the parent that we need to apply the change.
substate.revert(&self.changesets[revert_from..]);
let substate_with_sp = SubStateWithProvider::new(
&substate,
provider,
&side_chain_block_hashes,
canonical_block_hashes,
);
let changeset = Self::validate_and_execute(
block.clone(),
parent,
substate_with_sp,
consensus,
factory,
)?;
substate.apply_one(&changeset);
let chain = Self {
substate,
changesets: vec![changeset],
blocks: BTreeMap::from([(block.number, block)]),
};
// if all is okay, return new chain back. Present chain is not modified.
Ok(chain)
}
/// Validate and execute block and return execution result or error.
fn validate_and_execute<SP: StateProvider, C: Consensus, EF: ExecutorFactory>(
block: SealedBlockWithSenders,
parent_block: &SealedHeader,
substate: SubStateWithProvider<'_, SP>,
consensus: &C,
factory: &EF,
) -> Result<ExecutionResult, Error> {
consensus.validate_header(&block, U256::MAX)?;
consensus.pre_validate_header(&block, parent_block)?;
consensus.pre_validate_block(&block)?;
let (unseal, senders) = block.into_components();
let unseal = unseal.unseal();
let res = factory.with_sp(substate).execute_and_verify_receipt(
&unseal,
U256::MAX,
Some(senders),
)?;
Ok(res)
}
/// Append block to this chain
pub fn append_block<SP: StateProvider, C: Consensus, EF: ExecutorFactory>(
&mut self,
block: SealedBlockWithSenders,
side_chain_block_hashes: BTreeMap<BlockNumber, BlockHash>,
canonical_block_hashes: &BTreeMap<BlockNumber, BlockHash>,
provider: &SP,
consensus: &C,
factory: &EF,
) -> Result<(), Error> {
let (_, parent_block) = self.blocks.last_key_value().expect("Chain has at least one block");
let changeset = Self::validate_and_execute(
block.clone(),
parent_block,
SubStateWithProvider::new(
&self.substate,
provider,
&side_chain_block_hashes,
canonical_block_hashes,
),
consensus,
factory,
)?;
self.substate.apply_one(&changeset);
self.changesets.push(changeset);
self.blocks.insert(block.number, block);
Ok(())
}
/// Merge two chains into one by appending received chain to the current one.
/// Take substate from newest one.
pub fn append_chain(&mut self, chain: Chain) -> Result<(), Error> {
let chain_tip = self.tip();
if chain_tip.hash != chain.fork_block_hash() {
return Err(ExecError::AppendChainDoesntConnect {
chain_tip: chain_tip.num_hash(),
other_chain_fork: chain.fork_block().num_hash(),
}
.into())
}
self.blocks.extend(chain.blocks.into_iter());
self.changesets.extend(chain.changesets.into_iter());
self.substate = chain.substate;
Ok(())
}
/// Split chain at the number or hash, block with given number will be included at first chain.
/// If any chain is empty (Does not have blocks) None will be returned.
///
/// If block hash is not found ChainSplit::NoSplitPending is returned.
///
/// Subtate state will be only found in second chain. First change substate will be
/// invalid.
pub fn split(mut self, split_at: SplitAt) -> ChainSplit {
let chain_tip = *self.blocks.last_entry().expect("chain is never empty").key();
let block_number = match split_at {
SplitAt::Hash(block_hash) => {
let block_number = self.blocks.iter().find_map(|(num, block)| {
if block.hash() == block_hash {
Some(*num)
} else {
None
}
});
let Some(block_number) = block_number else { return ChainSplit::NoSplitPending(self)};
// If block number is same as tip whole chain is becoming canonical.
if block_number == chain_tip {
return ChainSplit::NoSplitCanonical(self)
}
block_number
}
SplitAt::Number(block_number) => {
if block_number >= chain_tip {
return ChainSplit::NoSplitCanonical(self)
}
if block_number < *self.blocks.first_entry().expect("chain is never empty").key() {
return ChainSplit::NoSplitPending(self)
}
block_number
}
};
let higher_number_blocks = self.blocks.split_off(&(block_number + 1));
let (first_changesets, second_changeset) = self.changesets.split_at(self.blocks.len());
ChainSplit::Split {
canonical: Chain {
substate: SubStateData::default(),
changesets: first_changesets.to_vec(),
blocks: self.blocks,
},
pending: Chain {
substate: self.substate,
changesets: second_changeset.to_vec(),
blocks: higher_number_blocks,
},
}
}
}
/// Used in spliting the chain.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum SplitAt {
/// Split at block number.
Number(BlockNumber),
/// Split at block hash.
Hash(BlockHash),
}
/// Result of spliting chain.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ChainSplit {
/// Chain is not splited. Pending chain is returned.
/// Given block split is higher than last block.
/// Or in case of split by hash when hash is unknown.
NoSplitPending(Chain),
/// Chain is not splited. Canonical chain is returned.
/// Given block split is lower than first block.
NoSplitCanonical(Chain),
/// Chain is splited in two.
/// Given block split is contained in first chain.
Split {
/// Left contains lower block number that get canonicalized.
/// And substate is empty and not usable.
canonical: Chain,
/// Right contains higher block number, that is still pending.
/// And substate from original chain is moved here.
pending: Chain,
},
}
#[cfg(test)]
mod tests {
use super::*;
use crate::substate::AccountSubState;
use reth_primitives::{H160, H256};
use reth_provider::execution_result::AccountInfoChangeSet;
#[test]
fn chain_apend() {
let block = SealedBlockWithSenders::default();
let block1_hash = H256([0x01; 32]);
let block2_hash = H256([0x02; 32]);
let block3_hash = H256([0x03; 32]);
let block4_hash = H256([0x04; 32]);
let mut block1 = block.clone();
let mut block2 = block.clone();
let mut block3 = block.clone();
let mut block4 = block.clone();
block1.block.header.hash = block1_hash;
block2.block.header.hash = block2_hash;
block3.block.header.hash = block3_hash;
block4.block.header.hash = block4_hash;
block3.block.header.header.parent_hash = block2_hash;
let mut chain1 = Chain {
substate: Default::default(),
changesets: vec![],
blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]),
};
let chain2 = Chain {
substate: Default::default(),
changesets: vec![],
blocks: BTreeMap::from([(3, block3.clone()), (4, block4.clone())]),
};
assert_eq!(chain1.append_chain(chain2.clone()), Ok(()));
// chain1 got changed so this will fail
assert!(chain1.append_chain(chain2).is_err());
}
#[test]
fn test_number_split() {
let mut substate = SubStateData::default();
let mut account = AccountSubState::default();
account.info.nonce = 10;
substate.accounts.insert(H160([1; 20]), account);
let mut exec1 = ExecutionResult::default();
exec1.block_changesets.insert(H160([2; 20]), AccountInfoChangeSet::default());
let mut exec2 = ExecutionResult::default();
exec2.block_changesets.insert(H160([3; 20]), AccountInfoChangeSet::default());
let mut block1 = SealedBlockWithSenders::default();
let block1_hash = H256([15; 32]);
block1.hash = block1_hash;
block1.senders.push(H160([4; 20]));
let mut block2 = SealedBlockWithSenders::default();
let block2_hash = H256([16; 32]);
block2.hash = block2_hash;
block2.senders.push(H160([4; 20]));
let chain = Chain {
substate: substate.clone(),
changesets: vec![exec1.clone(), exec2.clone()],
blocks: BTreeMap::from([(1, block1.clone()), (2, block2.clone())]),
};
let chain_split1 = Chain {
substate: SubStateData::default(),
changesets: vec![exec1],
blocks: BTreeMap::from([(1, block1.clone())]),
};
let chain_split2 = Chain {
substate,
changesets: vec![exec2.clone()],
blocks: BTreeMap::from([(2, block2.clone())]),
};
// split in two
assert_eq!(
chain.clone().split(SplitAt::Hash(block1_hash)),
ChainSplit::Split { canonical: chain_split1.clone(), pending: chain_split2.clone() }
);
// split at unknown block hash
assert_eq!(
chain.clone().split(SplitAt::Hash(H256([100; 32]))),
ChainSplit::NoSplitPending(chain.clone())
);
// split at higher number
assert_eq!(
chain.clone().split(SplitAt::Number(10)),
ChainSplit::NoSplitCanonical(chain.clone())
);
// split at lower number
assert_eq!(
chain.clone().split(SplitAt::Number(0)),
ChainSplit::NoSplitPending(chain.clone())
);
}
}

918
crates/executor/src/blockchain_tree/mod.rs Normal file
View File

@@ -0,0 +1,918 @@
//! Implementation of [`BlockchainTree`]
pub mod block_indices;
pub mod chain;
use self::{
block_indices::BlockIndices,
chain::{ChainSplit, SplitAt},
};
use chain::{BlockChainId, Chain, ForkBlock};
use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx};
use reth_interfaces::{consensus::Consensus, executor::Error as ExecError, Error};
use reth_primitives::{BlockHash, BlockNumber, ChainSpec, SealedBlock, SealedBlockWithSenders};
use reth_provider::{
ExecutorFactory, HeaderProvider, ShareableDatabase, StateProvider, StateProviderFactory,
Transaction,
};
use std::{
collections::{BTreeMap, HashMap},
sync::Arc,
};
#[cfg_attr(doc, aquamarine::aquamarine)]
/// Tree of chains and its identifications.
///
/// Mermaid flowchart represent all blocks that can appear in blockchain.
/// Green blocks belong to canonical chain and are saved inside database table, they are our main
/// chain. Pending blocks and sidechains are found in memory inside [`BlockchainTree`].
/// Both pending and sidechains have same mechanisms only difference is when they got committed to
/// database. For pending it is just append operation but for sidechains they need to move current
/// canonical blocks to BlockchainTree flush sidechain to the database to become canonical chain.
/// ```mermaid
/// flowchart BT
/// subgraph canonical chain
/// CanonState:::state
/// block0canon:::canon -->block1canon:::canon -->block2canon:::canon -->block3canon:::canon --> block4canon:::canon --> block5canon:::canon
/// end
/// block5canon --> block6pending1:::pending
/// block5canon --> block6pending2:::pending
/// subgraph sidechain2
/// S2State:::state
/// block3canon --> block4s2:::sidechain --> block5s2:::sidechain
/// end
/// subgraph sidechain1
/// S1State:::state
/// block2canon --> block3s1:::sidechain --> block4s1:::sidechain --> block5s1:::sidechain --> block6s1:::sidechain
/// end
/// classDef state fill:#1882C4
/// classDef canon fill:#8AC926
/// classDef pending fill:#FFCA3A
/// classDef sidechain fill:#FF595E
/// ```
///
///
/// main functions:
/// * insert_block: Connect block to chain, execute it and if valid insert block inside tree.
/// * finalize_block: Remove chains that join to now finalized block, as chain becomes invalid.
/// * make_canonical: Check if we have the hash of block that we want to finalize and commit it to
/// db. If we dont have the block, pipeline syncing should start to fetch the blocks from p2p. Do
/// reorg in tables if canonical chain if needed.
pub struct BlockchainTree<DB: Database, C: Consensus, EF: ExecutorFactory> {
/// chains and present data
chains: HashMap<BlockChainId, Chain>,
/// Static blockchain id generator
block_chain_id_generator: u64,
/// Indices to block and their connection.
block_indices: BlockIndices,
/// Number of block after finalized block that we are storing. It should be more then
/// finalization window
max_blocks_in_chain: u64,
/// Finalization windows. Number of blocks that can be reorged
max_reorg_depth: u64,
/// Externals
externals: Externals<DB, C, EF>,
}
/// Container for external abstractions.
struct Externals<DB: Database, C: Consensus, EF: ExecutorFactory> {
/// Save sidechain, do reorgs and push new block to canonical chain that is inside db.
db: DB,
/// Consensus checks
consensus: C,
/// Create executor to execute blocks.
executor_factory: EF,
/// Chain spec
chain_spec: Arc<ChainSpec>,
}
impl<DB: Database, C: Consensus, EF: ExecutorFactory> Externals<DB, C, EF> {
/// Return sharable database helper structure.
fn sharable_db(&self) -> ShareableDatabase<&DB> {
ShareableDatabase::new(&self.db, self.chain_spec.clone())
}
}
/// Helper structure that wraps chains and indices to search for block hash accross the chains.
pub struct BlockHashes<'a> {
/// Chains
pub chains: &'a mut HashMap<BlockChainId, Chain>,
/// Indices
pub indices: &'a BlockIndices,
}
impl<DB: Database, C: Consensus, EF: ExecutorFactory> BlockchainTree<DB, C, EF> {
/// New blockchain tree
pub fn new(
db: DB,
consensus: C,
executor_factory: EF,
chain_spec: Arc<ChainSpec>,
max_reorg_depth: u64,
max_blocks_in_chain: u64,
num_of_additional_canonical_block_hashes: u64,
) -> Result<Self, Error> {
if max_reorg_depth > max_blocks_in_chain {
panic!("Side chain size should be more then finalization window");
}
let last_canonical_hashes = db
.tx()?
.cursor_read::<tables::CanonicalHeaders>()?
.walk_back(None)?
.take((max_reorg_depth + num_of_additional_canonical_block_hashes) as usize)
.collect::<Result<Vec<(BlockNumber, BlockHash)>, _>>()?;
// TODO(rakita) save last finalized block inside database but for now just take
// tip-max_reorg_depth
// task: https://github.com/paradigmxyz/reth/issues/1712
let (last_finalized_block_number, _) =
if last_canonical_hashes.len() > max_reorg_depth as usize {
last_canonical_hashes[max_reorg_depth as usize]
} else {
// it is in reverse order from tip to N
last_canonical_hashes.last().cloned().unwrap_or_default()
};
let externals = Externals { db, consensus, executor_factory, chain_spec };
Ok(Self {
externals,
block_chain_id_generator: 0,
chains: Default::default(),
block_indices: BlockIndices::new(
last_finalized_block_number,
num_of_additional_canonical_block_hashes,
BTreeMap::from_iter(last_canonical_hashes.into_iter()),
),
max_blocks_in_chain,
max_reorg_depth,
})
}
/// Fork side chain or append the block if parent is the top of the chain
fn fork_side_chain(
&mut self,
block: SealedBlockWithSenders,
chain_id: BlockChainId,
) -> Result<(), Error> {
let block_hashes = self.all_chain_hashes(chain_id);
// get canonical fork.
let canonical_fork =
self.canonical_fork(chain_id).ok_or(ExecError::BlockChainIdConsistency { chain_id })?;
// get chain that block needs to join to.
let parent_chain = self
.chains
.get_mut(&chain_id)
.ok_or(ExecError::BlockChainIdConsistency { chain_id })?;
let chain_tip = parent_chain.tip().hash();
let canonical_block_hashes = self.block_indices.canonical_chain();
// get canonical tip
let (_, canonical_tip_hash) =
canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default();
let db = self.externals.sharable_db();
let provider = if canonical_fork.hash == canonical_tip_hash {
Box::new(db.latest()?) as Box<dyn StateProvider>
} else {
Box::new(db.history_by_block_number(canonical_fork.number)?) as Box<dyn StateProvider>
};
// append the block if it is continuing the chain.
if chain_tip == block.parent_hash {
let block_hash = block.hash();
let block_number = block.number;
parent_chain.append_block(
block,
block_hashes,
canonical_block_hashes,
&provider,
&self.externals.consensus,
&self.externals.executor_factory,
)?;
drop(provider);
self.block_indices.insert_non_fork_block(block_number, block_hash, chain_id)
} else {
let chain = parent_chain.new_chain_fork(
block,
block_hashes,
canonical_block_hashes,
&provider,
&self.externals.consensus,
&self.externals.executor_factory,
)?;
// release the lifetime with a drop
drop(provider);
self.insert_chain(chain);
}
Ok(())
}
/// Fork canonical chain by creating new chain
pub fn fork_canonical_chain(&mut self, block: SealedBlockWithSenders) -> Result<(), Error> {
let canonical_block_hashes = self.block_indices.canonical_chain();
let (_, canonical_tip) =
canonical_block_hashes.last_key_value().map(|(i, j)| (*i, *j)).unwrap_or_default();
// create state provider
let db = self.externals.sharable_db();
let parent_header = db
.header(&block.parent_hash)?
.ok_or(ExecError::CanonicalChain { block_hash: block.parent_hash })?;
let provider = if block.parent_hash == canonical_tip {
Box::new(db.latest()?) as Box<dyn StateProvider>
} else {
Box::new(db.history_by_block_number(block.number - 1)?) as Box<dyn StateProvider>
};
let parent_header = parent_header.seal(block.parent_hash);
let chain = Chain::new_canonical_fork(
&block,
&parent_header,
canonical_block_hashes,
&provider,
&self.externals.consensus,
&self.externals.executor_factory,
)?;
drop(provider);
self.insert_chain(chain);
Ok(())
}
/// Get all block hashes from chain that are not canonical. This is one time operation per
/// block. Reason why this is not caches is to save memory.
fn all_chain_hashes(&self, chain_id: BlockChainId) -> BTreeMap<BlockNumber, BlockHash> {
// find chain and iterate over it,
let mut chain_id = chain_id;
let mut hashes = BTreeMap::new();
loop {
let Some(chain) = self.chains.get(&chain_id) else { return hashes };
hashes.extend(chain.blocks().values().map(|b| (b.number, b.hash())));
let fork_block = chain.fork_block_hash();
if let Some(next_chain_id) = self.block_indices.get_blocks_chain_id(&fork_block) {
chain_id = next_chain_id;
} else {
// if there is no fork block that point to other chains, break the loop.
// it means that this fork joins to canonical block.
break
}
}
hashes
}
/// Getting the canonical fork would tell use what kind of Provider we should execute block on.
/// If it is latest state provider or history state provider
/// Return None if chain_id is not known.
fn canonical_fork(&self, chain_id: BlockChainId) -> Option<ForkBlock> {
let mut chain_id = chain_id;
let mut fork;
loop {
// chain fork block
fork = self.chains.get(&chain_id)?.fork_block();
// get fork block chain
if let Some(fork_chain_id) = self.block_indices.get_blocks_chain_id(&fork.hash) {
chain_id = fork_chain_id;
continue
}
break
}
if self.block_indices.canonical_hash(&fork.number) == Some(fork.hash) {
Some(fork)
} else {
None
}
}
/// Insert chain to tree and ties the blocks to it.
/// Helper function that handles indexing and inserting.
fn insert_chain(&mut self, chain: Chain) -> BlockChainId {
let chain_id = self.block_chain_id_generator;
self.block_chain_id_generator += 1;
self.block_indices.insert_chain(chain_id, &chain);
// add chain_id -> chain index
self.chains.insert(chain_id, chain);
chain_id
}
/// Insert block inside tree. recover transaction signers and
/// internaly call [`BlockchainTree::insert_block_with_senders`] fn.
pub fn insert_block(&mut self, block: SealedBlock) -> Result<bool, Error> {
let block = block.seal_with_senders().ok_or(ExecError::SenderRecoveryError)?;
self.insert_block_with_senders(&block)
}
/// Insert block with senders inside tree.
/// Returns `true` if:
/// 1. It is part of the blockchain tree
/// 2. It is part of the canonical chain
/// 3. Its parent is part of the blockchain tree and we can fork at the parent
/// 4. Its parent is part of the canonical chain and we can fork at the parent
/// Otherwise will return `false`, indicating that neither the block nor its parent
/// is part of the chain or any sidechains. This means that if block becomes canonical
/// we need to fetch the missing blocks over p2p.
pub fn insert_block_with_senders(
&mut self,
block: &SealedBlockWithSenders,
) -> Result<bool, Error> {
// check if block number is inside pending block slide
let last_finalized_block = self.block_indices.last_finalized_block();
if block.number <= last_finalized_block {
return Err(ExecError::PendingBlockIsFinalized {
block_number: block.number,
block_hash: block.hash(),
last_finalized: last_finalized_block,
}
.into())
}
// we will not even try to insert blocks that are too far in future.
if block.number > last_finalized_block + self.max_blocks_in_chain {
return Err(ExecError::PendingBlockIsInFuture {
block_number: block.number,
block_hash: block.hash(),
last_finalized: last_finalized_block,
}
.into())
}
// check if block is already inside Tree
if self.block_indices.contains_pending_block_hash(block.hash()) {
// block is known return that is inserted
return Ok(true)
}
// check if block is part of canonical chain
if self.block_indices.canonical_hash(&block.number) == Some(block.hash()) {
// block is part of canonical chain
return Ok(true)
}
// check if block parent can be found in Tree
if let Some(parent_chain) = self.block_indices.get_blocks_chain_id(&block.parent_hash) {
self.fork_side_chain(block.clone(), parent_chain)?;
// TODO save pending block to database
// https://github.com/paradigmxyz/reth/issues/1713
return Ok(true)
}
// if not found, check if the parent can be found inside canonical chain.
if Some(block.parent_hash) == self.block_indices.canonical_hash(&(block.number - 1)) {
// create new chain that points to that block
self.fork_canonical_chain(block.clone())?;
// TODO save pending block to database
// https://github.com/paradigmxyz/reth/issues/1713
return Ok(true)
}
// NOTE: Block doesn't have a parent, and if we receive this block in `make_canonical`
// function this could be a trigger to initiate p2p syncing, as we are missing the
// parent.
Ok(false)
}
/// Do finalization of blocks. Remove them from tree
pub fn finalize_block(&mut self, finalized_block: BlockNumber) {
let mut remove_chains = self.block_indices.finalize_canonical_blocks(finalized_block);
while let Some(chain_id) = remove_chains.pop_first() {
if let Some(chain) = self.chains.remove(&chain_id) {
remove_chains.extend(self.block_indices.remove_chain(&chain));
}
}
}
/// Update canonical hashes. Reads last N canonical blocks from database and update all indices.
pub fn update_canonical_hashes(
&mut self,
last_finalized_block: BlockNumber,
) -> Result<(), Error> {
self.finalize_block(last_finalized_block);
let num_of_canonical_hashes =
self.max_reorg_depth + self.block_indices.num_of_additional_canonical_block_hashes();
let last_canonical_hashes = self
.externals
.db
.tx()?
.cursor_read::<tables::CanonicalHeaders>()?
.walk_back(None)?
.take(num_of_canonical_hashes as usize)
.collect::<Result<BTreeMap<BlockNumber, BlockHash>, _>>()?;
let mut remove_chains = self.block_indices.update_block_hashes(last_canonical_hashes);
// remove all chains that got discarded
while let Some(chain_id) = remove_chains.first() {
if let Some(chain) = self.chains.remove(chain_id) {
remove_chains.extend(self.block_indices.remove_chain(&chain));
}
}
Ok(())
}
/// Split chain and return canonical part of it. Pending part reinsert inside tree
/// with same chain_id.
fn split_chain(&mut self, chain_id: BlockChainId, chain: Chain, split_at: SplitAt) -> Chain {
match chain.split(split_at) {
ChainSplit::Split { canonical, pending } => {
// rest of splited chain is inserted back with same chain_id.
self.block_indices.insert_chain(chain_id, &pending);
self.chains.insert(chain_id, pending);
canonical
}
ChainSplit::NoSplitCanonical(canonical) => canonical,
ChainSplit::NoSplitPending(_) => {
panic!("Should not happen as block indices guarantee structure of blocks")
}
}
}
/// Make block and its parent canonical. Unwind chains to database if necessary.
///
/// If block is already part of canonical chain return Ok.
pub fn make_canonical(&mut self, block_hash: &BlockHash) -> Result<(), Error> {
let chain_id = if let Some(chain_id) = self.block_indices.get_blocks_chain_id(block_hash) {
chain_id
} else {
// If block is already canonical don't return error.
if self.block_indices.is_block_hash_canonical(block_hash) {
return Ok(())
}
return Err(ExecError::BlockHashNotFoundInChain { block_hash: *block_hash }.into())
};
let chain = self.chains.remove(&chain_id).expect("To be present");
// we are spliting chain as there is possibility that only part of chain get canonicalized.
let canonical = self.split_chain(chain_id, chain, SplitAt::Hash(*block_hash));
let mut block_fork = canonical.fork_block();
let mut block_fork_number = canonical.fork_block_number();
let mut chains_to_promote = vec![canonical];
// loop while fork blocks are found in Tree.
while let Some(chain_id) = self.block_indices.get_blocks_chain_id(&block_fork.hash) {
let chain = self.chains.remove(&chain_id).expect("To fork to be present");
block_fork = chain.fork_block();
let canonical = self.split_chain(chain_id, chain, SplitAt::Number(block_fork_number));
block_fork_number = canonical.fork_block_number();
chains_to_promote.push(canonical);
}
let old_tip = self.block_indices.canonical_tip();
// Merge all chain into one chain.
let mut new_canon_chain = chains_to_promote.pop().expect("There is at least one block");
for chain in chains_to_promote.into_iter().rev() {
new_canon_chain.append_chain(chain).expect("We have just build the chain.");
}
// update canonical index
self.block_indices.canonicalize_blocks(new_canon_chain.blocks());
// if joins to the tip
if new_canon_chain.fork_block_hash() == old_tip.hash {
// append to database
self.commit_canonical(new_canon_chain)?;
} else {
// it forks to canonical block that is not the tip.
let canon_fork = new_canon_chain.fork_block();
// sanity check
if self.block_indices.canonical_hash(&canon_fork.number) != Some(canon_fork.hash) {
unreachable!("all chains should point to canonical chain.");
}
// revert `N` blocks from current canonical chain and put them inside BlockchanTree
// This is main reorgs on tables.
let old_canon_chain = self.revert_canonical(canon_fork.number)?;
self.commit_canonical(new_canon_chain)?;
// TODO we can potentially merge now reverted canonical chain with
// one of the chain from the tree. Low priority.
// insert old canonical chain to BlockchainTree.
self.insert_chain(old_canon_chain);
}
Ok(())
}
/// Commit chain for it to become canonical. Assume we are doing pending operation to db.
fn commit_canonical(&mut self, chain: Chain) -> Result<(), Error> {
let mut tx = Transaction::new(&self.externals.db)?;
let new_tip = chain.tip().number;
let (blocks, changesets, _) = chain.into_inner();
for item in blocks.into_iter().zip(changesets.into_iter()) {
let ((_, block), changeset) = item;
tx.insert_block(block, self.externals.chain_spec.as_ref(), changeset)
.map_err(|e| ExecError::CanonicalCommit { inner: e.to_string() })?;
}
// update pipeline progress.
tx.update_pipeline_stages(new_tip)
.map_err(|e| ExecError::PipelineStatusUpdate { inner: e.to_string() })?;
tx.commit()?;
Ok(())
}
/// Revert canonical blocks from database and insert them to pending table
/// Revert should be non inclusive, and revert_until should stay in db.
/// Return the chain that represent reverted canonical blocks.
fn revert_canonical(&mut self, revert_until: BlockNumber) -> Result<Chain, Error> {
// read data that is needed for new sidechain
let mut tx = Transaction::new(&self.externals.db)?;
// read block and execution result from database. and remove traces of block from tables.
let blocks_and_execution = tx
.take_block_and_execution_range(
self.externals.chain_spec.as_ref(),
(revert_until + 1)..,
)
.map_err(|e| ExecError::CanonicalRevert { inner: e.to_string() })?;
// update pipeline progress.
tx.update_pipeline_stages(revert_until)
.map_err(|e| ExecError::PipelineStatusUpdate { inner: e.to_string() })?;
tx.commit()?;
let chain = Chain::new(blocks_and_execution);
Ok(chain)
}
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use super::*;
use parking_lot::Mutex;
use reth_db::{
mdbx::{test_utils::create_test_rw_db, Env, WriteMap},
transaction::DbTxMut,
};
use reth_interfaces::test_utils::TestConsensus;
use reth_primitives::{hex_literal::hex, proofs::EMPTY_ROOT, ChainSpecBuilder, H256, MAINNET};
use reth_provider::{
execution_result::ExecutionResult, insert_block, test_utils::blocks::BlockChainTestData,
BlockExecutor,
};
struct TestFactory {
exec_result: Arc<Mutex<Vec<ExecutionResult>>>,
chain_spec: Arc<ChainSpec>,
}
impl TestFactory {
fn new(chain_spec: Arc<ChainSpec>) -> Self {
Self { exec_result: Arc::new(Mutex::new(Vec::new())), chain_spec }
}
fn extend(&self, exec_res: Vec<ExecutionResult>) {
self.exec_result.lock().extend(exec_res.into_iter());
}
}
struct TestExecutor(Option<ExecutionResult>);
impl<SP: StateProvider> BlockExecutor<SP> for TestExecutor {
fn execute(
&mut self,
_block: &reth_primitives::Block,
_total_difficulty: reth_primitives::U256,
_senders: Option<Vec<reth_primitives::Address>>,
) -> Result<ExecutionResult, ExecError> {
self.0.clone().ok_or(ExecError::VerificationFailed)
}
fn execute_and_verify_receipt(
&mut self,
_block: &reth_primitives::Block,
_total_difficulty: reth_primitives::U256,
_senders: Option<Vec<reth_primitives::Address>>,
) -> Result<ExecutionResult, ExecError> {
self.0.clone().ok_or(ExecError::VerificationFailed)
}
}
impl ExecutorFactory for TestFactory {
type Executor<T: StateProvider> = TestExecutor;
fn with_sp<SP: StateProvider>(&self, _sp: SP) -> Self::Executor<SP> {
let exec_res = self.exec_result.lock().pop();
TestExecutor(exec_res)
}
fn chain_spec(&self) -> &ChainSpec {
self.chain_spec.as_ref()
}
}
type TestExternals = (Arc<Env<WriteMap>>, TestConsensus, TestFactory, Arc<ChainSpec>);
fn externals(exec_res: Vec<ExecutionResult>) -> TestExternals {
let db = create_test_rw_db();
let consensus = TestConsensus::default();
let chain_spec = Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(MAINNET.genesis.clone())
.shanghai_activated()
.build(),
);
let executor_factory = TestFactory::new(chain_spec.clone());
executor_factory.extend(exec_res);
(db, consensus, executor_factory, chain_spec)
}
fn setup(mut genesis: SealedBlock, externals: &TestExternals) {
// insert genesis to db.
genesis.header.header.number = 10;
genesis.header.header.state_root = EMPTY_ROOT;
let tx_mut = externals.0.tx_mut().unwrap();
insert_block(&tx_mut, genesis.clone(), None, false, Some((0, 0))).unwrap();
// insert first 10 blocks
for i in 0..10 {
tx_mut.put::<tables::CanonicalHeaders>(i, H256([100 + i as u8; 32])).unwrap();
}
tx_mut.commit().unwrap();
}
/// Test data structure that will check tree internals
#[derive(Default, Debug)]
struct TreeTester {
/// Number of chains
chain_num: Option<usize>,
/// Check block to chain index
block_to_chain: Option<HashMap<BlockHash, BlockChainId>>,
/// Check fork to child index
fork_to_child: Option<HashMap<BlockHash, HashSet<BlockHash>>>,
}
impl TreeTester {
fn with_chain_num(mut self, chain_num: usize) -> Self {
self.chain_num = Some(chain_num);
self
}
fn with_block_to_chain(mut self, block_to_chain: HashMap<BlockHash, BlockChainId>) -> Self {
self.block_to_chain = Some(block_to_chain);
self
}
fn with_fork_to_child(
mut self,
fork_to_child: HashMap<BlockHash, HashSet<BlockHash>>,
) -> Self {
self.fork_to_child = Some(fork_to_child);
self
}
fn assert<DB: Database, C: Consensus, EF: ExecutorFactory>(
self,
tree: &BlockchainTree<DB, C, EF>,
) {
if let Some(chain_num) = self.chain_num {
assert_eq!(tree.chains.len(), chain_num);
}
if let Some(block_to_chain) = self.block_to_chain {
assert_eq!(*tree.block_indices.blocks_to_chain(), block_to_chain);
}
if let Some(fork_to_child) = self.fork_to_child {
assert_eq!(*tree.block_indices.fork_to_child(), fork_to_child);
}
}
}
#[test]
fn sanity_path() {
let data = BlockChainTestData::default();
let (mut block1, exec1) = data.blocks[0].clone();
block1.number = 11;
block1.state_root =
H256(hex!("5d035ccb3e75a9057452ff060b773b213ec1fc353426174068edfc3971a0b6bd"));
let (mut block2, exec2) = data.blocks[1].clone();
block2.number = 12;
block2.state_root =
H256(hex!("90101a13dd059fa5cca99ed93d1dc23657f63626c5b8f993a2ccbdf7446b64f8"));
// test pops execution results from vector, so order is from last to first.ß
let externals = externals(vec![exec2.clone(), exec1.clone(), exec2.clone(), exec1.clone()]);
// last finalized block would be number 9.
setup(data.genesis, &externals);
// make tree
let (db, consensus, exec_factory, chain_spec) = externals;
let mut tree =
BlockchainTree::new(db, consensus, exec_factory, chain_spec, 1, 2, 3).unwrap();
// genesis block 10 is already canonical
assert_eq!(tree.make_canonical(&H256::zero()), Ok(()));
// insert block2 hits max chain size
assert_eq!(
tree.insert_block_with_senders(&block2),
Err(ExecError::PendingBlockIsInFuture {
block_number: block2.number,
block_hash: block2.hash(),
last_finalized: 9,
}
.into())
);
// make genesis block 10 as finalized
tree.finalize_block(10);
// block 2 parent is not known.
assert_eq!(tree.insert_block_with_senders(&block2), Ok(false));
// insert block1
assert_eq!(tree.insert_block_with_senders(&block1), Ok(true));
// already inserted block will return true.
assert_eq!(tree.insert_block_with_senders(&block1), Ok(true));
// insert block2
assert_eq!(tree.insert_block_with_senders(&block2), Ok(true));
// Trie state:
// b2 (pending block)
// |
// |
// b1 (pending block)
// /
// /
// g1 (canonical blocks)
// |
TreeTester::default()
.with_chain_num(1)
.with_block_to_chain(HashMap::from([(block1.hash, 0), (block2.hash, 0)]))
.with_fork_to_child(HashMap::from([(block1.parent_hash, HashSet::from([block1.hash]))]))
.assert(&tree);
// make block1 canonical
assert_eq!(tree.make_canonical(&block1.hash()), Ok(()));
// make block2 canonical
assert_eq!(tree.make_canonical(&block2.hash()), Ok(()));
// Trie state:
// b2 (canonical block)
// |
// |
// b1 (canonical block)
// |
// |
// g1 (canonical blocks)
// |
TreeTester::default()
.with_chain_num(0)
.with_block_to_chain(HashMap::from([]))
.with_fork_to_child(HashMap::from([]))
.assert(&tree);
let mut block1a = block1.clone();
let block1a_hash = H256([0x33; 32]);
block1a.hash = block1a_hash;
let mut block2a = block2.clone();
let block2a_hash = H256([0x34; 32]);
block2a.hash = block2a_hash;
// reinsert two blocks that point to canonical chain
assert_eq!(tree.insert_block_with_senders(&block1a), Ok(true));
TreeTester::default()
.with_chain_num(1)
.with_block_to_chain(HashMap::from([(block1a_hash, 1)]))
.with_fork_to_child(HashMap::from([(
block1.parent_hash,
HashSet::from([block1a_hash]),
)]))
.assert(&tree);
assert_eq!(tree.insert_block_with_senders(&block2a), Ok(true));
// Trie state:
// b2 b2a (side chain)
// | /
// | /
// b1 b1a (side chain)
// | /
// |/
// g1 (10)
// |
TreeTester::default()
.with_chain_num(2)
.with_block_to_chain(HashMap::from([(block1a_hash, 1), (block2a_hash, 2)]))
.with_fork_to_child(HashMap::from([
(block1.parent_hash, HashSet::from([block1a_hash])),
(block1.hash(), HashSet::from([block2a_hash])),
]))
.assert(&tree);
// make b2a canonical
assert_eq!(tree.make_canonical(&block2a_hash), Ok(()));
// Trie state:
// b2a b2 (side chain)
// | /
// | /
// b1 b1a (side chain)
// | /
// |/
// g1 (10)
// |
TreeTester::default()
.with_chain_num(2)
.with_block_to_chain(HashMap::from([(block1a_hash, 1), (block2.hash, 3)]))
.with_fork_to_child(HashMap::from([
(block1.parent_hash, HashSet::from([block1a_hash])),
(block1.hash(), HashSet::from([block2.hash])),
]))
.assert(&tree);
assert_eq!(tree.make_canonical(&block1a_hash), Ok(()));
// Trie state:
// b2a b2 (side chain)
// | /
// | /
// b1a b1 (side chain)
// | /
// |/
// g1 (10)
// |
TreeTester::default()
.with_chain_num(2)
.with_block_to_chain(HashMap::from([
(block1.hash, 4),
(block2a_hash, 4),
(block2.hash, 3),
]))
.with_fork_to_child(HashMap::from([
(block1.parent_hash, HashSet::from([block1.hash])),
(block1.hash(), HashSet::from([block2.hash])),
]))
.assert(&tree);
// make b2 canonical
assert_eq!(tree.make_canonical(&block2.hash()), Ok(()));
// Trie state:
// b2 b2a (side chain)
// | /
// | /
// b1 b1a (side chain)
// | /
// |/
// g1 (10)
// |
TreeTester::default()
.with_chain_num(2)
.with_block_to_chain(HashMap::from([(block1a_hash, 5), (block2a_hash, 4)]))
.with_fork_to_child(HashMap::from([
(block1.parent_hash, HashSet::from([block1a_hash])),
(block1.hash(), HashSet::from([block2a_hash])),
]))
.assert(&tree);
// finalize b1 that would make b1a removed from tree
tree.finalize_block(11);
// Trie state:
// b2 b2a (side chain)
// | /
// | /
// b1 (canon)
// |
// g1 (10)
// |
TreeTester::default()
.with_chain_num(1)
.with_block_to_chain(HashMap::from([(block2a_hash, 4)]))
.with_fork_to_child(HashMap::from([(block1.hash(), HashSet::from([block2a_hash]))]))
.assert(&tree);
// update canonical block to b2, this would make b2a be removed
assert_eq!(tree.update_canonical_hashes(12), Ok(()));
// Trie state:
// b2 (canon)
// |
// b1 (canon)
// |
// g1 (10)
// |
TreeTester::default()
.with_chain_num(0)
.with_block_to_chain(HashMap::from([]))
.with_fork_to_child(HashMap::from([]))
.assert(&tree);
}
}

2
crates/executor/src/lib.rs
View File

@@ -8,9 +8,11 @@
//! Reth executor executes transaction in block of data.
pub mod eth_dao_fork;
pub mod substate;
/// Execution result types.
pub use reth_provider::execution_result;
pub mod blockchain_tree;
/// Executor
pub mod executor;

306
crates/executor/src/substate.rs Normal file
View File

@@ -0,0 +1,306 @@
//! Substate for blockchain trees
use reth_interfaces::{provider::ProviderError, Result};
use reth_primitives::{Account, Address, BlockHash, BlockNumber, Bytecode, Bytes, H256, U256};
use reth_provider::{AccountProvider, BlockHashProvider, StateProvider};
use std::collections::{hash_map::Entry, BTreeMap, HashMap};
use crate::execution_result::{AccountInfoChangeSet, ExecutionResult};
/// Memory backend, storing all state values in a `Map` in memory.
#[derive(Debug, Default, Clone, PartialEq, Eq)]
pub struct SubStateData {
/// Account info where None means it is not existing. Not existing state is needed for Pre
/// TANGERINE forks. `code` is always `None`, and bytecode can be found in `contracts`.
pub accounts: HashMap<Address, AccountSubState>,
/// New bytecodes
pub bytecodes: HashMap<H256, (u32, Bytecode)>,
}
impl SubStateData {
/// Apply changesets to substate.
pub fn apply(&mut self, changesets: &[ExecutionResult]) {
for changeset in changesets {
self.apply_one(changeset)
}
}
/// Apply one changeset to substate.
pub fn apply_one(&mut self, changeset: &ExecutionResult) {
for tx_changeset in changeset.tx_changesets.iter() {
// apply accounts
for (address, account_change) in tx_changeset.changeset.iter() {
// revert account
self.apply_account(address, &account_change.account);
// revert its storage
self.apply_storage(address, &account_change.storage);
}
// apply bytecodes
for (hash, bytecode) in tx_changeset.new_bytecodes.iter() {
self.bytecodes.entry(*hash).or_insert((0, Bytecode(bytecode.clone()))).0 += 1;
}
}
// apply block reward
for (address, change) in changeset.block_changesets.iter() {
self.apply_account(address, change)
}
}
/// Apply account changeset to substate
fn apply_account(&mut self, address: &Address, change: &AccountInfoChangeSet) {
match change {
AccountInfoChangeSet::Created { new } => match self.accounts.entry(*address) {
Entry::Vacant(entry) => {
entry.insert(AccountSubState::created_account(*new));
}
Entry::Occupied(mut entry) => {
let account = entry.get_mut();
// increment counter
account.inc_storage_counter();
account.info = *new;
}
},
AccountInfoChangeSet::Destroyed { .. } => {
// set selfdestructed account
let account = self.accounts.entry(*address).or_default();
account.inc_storage_counter();
account.info = Default::default();
account.storage.clear();
}
AccountInfoChangeSet::Changed { old, .. } => {
self.accounts.entry(*address).or_default().info = *old;
}
AccountInfoChangeSet::NoChange { is_empty } => {
if *is_empty {
self.accounts.entry(*address).or_default();
}
}
}
}
/// Apply storage changeset to substate
fn apply_storage(&mut self, address: &Address, storage: &BTreeMap<U256, (U256, U256)>) {
if let Entry::Occupied(mut entry) = self.accounts.entry(*address) {
let account_storage = &mut entry.get_mut().storage;
for (key, (_, new_value)) in storage {
let key = H256(key.to_be_bytes());
account_storage.insert(key, *new_value);
}
}
}
/// Revert to old state in substate. Changesets will be reverted in reverse order,
pub fn revert(&mut self, changesets: &[ExecutionResult]) {
for changeset in changesets.iter().rev() {
// revert block changeset
for (address, change) in changeset.block_changesets.iter() {
self.revert_account(address, change)
}
for tx_changeset in changeset.tx_changesets.iter() {
// revert bytecodes
for (hash, _) in tx_changeset.new_bytecodes.iter() {
match self.bytecodes.entry(*hash) {
Entry::Vacant(_) => panic!("Bytecode should be present"),
Entry::Occupied(mut entry) => {
let (cnt, _) = entry.get_mut();
*cnt -= 1;
if *cnt == 0 {
entry.remove_entry();
}
}
}
}
// revert accounts
for (address, account_change) in tx_changeset.changeset.iter() {
// revert account
self.revert_account(address, &account_change.account);
// revert its storage
self.revert_storage(address, &account_change.storage);
}
}
}
}
/// Revert storage
fn revert_storage(&mut self, address: &Address, storage: &BTreeMap<U256, (U256, U256)>) {
if let Entry::Occupied(mut entry) = self.accounts.entry(*address) {
let account_storage = &mut entry.get_mut().storage;
for (key, (old_value, _)) in storage {
let key = H256(key.to_be_bytes());
account_storage.insert(key, *old_value);
}
}
}
/// Revert account
fn revert_account(&mut self, address: &Address, change: &AccountInfoChangeSet) {
match change {
AccountInfoChangeSet::Created { .. } => {
match self.accounts.entry(*address) {
Entry::Vacant(_) => {
// We inserted this account in apply fn.
panic!("It should be present, something is broken");
}
Entry::Occupied(mut entry) => {
let val = entry.get_mut();
if val.decr_storage_counter() {
// remove account that we didn't change from substate
entry.remove_entry();
return
}
val.info = Account::default();
val.storage.clear();
}
};
}
AccountInfoChangeSet::Destroyed { old } => match self.accounts.entry(*address) {
Entry::Vacant(_) => {
// We inserted this account in apply fn.
panic!("It should be present, something is broken");
}
Entry::Occupied(mut entry) => {
let val = entry.get_mut();
// Contrary to Created we are not removing this account as we dont know if
// this account was changer or not by `Changed` changeset.
val.decr_storage_counter();
val.info = *old;
}
},
AccountInfoChangeSet::Changed { old, .. } => {
self.accounts.entry(*address).or_default().info = *old;
}
AccountInfoChangeSet::NoChange { is_empty: _ } => {
// do nothing
}
}
}
}
/// Account changes in substate
#[derive(Debug, Clone, Default, Eq, PartialEq)]
pub struct AccountSubState {
/// New account state
pub info: Account,
/// If account is selfdestructed or newly created, storage will be cleared.
/// and we dont need to ask the provider for data.
/// As we need to have as
pub storage_is_clear: Option<u32>,
/// storage slots
pub storage: HashMap<H256, U256>,
}
impl AccountSubState {
/// Increment storage counter to mark this storage was cleared
pub fn inc_storage_counter(&mut self) {
self.storage_is_clear = Some(self.storage_is_clear.unwrap_or_default() + 1);
}
/// Decrement storage counter to represent that changeset that cleared storage was reverted.
pub fn decr_storage_counter(&mut self) -> bool {
let Some(cnt) = self.storage_is_clear else { return false};
if cnt == 1 {
self.storage_is_clear = None;
return true
}
false
}
/// Account is created
pub fn created_account(info: Account) -> Self {
Self { info, storage_is_clear: Some(1), storage: HashMap::new() }
}
/// Should we ask the provider for storage data
pub fn ask_provider(&self) -> bool {
self.storage_is_clear.is_none()
}
}
/// Wrapper around substate and provider, it decouples the database that can be Latest or historical
/// with substate changes that happened previously.
pub struct SubStateWithProvider<'a, SP: StateProvider> {
/// Substate
substate: &'a SubStateData,
/// Provider
provider: SP,
/// side chain block hashes
sidechain_block_hashes: &'a BTreeMap<BlockNumber, BlockHash>,
/// Last N canonical hashes,
canonical_block_hashes: &'a BTreeMap<BlockNumber, BlockHash>,
}
impl<'a, SP: StateProvider> SubStateWithProvider<'a, SP> {
/// Create new substate with provider
pub fn new(
substate: &'a SubStateData,
provider: SP,
sidechain_block_hashes: &'a BTreeMap<BlockNumber, BlockHash>,
canonical_block_hashes: &'a BTreeMap<BlockNumber, BlockHash>,
) -> Self {
Self { substate, provider, sidechain_block_hashes, canonical_block_hashes }
}
}
/* Implement StateProvider traits */
impl<'a, SP: StateProvider> BlockHashProvider for SubStateWithProvider<'a, SP> {
fn block_hash(&self, number: U256) -> Result<Option<H256>> {
// All block numbers fit inside u64 and revm checks if it is last 256 block numbers.
let block_number = number.as_limbs()[0];
if let Some(sidechain_block_hash) = self.sidechain_block_hashes.get(&block_number).cloned()
{
return Ok(Some(sidechain_block_hash))
}
Ok(Some(
self.canonical_block_hashes
.get(&block_number)
.cloned()
.ok_or(ProviderError::BlockchainTreeBlockHash { block_number })?,
))
}
}
impl<'a, SP: StateProvider> AccountProvider for SubStateWithProvider<'a, SP> {
fn basic_account(&self, address: Address) -> Result<Option<Account>> {
if let Some(account) = self.substate.accounts.get(&address).map(|acc| acc.info) {
return Ok(Some(account))
}
self.provider.basic_account(address)
}
}
impl<'a, SP: StateProvider> StateProvider for SubStateWithProvider<'a, SP> {
fn storage(
&self,
account: Address,
storage_key: reth_primitives::StorageKey,
) -> Result<Option<reth_primitives::StorageValue>> {
if let Some(substate_account) = self.substate.accounts.get(&account) {
if let Some(storage) = substate_account.storage.get(&storage_key) {
return Ok(Some(*storage))
}
if !substate_account.ask_provider() {
return Ok(Some(U256::ZERO))
}
}
self.provider.storage(account, storage_key)
}
/// Get account and storage proofs.
fn proof(
&self,
_address: Address,
_keys: &[H256],
) -> Result<(Vec<Bytes>, H256, Vec<Vec<Bytes>>)> {
Err(ProviderError::HistoryStateRoot.into())
}
fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytecode>> {
if let Some((_, bytecode)) = self.substate.bytecodes.get(&code_hash).cloned() {
return Ok(Some(bytecode))
}
self.provider.bytecode_by_hash(code_hash)
}
}