feat(engine): new payload execution (#631)

* feat(engine): new payload execution

* address PR comments

* rm unused dev deps

* add comment about lru

* remove par_iter

crates/storage/provider/Cargo.toml

@@ -30,6 +30,7 @@ rand = "0.8.5"
 arbitrary = { version = "1.1.7", features = ["derive"], optional = true }
 secp256k1 = { version = "0.24.2", default-features = false, features = ["alloc", "recovery", "rand"], optional = true }
 modular-bitfield = "0.11.2"
+parking_lot = { version = "0.12", optional = true }
 
 [dev-dependencies]
 reth-db = { path = "../db", features = ["test-utils"] }
@@ -39,7 +40,8 @@ tokio-stream = { version = "0.1.11", features = ["sync"] }
 arbitrary = { version = "1.1.7", features = ["derive"]}
 hex-literal = "0.3"
 secp256k1 = { version = "0.24.2", default-features = false, features = ["alloc", "recovery", "rand"] }
+parking_lot = "0.12"
 
 [features]
 bench = []
-test-utils = ["tokio-stream/sync", "secp256k1"]
+test-utils = ["tokio-stream/sync", "secp256k1", "parking_lot"]

crates/storage/provider/src/block.rs

@@ -10,6 +10,14 @@ use reth_primitives::{
     Block, BlockHash, BlockHashOrNumber, Header, SealedBlock, H256, U256,
 };
 
+/// Client trait for fetching block hashes by number.
+#[auto_impl(&)]
+pub trait BlockHashProvider: Send + Sync {
+    /// Get the hash of the block with the given number. Returns `None` if no block with this number
+    /// exists.
+    fn block_hash(&self, number: U256) -> Result<Option<H256>>;
+}
+
 /// Client trait for fetching `Header` related data.
 #[auto_impl(&)]
 pub trait HeaderProvider: Send + Sync {
@@ -37,7 +45,7 @@ pub trait HeaderProvider: Send + Sync {
 }
 
 /// Api trait for fetching `Block` related data.
-pub trait BlockProvider: Send + Sync {
+pub trait BlockProvider: BlockHashProvider + Send + Sync {
     /// Returns the current info for the chain.
     fn chain_info(&self) -> Result<ChainInfo>;
 
@@ -89,10 +97,6 @@ pub trait BlockProvider: Send + Sync {
 
     /// Gets the `Block` for the given hash. Returns `None` if no block with this hash exists.
     fn block_number(&self, hash: H256) -> Result<Option<reth_primitives::BlockNumber>>;
-
-    /// Get the hash of the block with the given number. Returns `None` if no block with this number
-    /// exists.
-    fn block_hash(&self, number: U256) -> Result<Option<H256>>;
 }
 
 /// Current status of the blockchain's head.

crates/storage/provider/src/db_provider/block.rs

@@ -1,4 +1,4 @@
-use crate::{BlockProvider, ChainInfo, HeaderProvider, ProviderImpl};
+use crate::{block::BlockHashProvider, BlockProvider, ChainInfo, HeaderProvider, ProviderImpl};
 use reth_db::{database::Database, tables, transaction::DbTx};
 use reth_interfaces::Result;
 use reth_primitives::{rpc::BlockId, Block, BlockHash, BlockNumber, Header, H256, U256};
@@ -26,6 +26,15 @@ impl<DB: Database> HeaderProvider for ProviderImpl<DB> {
     }
 }
 
+impl<DB: Database> BlockHashProvider for ProviderImpl<DB> {
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        // TODO: This unwrap is potentially unsafe
+        self.db
+            .view(|tx| tx.get::<tables::CanonicalHeaders>(number.try_into().unwrap()))?
+            .map_err(Into::into)
+    }
+}
+
 impl<DB: Database> BlockProvider for ProviderImpl<DB> {
     fn chain_info(&self) -> Result<ChainInfo> {
         Ok(ChainInfo {
@@ -44,11 +53,4 @@ impl<DB: Database> BlockProvider for ProviderImpl<DB> {
     fn block_number(&self, hash: H256) -> Result<Option<BlockNumber>> {
         self.db.view(|tx| tx.get::<tables::HeaderNumbers>(hash))?.map_err(Into::into)
     }
-
-    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
-        // TODO: This unwrap is potentially unsafe
-        self.db
-            .view(|tx| tx.get::<tables::CanonicalHeaders>(number.try_into().unwrap()))?
-            .map_err(Into::into)
-    }
 }

crates/storage/provider/src/db_provider/storage.rs

@@ -1,5 +1,7 @@
 use super::ProviderImpl;
-use crate::{AccountProvider, Error, StateProvider, StateProviderFactory};
+use crate::{
+    block::BlockHashProvider, AccountProvider, Error, StateProvider, StateProviderFactory,
+};
 use reth_db::{
     cursor::{DbCursorRO, DbDupCursorRO},
     database::{Database, DatabaseGAT},
@@ -78,6 +80,12 @@ impl<'a, TX: DbTx<'a>> AccountProvider for StateProviderImplHistory<'a, TX> {
     }
 }
 
+impl<'a, TX: DbTx<'a>> BlockHashProvider for StateProviderImplHistory<'a, TX> {
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        StateProviderImplRefHistory::new(&self.tx, self.transition).block_hash(number)
+    }
+}
+
 impl<'a, TX: DbTx<'a>> StateProvider for StateProviderImplHistory<'a, TX> {
     fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>> {
         StateProviderImplRefHistory::new(&self.tx, self.transition).storage(account, storage_key)
@@ -86,10 +94,6 @@ impl<'a, TX: DbTx<'a>> StateProvider for StateProviderImplHistory<'a, TX> {
     fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>> {
         StateProviderImplRefHistory::new(&self.tx, self.transition).bytecode_by_hash(code_hash)
     }
-
-    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
-        StateProviderImplRefHistory::new(&self.tx, self.transition).block_hash(number)
-    }
 }
 /// State provider with given hash
 ///
@@ -123,6 +127,13 @@ impl<'a, 'b, TX: DbTx<'a>> AccountProvider for StateProviderImplRefHistory<'a, '
     }
 }
 
+impl<'a, 'b, TX: DbTx<'a>> BlockHashProvider for StateProviderImplRefHistory<'a, 'b, TX> {
+    /// Get block hash by number.
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        self.tx.get::<tables::CanonicalHeaders>(number.as_u64()).map_err(Into::into)
+    }
+}
+
 impl<'a, 'b, TX: DbTx<'a>> StateProvider for StateProviderImplRefHistory<'a, 'b, TX> {
     /// Get storage.
     fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>> {
@@ -156,11 +167,6 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for StateProviderImplRefHistory<'a, 'b,
     fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>> {
         self.tx.get::<tables::Bytecodes>(code_hash).map_err(Into::into).map(|r| r.map(Bytes::from))
     }
-
-    /// Get block hash by number.
-    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
-        self.tx.get::<tables::CanonicalHeaders>(number.as_u64()).map_err(Into::into)
-    }
 }
 
 /// State provider for latests state
@@ -185,6 +191,12 @@ impl<'a, TX: DbTx<'a>> AccountProvider for StateProviderImplLatest<'a, TX> {
     }
 }
 
+impl<'a, TX: DbTx<'a>> BlockHashProvider for StateProviderImplLatest<'a, TX> {
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        StateProviderImplRefLatest::new(&self.db).block_hash(number)
+    }
+}
+
 impl<'a, TX: DbTx<'a>> StateProvider for StateProviderImplLatest<'a, TX> {
     fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>> {
         StateProviderImplRefLatest::new(&self.db).storage(account, storage_key)
@@ -193,10 +205,6 @@ impl<'a, TX: DbTx<'a>> StateProvider for StateProviderImplLatest<'a, TX> {
     fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>> {
         StateProviderImplRefLatest::new(&self.db).bytecode_by_hash(code_hash)
     }
-
-    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
-        StateProviderImplRefLatest::new(&self.db).block_hash(number)
-    }
 }
 
 /// State Provider over latest state that takes tx reference
@@ -221,6 +229,13 @@ impl<'a, 'b, TX: DbTx<'a>> AccountProvider for StateProviderImplRefLatest<'a, 'b
     }
 }
 
+impl<'a, 'b, TX: DbTx<'a>> BlockHashProvider for StateProviderImplRefLatest<'a, 'b, TX> {
+    /// Get block hash by number.
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        self.db.get::<tables::CanonicalHeaders>(number.as_u64()).map_err(Into::into)
+    }
+}
+
 impl<'a, 'b, TX: DbTx<'a>> StateProvider for StateProviderImplRefLatest<'a, 'b, TX> {
     /// Get storage.
     fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>> {
@@ -237,9 +252,4 @@ impl<'a, 'b, TX: DbTx<'a>> StateProvider for StateProviderImplRefLatest<'a, 'b,
     fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>> {
         self.db.get::<tables::Bytecodes>(code_hash).map_err(Into::into).map(|r| r.map(Bytes::from))
     }
-
-    /// Get block hash by number.
-    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
-        self.db.get::<tables::CanonicalHeaders>(number.as_u64()).map_err(Into::into)
-    }
 }

crates/storage/provider/src/lib.rs

@@ -16,7 +16,9 @@ mod state;
 /// Common test helpers for mocking the Provider.
 pub mod test_utils;
 
-pub use block::{insert_canonical_block, BlockProvider, ChainInfo, HeaderProvider};
+pub use block::{
+    insert_canonical_block, BlockHashProvider, BlockProvider, ChainInfo, HeaderProvider,
+};
 pub use db_provider::{
     self as db, ProviderImpl, StateProviderImplHistory, StateProviderImplLatest,
     StateProviderImplRefHistory, StateProviderImplRefLatest,

crates/storage/provider/src/state.rs

@@ -1,24 +1,26 @@
+use auto_impl::auto_impl;
 use reth_interfaces::Result;
 use reth_primitives::{
-    Account, Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, H256, U256,
+    Account, Address, BlockHash, BlockNumber, Bytes, StorageKey, StorageValue, H256,
 };
 
+use crate::block::BlockHashProvider;
+
 /// Account provider
+#[auto_impl(&)]
 pub trait AccountProvider: Send + Sync {
     /// Get basic account information.
     fn basic_account(&self, address: Address) -> Result<Option<Account>>;
 }
 
 /// Function needed for executor.
-pub trait StateProvider: AccountProvider + Send + Sync {
+#[auto_impl(&)]
+pub trait StateProvider: BlockHashProvider + AccountProvider + Send + Sync {
     /// Get storage.
     fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>>;
 
     /// Get account code by its hash
     fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>>;
-
-    /// Get block hash by number.
-    fn block_hash(&self, number: U256) -> Result<Option<H256>>;
 }
 
 /// Light wrapper that creates StateProvider.

crates/storage/provider/src/test_utils/mock.rs

@@ -0,0 +1,177 @@
+use parking_lot::Mutex;
+use reth_interfaces::Result;
+use reth_primitives::{
+    keccak256,
+    rpc::{BlockId, BlockNumber},
+    Account, Address, Block, BlockHash, Bytes, Header, StorageKey, StorageValue, H256, U256,
+};
+use std::{collections::HashMap, sync::Arc};
+
+use crate::{
+    AccountProvider, BlockHashProvider, BlockProvider, ChainInfo, HeaderProvider, StateProvider,
+};
+
+/// A mock implementation for Provider interfaces.
+#[derive(Debug, Clone, Default)]
+pub struct MockEthProvider {
+    /// Local block store
+    pub blocks: Arc<Mutex<HashMap<H256, Block>>>,
+    /// Local header store
+    pub headers: Arc<Mutex<HashMap<H256, Header>>>,
+    /// Local account store
+    pub accounts: Arc<Mutex<HashMap<Address, ExtendedAccount>>>,
+}
+
+/// An extended account for local store
+#[derive(Debug, Clone)]
+pub struct ExtendedAccount {
+    account: Account,
+    bytecode: Option<Bytes>,
+    storage: HashMap<StorageKey, StorageValue>,
+}
+
+impl ExtendedAccount {
+    /// Create new instance of extended account
+    pub fn new(nonce: u64, balance: U256) -> Self {
+        Self {
+            account: Account { nonce, balance, bytecode_hash: None },
+            bytecode: None,
+            storage: Default::default(),
+        }
+    }
+
+    /// Set bytecode and bytecode hash on the extended account
+    pub fn with_bytecode(mut self, bytecode: Bytes) -> Self {
+        let hash = keccak256(&bytecode);
+        self.account.bytecode_hash = Some(hash);
+        self.bytecode = Some(bytecode);
+        self
+    }
+}
+
+impl MockEthProvider {
+    /// Add block to local block store
+    pub fn add_block(&self, hash: H256, block: Block) {
+        self.blocks.lock().insert(hash, block);
+    }
+
+    /// Add multiple blocks to local block store
+    pub fn extend_blocks(&self, iter: impl IntoIterator<Item = (H256, Block)>) {
+        for (hash, block) in iter.into_iter() {
+            self.add_block(hash, block)
+        }
+    }
+
+    /// Add header to local header store
+    pub fn add_header(&self, hash: H256, header: Header) {
+        self.headers.lock().insert(hash, header);
+    }
+
+    /// Add multiple headers to local header store
+    pub fn extend_headers(&self, iter: impl IntoIterator<Item = (H256, Header)>) {
+        for (hash, header) in iter.into_iter() {
+            self.add_header(hash, header)
+        }
+    }
+
+    /// Add account to local account store
+    pub fn add_account(&self, address: Address, account: ExtendedAccount) {
+        self.accounts.lock().insert(address, account);
+    }
+
+    /// Add account to local account store
+    pub fn extend_accounts(&self, iter: impl IntoIterator<Item = (Address, ExtendedAccount)>) {
+        for (address, account) in iter.into_iter() {
+            self.add_account(address, account)
+        }
+    }
+}
+
+impl HeaderProvider for MockEthProvider {
+    fn header(&self, block_hash: &BlockHash) -> Result<Option<Header>> {
+        let lock = self.headers.lock();
+        Ok(lock.get(block_hash).cloned())
+    }
+
+    fn header_by_number(&self, num: u64) -> Result<Option<Header>> {
+        let lock = self.headers.lock();
+        Ok(lock.values().find(|h| h.number == num).cloned())
+    }
+
+    fn header_td(&self, hash: &BlockHash) -> Result<Option<U256>> {
+        let lock = self.headers.lock();
+        Ok(lock.get(hash).map(|target| {
+            lock.values()
+                .filter(|h| h.number < target.number)
+                .fold(target.difficulty, |td, h| td + h.difficulty)
+        }))
+    }
+}
+
+impl BlockHashProvider for MockEthProvider {
+    fn block_hash(&self, number: U256) -> Result<Option<H256>> {
+        let lock = self.blocks.lock();
+
+        let hash =
+            lock.iter().find_map(
+                |(hash, b)| {
+                    if b.number == number.as_u64() {
+                        Some(*hash)
+                    } else {
+                        None
+                    }
+                },
+            );
+        Ok(hash)
+    }
+}
+
+impl BlockProvider for MockEthProvider {
+    fn chain_info(&self) -> Result<ChainInfo> {
+        todo!()
+    }
+
+    fn block(&self, id: BlockId) -> Result<Option<Block>> {
+        let lock = self.blocks.lock();
+        match id {
+            BlockId::Hash(hash) => Ok(lock.get(&hash).cloned()),
+            BlockId::Number(BlockNumber::Number(num)) => {
+                Ok(lock.values().find(|b| b.number == num.as_u64()).cloned())
+            }
+            _ => {
+                unreachable!("unused in network tests")
+            }
+        }
+    }
+
+    fn block_number(&self, hash: H256) -> Result<Option<reth_primitives::BlockNumber>> {
+        let lock = self.blocks.lock();
+        let num = lock.iter().find_map(|(h, b)| if *h == hash { Some(b.number) } else { None });
+        Ok(num)
+    }
+}
+
+impl AccountProvider for MockEthProvider {
+    fn basic_account(&self, address: Address) -> Result<Option<Account>> {
+        Ok(self.accounts.lock().get(&address).cloned().map(|a| a.account))
+    }
+}
+
+impl StateProvider for MockEthProvider {
+    fn bytecode_by_hash(&self, code_hash: H256) -> Result<Option<Bytes>> {
+        let lock = self.accounts.lock();
+        Ok(lock.values().find_map(|account| {
+            match (account.account.bytecode_hash.as_ref(), account.bytecode.as_ref()) {
+                (Some(bytecode_hash), Some(bytecode)) if *bytecode_hash == code_hash => {
+                    Some(bytecode.clone())
+                }
+                _ => None,
+            }
+        }))
+    }
+
+    fn storage(&self, account: Address, storage_key: StorageKey) -> Result<Option<StorageValue>> {
+        let lock = self.accounts.lock();
+        Ok(lock.get(&account).and_then(|account| account.storage.get(&storage_key)).cloned())
+    }
+}

crates/storage/provider/src/test_utils/mod.rs

@@ -1,3 +1,5 @@
-mod api;
+mod mock;
+mod noop;
 
-pub use api::TestApi;
+pub use mock::{ExtendedAccount, MockEthProvider};
+pub use noop::NoopProvider;

crates/storage/provider/src/test_utils/noop.rs

@@ -1,14 +1,20 @@
-use crate::{BlockProvider, ChainInfo, HeaderProvider};
+use crate::{BlockHashProvider, BlockProvider, ChainInfo, HeaderProvider};
 use reth_interfaces::Result;
 use reth_primitives::{rpc::BlockId, Block, BlockHash, BlockNumber, Header, H256, U256};
 
 /// Supports various api interfaces for testing purposes.
 #[derive(Debug, Clone, Default)]
 #[non_exhaustive]
-pub struct TestApi;
+pub struct NoopProvider;
 
 /// Noop implementation for testing purposes
-impl BlockProvider for TestApi {
+impl BlockHashProvider for NoopProvider {
+    fn block_hash(&self, _number: U256) -> Result<Option<H256>> {
+        Ok(None)
+    }
+}
+
+impl BlockProvider for NoopProvider {
     fn chain_info(&self) -> Result<ChainInfo> {
         Ok(ChainInfo {
             best_hash: Default::default(),
@@ -25,13 +31,9 @@ impl BlockProvider for TestApi {
     fn block_number(&self, _hash: H256) -> Result<Option<BlockNumber>> {
         Ok(None)
     }
-
-    fn block_hash(&self, _number: U256) -> Result<Option<H256>> {
-        Ok(None)
-    }
 }
 
-impl HeaderProvider for TestApi {
+impl HeaderProvider for NoopProvider {
     fn header(&self, _block_hash: &BlockHash) -> Result<Option<Header>> {
         Ok(None)
     }