feat(stateless): Run EEST tests in stateless block validator & bug fixes (#18140)

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

testing/ef-tests/.gitignore

@@ -1 +1,2 @@
-ethereum-tests
+ethereum-tests
+execution-spec-tests

testing/ef-tests/src/cases/blockchain_test.rs

@@ -23,26 +23,31 @@ use reth_revm::{database::StateProviderDatabase, witness::ExecutionWitnessRecord
 use reth_stateless::{validation::stateless_validation, ExecutionWitness};
 use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot};
 use reth_trie_db::DatabaseStateRoot;
-use std::{collections::BTreeMap, fs, path::Path, sync::Arc};
+use std::{
+    collections::BTreeMap,
+    fs,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
 
 /// A handler for the blockchain test suite.
 #[derive(Debug)]
 pub struct BlockchainTests {
-    suite: String,
+    suite_path: PathBuf,
 }
 
 impl BlockchainTests {
-    /// Create a new handler for a subset of the blockchain test suite.
-    pub const fn new(suite: String) -> Self {
-        Self { suite }
+    /// Create a new suite for tests with blockchain tests format.
+    pub const fn new(suite_path: PathBuf) -> Self {
+        Self { suite_path }
     }
 }
 
 impl Suite for BlockchainTests {
     type Case = BlockchainTestCase;
 
-    fn suite_name(&self) -> String {
-        format!("BlockchainTests/{}", self.suite)
+    fn suite_path(&self) -> &Path {
+        &self.suite_path
     }
 }
 
@@ -157,7 +162,7 @@ impl Case for BlockchainTestCase {
     fn run(&self) -> Result<(), Error> {
         // If the test is marked for skipping, return a Skipped error immediately.
         if self.skip {
-            return Err(Error::Skipped)
+            return Err(Error::Skipped);
         }
 
         // Iterate through test cases, filtering by the network type to exclude specific forks.
@@ -306,18 +311,25 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
         parent = block.clone()
     }
 
-    // Validate the post-state for the test case.
-    //
-    // If we get here then it means that the post-state root checks
-    // made after we execute each block was successful.
-    //
-    // If an error occurs here, then it is:
-    // - Either an issue with the test setup
-    // - Possibly an error in the test case where the post-state root in the last block does not
-    //   match the post-state values.
-    let expected_post_state = case.post_state.as_ref().ok_or(Error::MissingPostState)?;
-    for (&address, account) in expected_post_state {
-        account.assert_db(address, provider.tx_ref())?;
+    match &case.post_state {
+        Some(expected_post_state) => {
+            // Validate the post-state for the test case.
+            //
+            // If we get here then it means that the post-state root checks
+            // made after we execute each block was successful.
+            //
+            // If an error occurs here, then it is:
+            // - Either an issue with the test setup
+            // - Possibly an error in the test case where the post-state root in the last block does
+            //   not match the post-state values.
+            for (address, account) in expected_post_state {
+                account.assert_db(*address, provider.tx_ref())?;
+            }
+        }
+        None => {
+            // Some test may not have post-state (e.g., state-heavy benchmark tests).
+            // In this case, we can skip the post-state validation.
+        }
     }
 
     // Now validate using the stateless client if everything else passes

testing/ef-tests/src/models.rs

@@ -5,7 +5,7 @@ use alloy_consensus::Header as RethHeader;
 use alloy_eips::eip4895::Withdrawals;
 use alloy_genesis::GenesisAccount;
 use alloy_primitives::{keccak256, Address, Bloom, Bytes, B256, B64, U256};
-use reth_chainspec::{ChainSpec, ChainSpecBuilder};
+use reth_chainspec::{ChainSpec, ChainSpecBuilder, EthereumHardfork, ForkCondition};
 use reth_db_api::{cursor::DbDupCursorRO, tables, transaction::DbTx};
 use reth_primitives_traits::SealedHeader;
 use serde::Deserialize;
@@ -294,9 +294,14 @@ pub enum ForkSpec {
     /// London
     London,
     /// Paris aka The Merge
+    #[serde(alias = "Paris")]
     Merge,
+    /// Paris to Shanghai at time 15k
+    ParisToShanghaiAtTime15k,
     /// Shanghai
     Shanghai,
+    /// Shanghai to Cancun at time 15k
+    ShanghaiToCancunAtTime15k,
     /// Merge EOF test
     #[serde(alias = "Merge+3540+3670")]
     MergeEOF,
@@ -308,39 +313,63 @@ pub enum ForkSpec {
     MergePush0,
     /// Cancun
     Cancun,
+    /// Cancun to Prague at time 15k
+    CancunToPragueAtTime15k,
     /// Prague
     Prague,
 }
 
 impl From<ForkSpec> for ChainSpec {
     fn from(fork_spec: ForkSpec) -> Self {
-        let spec_builder = ChainSpecBuilder::mainnet();
+        let spec_builder = ChainSpecBuilder::mainnet().reset();
 
         match fork_spec {
             ForkSpec::Frontier => spec_builder.frontier_activated(),
-            ForkSpec::Homestead | ForkSpec::FrontierToHomesteadAt5 => {
-                spec_builder.homestead_activated()
-            }
-            ForkSpec::EIP150 | ForkSpec::HomesteadToDaoAt5 | ForkSpec::HomesteadToEIP150At5 => {
-                spec_builder.tangerine_whistle_activated()
-            }
+            ForkSpec::FrontierToHomesteadAt5 => spec_builder
+                .frontier_activated()
+                .with_fork(EthereumHardfork::Homestead, ForkCondition::Block(5)),
+            ForkSpec::Homestead => spec_builder.homestead_activated(),
+            ForkSpec::HomesteadToDaoAt5 => spec_builder
+                .homestead_activated()
+                .with_fork(EthereumHardfork::Dao, ForkCondition::Block(5)),
+            ForkSpec::HomesteadToEIP150At5 => spec_builder
+                .homestead_activated()
+                .with_fork(EthereumHardfork::Tangerine, ForkCondition::Block(5)),
+            ForkSpec::EIP150 => spec_builder.tangerine_whistle_activated(),
             ForkSpec::EIP158 => spec_builder.spurious_dragon_activated(),
-            ForkSpec::Byzantium |
-            ForkSpec::EIP158ToByzantiumAt5 |
-            ForkSpec::ConstantinopleFix |
-            ForkSpec::ByzantiumToConstantinopleFixAt5 => spec_builder.byzantium_activated(),
+            ForkSpec::EIP158ToByzantiumAt5 => spec_builder
+                .spurious_dragon_activated()
+                .with_fork(EthereumHardfork::Byzantium, ForkCondition::Block(5)),
+            ForkSpec::Byzantium => spec_builder.byzantium_activated(),
+            ForkSpec::ByzantiumToConstantinopleAt5 => spec_builder
+                .byzantium_activated()
+                .with_fork(EthereumHardfork::Constantinople, ForkCondition::Block(5)),
+            ForkSpec::ByzantiumToConstantinopleFixAt5 => spec_builder
+                .byzantium_activated()
+                .with_fork(EthereumHardfork::Petersburg, ForkCondition::Block(5)),
+            ForkSpec::Constantinople => spec_builder.constantinople_activated(),
+            ForkSpec::ConstantinopleFix => spec_builder.petersburg_activated(),
             ForkSpec::Istanbul => spec_builder.istanbul_activated(),
             ForkSpec::Berlin => spec_builder.berlin_activated(),
-            ForkSpec::London | ForkSpec::BerlinToLondonAt5 => spec_builder.london_activated(),
+            ForkSpec::BerlinToLondonAt5 => spec_builder
+                .berlin_activated()
+                .with_fork(EthereumHardfork::London, ForkCondition::Block(5)),
+            ForkSpec::London => spec_builder.london_activated(),
             ForkSpec::Merge |
             ForkSpec::MergeEOF |
             ForkSpec::MergeMeterInitCode |
             ForkSpec::MergePush0 => spec_builder.paris_activated(),
+            ForkSpec::ParisToShanghaiAtTime15k => spec_builder
+                .paris_activated()
+                .with_fork(EthereumHardfork::Shanghai, ForkCondition::Timestamp(15_000)),
             ForkSpec::Shanghai => spec_builder.shanghai_activated(),
+            ForkSpec::ShanghaiToCancunAtTime15k => spec_builder
+                .shanghai_activated()
+                .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(15_000)),
             ForkSpec::Cancun => spec_builder.cancun_activated(),
-            ForkSpec::ByzantiumToConstantinopleAt5 | ForkSpec::Constantinople => {
-                panic!("Overridden with PETERSBURG")
-            }
+            ForkSpec::CancunToPragueAtTime15k => spec_builder
+                .cancun_activated()
+                .with_fork(EthereumHardfork::Prague, ForkCondition::Timestamp(15_000)),
             ForkSpec::Prague => spec_builder.prague_activated(),
         }
         .build()

testing/ef-tests/src/result.rs

@@ -17,9 +17,6 @@ pub enum Error {
     /// The test was skipped
     #[error("test was skipped")]
     Skipped,
-    /// No post state found in test
-    #[error("no post state found for validation")]
-    MissingPostState,
     /// Block processing failed
     /// Note: This includes but is not limited to execution.
     /// For example, the header number could be incorrect.

testing/ef-tests/src/suite.rs

@@ -12,25 +12,28 @@ pub trait Suite {
     /// The type of test cases in this suite.
     type Case: Case;
 
-    /// The name of the test suite used to locate the individual test cases.
-    ///
-    /// # Example
-    ///
-    /// - `GeneralStateTests`
-    /// - `BlockchainTests/InvalidBlocks`
-    /// - `BlockchainTests/TransitionTests`
-    fn suite_name(&self) -> String;
+    /// The path to the test suite directory.
+    fn suite_path(&self) -> &Path;
 
-    /// Load and run each contained test case.
+    /// Run all test cases in the suite.
+    fn run(&self) {
+        let suite_path = self.suite_path();
+        for entry in WalkDir::new(suite_path).min_depth(1).max_depth(1) {
+            let entry = entry.expect("Failed to read directory");
+            if entry.file_type().is_dir() {
+                self.run_only(entry.file_name().to_string_lossy().as_ref());
+            }
+        }
+    }
+
+    /// Load and run each contained test case for the provided sub-folder.
     ///
     /// # Note
     ///
     /// This recursively finds every test description in the resulting path.
-    fn run(&self) {
+    fn run_only(&self, name: &str) {
         // Build the path to the test suite directory
-        let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
-            .join("ethereum-tests")
-            .join(self.suite_name());
+        let suite_path = self.suite_path().join(name);
 
         // Verify that the path exists
         assert!(suite_path.exists(), "Test suite path does not exist: {suite_path:?}");
@@ -48,7 +51,7 @@ pub trait Suite {
         let results = Cases { test_cases }.run();
 
         // Assert that all tests in the suite pass
-        assert_tests_pass(&self.suite_name(), &suite_path, &results);
+        assert_tests_pass(name, &suite_path, &results);
     }
 }
 

testing/ef-tests/tests/tests.rs

@@ -2,13 +2,19 @@
 #![cfg(feature = "ef-tests")]
 
 use ef_tests::{cases::blockchain_test::BlockchainTests, suite::Suite};
+use std::path::PathBuf;
 
 macro_rules! general_state_test {
     ($test_name:ident, $dir:ident) => {
         #[test]
         fn $test_name() {
             reth_tracing::init_test_tracing();
-            BlockchainTests::new(format!("GeneralStateTests/{}", stringify!($dir))).run();
+            let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .join("ethereum-tests")
+                .join("BlockchainTests");
+
+            BlockchainTests::new(suite_path)
+                .run_only(&format!("GeneralStateTests/{}", stringify!($dir)));
         }
     };
 }
@@ -83,10 +89,24 @@ macro_rules! blockchain_test {
         #[test]
         fn $test_name() {
             reth_tracing::init_test_tracing();
-            BlockchainTests::new(format!("{}", stringify!($dir))).run();
+            let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+                .join("ethereum-tests")
+                .join("BlockchainTests");
+
+            BlockchainTests::new(suite_path).run_only(&format!("{}", stringify!($dir)));
         }
     };
 }
 
 blockchain_test!(valid_blocks, ValidBlocks);
 blockchain_test!(invalid_blocks, InvalidBlocks);
+
+#[test]
+fn eest_fixtures() {
+    reth_tracing::init_test_tracing();
+    let suite_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .join("execution-spec-tests")
+        .join("blockchain_tests");
+
+    BlockchainTests::new(suite_path).run();
+}