feat(stateless): enable test runs to return execution witness (#18740)

Signed-off-by: Ignacio Hagopian <jsign.uy@gmail.com>
Co-authored-by: Kevaundray Wedderburn <kevtheappdev@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>

crates/evm/evm/src/execute.rs

@@ -107,6 +107,23 @@ pub trait Executor<DB: Database>: Sized {
         Ok(BlockExecutionOutput { state: state.take_bundle(), result })
     }
 
+    /// Executes the EVM with the given input and accepts a state closure that is always invoked
+    /// with the EVM state after execution, even after failure.
+    fn execute_with_state_closure_always<F>(
+        mut self,
+        block: &RecoveredBlock<<Self::Primitives as NodePrimitives>::Block>,
+        mut f: F,
+    ) -> Result<BlockExecutionOutput<<Self::Primitives as NodePrimitives>::Receipt>, Self::Error>
+    where
+        F: FnMut(&State<DB>),
+    {
+        let result = self.execute_one(block);
+        let mut state = self.into_state();
+        f(&state);
+
+        Ok(BlockExecutionOutput { state: state.take_bundle(), result: result? })
+    }
+
     /// Executes the EVM with the given input and accepts a state hook closure that is invoked with
     /// the EVM state after execution.
     fn execute_with_state_hook<F>(

testing/ef-tests/src/cases/blockchain_test.rs

@@ -54,8 +54,10 @@ impl Suite for BlockchainTests {
 /// An Ethereum blockchain test.
 #[derive(Debug, PartialEq, Eq)]
 pub struct BlockchainTestCase {
-    tests: BTreeMap<String, BlockchainTest>,
+    /// The tests within this test case.
-    skip: bool,
+    pub tests: BTreeMap<String, BlockchainTest>,
+    /// Whether to skip this test case.
+    pub skip: bool,
 }
 
 impl BlockchainTestCase {
@@ -96,39 +98,45 @@ impl BlockchainTestCase {
     }
 
     /// Execute a single `BlockchainTest`, validating the outcome against the
-    /// expectations encoded in the JSON file.
+    /// expectations encoded in the JSON file. Returns the list of executed blocks
-    fn run_single_case(name: &str, case: &BlockchainTest) -> Result<(), Error> {
+    /// with their execution witnesses.
+    pub fn run_single_case(
+        name: &str,
+        case: &BlockchainTest,
+    ) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
         let expectation = Self::expected_failure(case);
         match run_case(case) {
             // All blocks executed successfully.
-            Ok(()) => {
+            Ok(program_inputs) => {
                 // Check if the test case specifies that it should have failed
                 if let Some((block, msg)) = expectation {
                     Err(Error::Assertion(format!(
                         "Test case: {name}\nExpected failure at block {block} - {msg}, but all blocks succeeded",
                     )))
                 } else {
-                    Ok(())
+                    Ok(program_inputs)
                 }
             }
 
             // A block processing failure occurred.
-            err @ Err(Error::BlockProcessingFailed { block_number, .. }) => match expectation {
+            Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }) => {
-                // It happened on exactly the block we were told to fail on
+                match expectation {
-                Some((expected, _)) if block_number == expected => Ok(()),
+                    // It happened on exactly the block we were told to fail on
+                    Some((expected, _)) if block_number == expected => Ok(partial_program_inputs),
 
-                // Uncle side‑chain edge case, we accept as long as it failed.
+                    // Uncle side‑chain edge case, we accept as long as it failed.
-                // But we don't check the exact block number.
+                    // But we don't check the exact block number.
-                _ if Self::is_uncle_sidechain_case(name) => Ok(()),
+                    _ if Self::is_uncle_sidechain_case(name) => Ok(partial_program_inputs),
 
-                // Expected failure, but block number does not match
+                    // Expected failure, but block number does not match
-                Some((expected, _)) => Err(Error::Assertion(format!(
+                    Some((expected, _)) => Err(Error::Assertion(format!(
-                    "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}",
+                        "Test case: {name}\nExpected failure at block {expected}\nGot failure at block {block_number}",
-                ))),
+                    ))),
 
-                // No failure expected at all - bubble up original error.
+                    // No failure expected at all - bubble up original error.
-                None => err,
+                    None => Err(Error::BlockProcessingFailed { block_number, partial_program_inputs, err }),
-            },
+                }
+            }
 
             // Non‑processing error – forward as‑is.
             //
@@ -170,14 +178,14 @@ impl Case for BlockchainTestCase {
             .iter()
             .filter(|(_, case)| !Self::excluded_fork(case.network))
             .par_bridge()
-            .try_for_each(|(name, case)| Self::run_single_case(name, case))?;
+            .try_for_each(|(name, case)| Self::run_single_case(name, case).map(|_| ()))?;
 
         Ok(())
     }
 }
 
-/// Executes a single `BlockchainTest`, returning an error if the blockchain state
+/// Executes a single `BlockchainTest` returning an error as soon as any block has a consensus
-/// does not match the expected outcome after all blocks are executed.
+/// validation failure.
 ///
 /// A `BlockchainTest` represents a self-contained scenario:
 /// - It initializes a fresh blockchain state.
@@ -186,9 +194,13 @@ impl Case for BlockchainTestCase {
 ///   outcome.
 ///
 /// Returns:
-/// - `Ok(())` if all blocks execute successfully and the final state is correct.
+/// - `Ok(_)` if all blocks execute successfully, returning recovered blocks and full block
-/// - `Err(Error)` if any block fails to execute correctly, or if the post-state validation fails.
+///   execution witness.
-fn run_case(case: &BlockchainTest) -> Result<(), Error> {
+/// - `Err(Error)` if any block fails to execute correctly, returning a partial block execution
+///   witness if the error is of variant `BlockProcessingFailed`.
+fn run_case(
+    case: &BlockchainTest,
+) -> Result<Vec<(RecoveredBlock<Block>, ExecutionWitness)>, Error> {
     // Create a new test database and initialize a provider for the test case.
     let chain_spec: Arc<ChainSpec> = Arc::new(case.network.into());
     let factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
@@ -202,22 +214,24 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
     .try_recover()
     .unwrap();
 
-    provider.insert_block(genesis_block.clone()).map_err(|err| Error::block_failed(0, err))?;
+    provider
+        .insert_block(genesis_block.clone())
+        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
 
     // Increment block number for receipts static file
     provider
         .static_file_provider()
         .latest_writer(StaticFileSegment::Receipts)
         .and_then(|mut writer| writer.increment_block(0))
-        .map_err(|err| Error::block_failed(0, err))?;
+        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
 
     let genesis_state = case.pre.clone().into_genesis_state();
     insert_genesis_state(&provider, genesis_state.iter())
-        .map_err(|err| Error::block_failed(0, err))?;
+        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
     insert_genesis_hashes(&provider, genesis_state.iter())
-        .map_err(|err| Error::block_failed(0, err))?;
+        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
     insert_genesis_history(&provider, genesis_state.iter())
-        .map_err(|err| Error::block_failed(0, err))?;
+        .map_err(|err| Error::block_failed(0, Default::default(), err))?;
 
     // Decode blocks
     let blocks = decode_blocks(&case.blocks)?;
@@ -233,16 +247,18 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
         // Insert the block into the database
         provider
             .insert_block(block.clone())
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
         // Commit static files, so we can query the headers for stateless execution below
         provider
             .static_file_provider()
             .commit()
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
 
         // Consensus checks before block execution
-        pre_execution_checks(chain_spec.clone(), &parent, block)
+        pre_execution_checks(chain_spec.clone(), &parent, block).map_err(|err| {
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            program_inputs.push((block.clone(), execution_witness_with_parent(&parent)));
+            Error::block_failed(block_number, program_inputs.clone(), err)
+        })?;
 
         let mut witness_record = ExecutionWitnessRecord::default();
 
@@ -252,14 +268,14 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
         let executor = executor_provider.batch_executor(state_db);
 
         let output = executor
-            .execute_with_state_closure(&(*block).clone(), |statedb: &State<_>| {
+            .execute_with_state_closure_always(&(*block).clone(), |statedb: &State<_>| {
                 witness_record.record_executed_state(statedb);
             })
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
 
         // Consensus checks after block execution
         validate_block_post_execution(block, &chain_spec, &output.receipts, &output.requests)
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
 
         // Generate the stateless witness
         // TODO: Most of this code is copy-pasted from debug_executionWitness
@@ -293,25 +309,26 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
             HashedPostState::from_bundle_state::<KeccakKeyHasher>(output.state.state());
         let (computed_state_root, _) =
             StateRoot::overlay_root_with_updates(provider.tx_ref(), hashed_state.clone())
-                .map_err(|err| Error::block_failed(block_number, err))?;
+                .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
         if computed_state_root != block.state_root {
             return Err(Error::block_failed(
                 block_number,
+                program_inputs.clone(),
                 Error::Assertion("state root mismatch".to_string()),
-            ))
+            ));
         }
 
         // Commit the post state/state diff to the database
         provider
             .write_state(&ExecutionOutcome::single(block.number, output), OriginalValuesKnown::Yes)
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
 
         provider
             .write_hashed_state(&hashed_state.into_sorted())
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
         provider
             .update_history_indices(block.number..=block.number)
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, program_inputs.clone(), err))?;
 
         // Since there were no errors, update the parent block
         parent = block.clone()
@@ -339,17 +356,17 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> {
     }
 
     // Now validate using the stateless client if everything else passes
-    for (block, execution_witness) in program_inputs {
+    for (block, execution_witness) in &program_inputs {
         stateless_validation(
-            block,
+            block.clone(),
-            execution_witness,
+            execution_witness.clone(),
             chain_spec.clone(),
             EthEvmConfig::new(chain_spec.clone()),
         )
         .expect("stateless validation failed");
     }
 
-    Ok(())
+    Ok(program_inputs)
 }
 
 fn decode_blocks(
@@ -362,10 +379,12 @@ fn decode_blocks(
         let block_number = (block_index + 1) as u64;
 
         let decoded = SealedBlock::<Block>::decode(&mut block.rlp.as_ref())
-            .map_err(|err| Error::block_failed(block_number, err))?;
+            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
 
-        let recovered_block =
+        let recovered_block = decoded
-            decoded.clone().try_recover().map_err(|err| Error::block_failed(block_number, err))?;
+            .clone()
+            .try_recover()
+            .map_err(|err| Error::block_failed(block_number, Default::default(), err))?;
 
         blocks.push(recovered_block);
     }
@@ -460,3 +479,9 @@ fn path_contains(path_str: &str, rhs: &[&str]) -> bool {
     let rhs = rhs.join(std::path::MAIN_SEPARATOR_STR);
     path_str.contains(&rhs)
 }
+
+fn execution_witness_with_parent(parent: &RecoveredBlock<Block>) -> ExecutionWitness {
+    let mut serialized_header = Vec::new();
+    parent.header().encode(&mut serialized_header);
+    ExecutionWitness { headers: vec![serialized_header.into()], ..Default::default() }
+}

testing/ef-tests/src/result.rs

@@ -2,7 +2,10 @@
 
 use crate::Case;
 use reth_db::DatabaseError;
+use reth_ethereum_primitives::Block;
+use reth_primitives_traits::RecoveredBlock;
 use reth_provider::ProviderError;
+use reth_stateless::ExecutionWitness;
 use std::path::{Path, PathBuf};
 use thiserror::Error;
 
@@ -24,6 +27,9 @@ pub enum Error {
     BlockProcessingFailed {
         /// The block number for the block that failed
         block_number: u64,
+        /// Contains the inputs necessary for the block stateless validation guest program used in
+        /// zkVMs to prove the block is invalid.
+        partial_program_inputs: Vec<(RecoveredBlock<Block>, ExecutionWitness)>,
         /// The specific error
         #[source]
         err: Box<dyn std::error::Error + Send + Sync>,
@@ -67,9 +73,10 @@ impl Error {
     /// Create a new [`Error::BlockProcessingFailed`] error.
     pub fn block_failed(
         block_number: u64,
+        partial_program_inputs: Vec<(RecoveredBlock<Block>, ExecutionWitness)>,
         err: impl std::error::Error + Send + Sync + 'static,
     ) -> Self {
-        Self::BlockProcessingFailed { block_number, err: Box::new(err) }
+        Self::BlockProcessingFailed { block_number, partial_program_inputs, err: Box::new(err) }
     }
 }