reth/testing/ef-tests/src/result.rs

//! Test results and errors

use crate::Case;
use reth_db::DatabaseError;
use reth_ethereum_primitives::Block;
use reth_primitives_traits::RecoveredBlock;
use reth_provider::ProviderError;
use reth_stateless::ExecutionWitness;
use std::path::{Path, PathBuf};
use thiserror::Error;

/// Test errors
///
/// # Note
///
/// `Error::Skipped` should not be treated as a test failure.
#[derive(Debug, Error)]
#[non_exhaustive]
pub enum Error {
    /// The test was skipped
    #[error("test was skipped")]
    Skipped,
    /// Block processing failed
    /// Note: This includes but is not limited to execution.
    /// For example, the header number could be incorrect.
    #[error("block {block_number} failed to process: {err}")]
    BlockProcessingFailed {
        /// The block number for the block that failed
        block_number: u64,
        /// Contains the inputs necessary for the block stateless validation guest program used in
        /// zkVMs to prove the block is invalid.
        partial_program_inputs: Vec<(RecoveredBlock<Block>, ExecutionWitness)>,
        /// The specific error
        #[source]
        err: Box<dyn std::error::Error + Send + Sync>,
    },
    /// An IO error occurred
    #[error("an error occurred interacting with the file system at {path}: {error}")]
    Io {
        /// The path to the file or directory
        path: PathBuf,
        /// The specific error
        #[source]
        error: std::io::Error,
    },
    /// A deserialization error occurred
    #[error("an error occurred deserializing the test at {path}: {error}")]
    CouldNotDeserialize {
        /// The path to the file we wanted to deserialize
        path: PathBuf,
        /// The specific error
        #[source]
        error: serde_json::Error,
    },
    /// A database error occurred.
    #[error(transparent)]
    Database(#[from] DatabaseError),
    /// A test assertion failed.
    #[error("test failed: {0}")]
    Assertion(String),
    /// An error internally in reth occurred.
    #[error("test failed: {0}")]
    Provider(#[from] ProviderError),
    /// An error occurred while decoding RLP.
    #[error("an error occurred deserializing RLP: {0}")]
    RlpDecodeError(#[from] alloy_rlp::Error),
    /// A consensus error occurred.
    #[error("an error occurred during consensus checks: {0}")]
    ConsensusError(#[from] reth_consensus::ConsensusError),
}

impl Error {
    /// Create a new [`Error::BlockProcessingFailed`] error.
    pub fn block_failed(
        block_number: u64,
        partial_program_inputs: Vec<(RecoveredBlock<Block>, ExecutionWitness)>,
        err: impl std::error::Error + Send + Sync + 'static,
    ) -> Self {
        Self::BlockProcessingFailed { block_number, partial_program_inputs, err: Box::new(err) }
    }
}

/// The result of running a test.
#[derive(Debug)]
pub struct CaseResult {
    /// A description of the test.
    pub desc: String,
    /// The full path to the test.
    pub path: PathBuf,
    /// The result of the test.
    pub result: Result<(), Error>,
}

impl CaseResult {
    /// Create a new test result.
    pub fn new(path: &Path, case: &impl Case, result: Result<(), Error>) -> Self {
        Self { desc: case.description(), path: path.into(), result }
    }
}

/// Assert that all the given tests passed and print the results to stdout.
pub(crate) fn assert_tests_pass(suite_name: &str, path: &Path, results: &[CaseResult]) {
    let (passed, failed, skipped) = categorize_results(results);

    print_results(suite_name, path, &passed, &failed, &skipped);

    assert!(failed.is_empty(), "Some tests failed (see above)");
}

/// Categorize test results into `(passed, failed, skipped)`.
pub(crate) fn categorize_results(
    results: &[CaseResult],
) -> (Vec<&CaseResult>, Vec<&CaseResult>, Vec<&CaseResult>) {
    let mut passed = Vec::new();
    let mut failed = Vec::new();
    let mut skipped = Vec::new();

    for case in results {
        match case.result.as_ref().err() {
            Some(Error::Skipped) => skipped.push(case),
            Some(_) => failed.push(case),
            None => passed.push(case),
        }
    }

    (passed, failed, skipped)
}

/// Display the given test results to stdout.
pub(crate) fn print_results(
    suite_name: &str,
    path: &Path,
    passed: &[&CaseResult],
    failed: &[&CaseResult],
    skipped: &[&CaseResult],
) {
    println!("Suite: {suite_name} (at {})", path.display());
    println!(
        "Ran {} tests ({} passed, {} failed, {} skipped)",
        passed.len() + failed.len() + skipped.len(),
        passed.len(),
        failed.len(),
        skipped.len()
    );

    for case in skipped {
        println!("[S] Case {} skipped", case.path.display());
    }

    for case in failed {
        let error = case.result.as_ref().unwrap_err();
        println!("[!] Case {} failed (description: {}): {}", case.path.display(), case.desc, error);
    }
}