diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 234ef2e56d..0b0f2fb846 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,8 +55,9 @@ jobs: token: ${{ secrets.CODECOV_TOKEN }} files: lcov.info flags: unit-tests + eth-blockchain: - name: Ethereum blockchain Tests (Stable) + name: ethereum blockchain tests (stable) runs-on: ubuntu-latest env: RUST_LOG: info,sync=error diff --git a/bin/reth/src/cli.rs b/bin/reth/src/cli.rs index 1cd42b80ff..936d46e02b 100644 --- a/bin/reth/src/cli.rs +++ b/bin/reth/src/cli.rs @@ -34,9 +34,6 @@ pub enum Commands { /// Start the node #[command(name = "node")] Node(node::Command), - /// Run Ethereum blockchain tests - #[command(name = "test-chain")] - TestEthChain(test_eth_chain::Command), /// Database debugging utilities #[command(name = "db")] Db(db::Command), @@ -51,6 +48,9 @@ pub enum Commands { /// P2P Debugging utilities #[command(name = "p2p")] P2P(p2p::Command), + /// Run Ethereum blockchain tests + #[command(name = "test-chain")] + TestEthChain(test_eth_chain::Command), } #[derive(Parser)] diff --git a/bin/reth/src/test_eth_chain/mod.rs b/bin/reth/src/test_eth_chain/mod.rs index 8f46995907..356477ba98 100644 --- a/bin/reth/src/test_eth_chain/mod.rs +++ b/bin/reth/src/test_eth_chain/mod.rs @@ -2,13 +2,16 @@ use clap::Parser; use eyre::eyre; +use futures::{stream::FuturesUnordered, StreamExt}; use std::path::PathBuf; -use tracing::{error, info}; +use tracing::{error, info, warn}; /// Models for parsing JSON blockchain tests pub mod models; /// Ethereum blockhain test runner pub mod runner; +use runner::TestOutcome; + /// Execute Ethereum blockchain tests by specifying path to json files #[derive(Debug, Parser)] pub struct Command { @@ -19,34 +22,37 @@ pub struct Command { impl Command { /// Execute the command pub async fn execute(self) -> eyre::Result<()> { - // note the use of `into_iter()` to consume `items` - let futs: Vec<_> = self + let mut futs: FuturesUnordered<_> = self .path .iter() .flat_map(|item| reth_cli_utils::find_all_files_with_postfix(item, ".json")) .map(|file| async { (runner::run_test(file.clone()).await, file) }) .collect(); - let results = futures::future::join_all(futs).await; - // await the tasks for resolve's to complete and give back our test results - let mut num_of_failed = 0; - let mut num_of_passed = 0; - for (result, file) in results { - match result { - Ok(_) => { - num_of_passed += 1; + let mut failed = 0; + let mut passed = 0; + let mut skipped = 0; + while let Some((result, file)) = futs.next().await { + match TestOutcome::from(result) { + TestOutcome::Passed => { + info!(target: "reth::cli", "[+] Test {file:?} passed."); + passed += 1; } - Err(error) => { - num_of_failed += 1; + TestOutcome::Skipped => { + warn!(target: "reth::cli", "[=] Test {file:?} skipped."); + skipped += 1; + } + TestOutcome::Failed(error) => { error!(target: "reth::cli", "Test {file:?} failed:\n{error}"); + failed += 1; } } } - info!(target: "reth::cli", "{num_of_passed}/{} tests passed\n", num_of_passed + num_of_failed); + info!(target: "reth::cli", "{passed}/{0} tests passed, {skipped}/{0} skipped, {failed}/{0} failed.\n", failed + passed + skipped); - if num_of_failed != 0 { - Err(eyre!("Failed {num_of_failed} tests")) + if failed != 0 { + Err(eyre!("Failed {failed} tests")) } else { Ok(()) } diff --git a/bin/reth/src/test_eth_chain/models.rs b/bin/reth/src/test_eth_chain/models.rs index ccc20aaa7c..a18b1edd0f 100644 --- a/bin/reth/src/test_eth_chain/models.rs +++ b/bin/reth/src/test_eth_chain/models.rs @@ -5,35 +5,35 @@ use reth_primitives::{ use serde::{self, Deserialize}; use std::collections::BTreeMap; -/// Blockchain test deserializer. +/// An Ethereum blockchain test. #[derive(Debug, PartialEq, Eq, Deserialize)] pub struct Test(pub BTreeMap); -/// Ethereum blockchain test data +/// Ethereum test data. #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BlockchainTestData { /// Genesis block header. pub genesis_block_header: Header, - /// Genesis rlp. + /// RLP encoded genesis block. #[serde(rename = "genesisRLP")] pub genesis_rlp: Option, - /// Blocks. + /// Block data. pub blocks: Vec, - /// Post state. + /// The expected post state. pub post_state: Option, - /// Pre state. + /// The test pre-state. pub pre: State, - /// Hash of best block. + /// Hash of the best block. pub lastblockhash: H256, - /// Network. + /// Network spec. pub network: ForkSpec, #[serde(default)] - /// Engine + /// Engine spec. pub self_engine: SealEngine, } -/// Ethereum blockchain test data Header. +/// A block header in an Ethereum blockchain test. #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Header { @@ -99,14 +99,14 @@ impl From
for SealedHeader { } } -/// Ethereum blockchain test data Block. +/// A block in an Ethereum blockchain test. #[derive(Debug, PartialEq, Eq, Deserialize)] #[serde(deny_unknown_fields)] #[serde(rename_all = "camelCase")] pub struct Block { /// Block header. pub block_header: Option
, - /// Rlp block bytes + /// RLP encoded block bytes pub rlp: Bytes, /// Transactions pub transactions: Option>, @@ -126,11 +126,7 @@ pub struct TransactionSequence { valid: String, } -/// Ethereum blockchain test data State. -//#[derive(Clone, Debug, Eq, PartialEq, Deserialize)] -//#[serde(deny_unknown_fields)] -//pub struct State(pub RootOrState); - +/// Ethereum blockchain test data state. #[derive(Clone, Debug, Eq, PartialEq, Deserialize)] pub struct State(pub BTreeMap); @@ -144,7 +140,7 @@ pub enum RootOrState { State(BTreeMap), } -/// Spec account +/// An account. #[derive(Debug, PartialEq, Eq, Deserialize, Clone)] #[serde(deny_unknown_fields)] pub struct Account { @@ -158,7 +154,7 @@ pub struct Account { pub storage: BTreeMap, } -/// Fork Spec +/// Fork specification. #[derive(Debug, PartialEq, Eq, PartialOrd, Hash, Ord, Deserialize)] pub enum ForkSpec { /// Frontier @@ -248,7 +244,7 @@ impl From for reth_executor::SpecUpgrades { } } -/// Json Block test possible engine kind. +/// Possible seal engines. #[derive(Debug, PartialEq, Eq, Default, Deserialize)] #[serde(rename_all = "camelCase")] pub enum SealEngine { diff --git a/bin/reth/src/test_eth_chain/runner.rs b/bin/reth/src/test_eth_chain/runner.rs index 17f9baf554..4ae75a749d 100644 --- a/bin/reth/src/test_eth_chain/runner.rs +++ b/bin/reth/src/test_eth_chain/runner.rs @@ -21,7 +21,27 @@ use std::{ ffi::OsStr, path::{Path, PathBuf}, }; -use tracing::{debug, info, trace}; +use tracing::{debug, trace}; + +/// The outcome of a test. +#[derive(Debug)] +pub enum TestOutcome { + /// The test was skipped. + Skipped, + /// The test passed. + Passed, + /// The test failed. + Failed(eyre::Report), +} + +impl From> for TestOutcome { + fn from(v: eyre::Result) -> TestOutcome { + match v { + Ok(outcome) => outcome, + Err(err) => TestOutcome::Failed(err), + } + } +} /// Tests are test edge cases that are not possible to happen on mainnet, so we are skipping them. pub fn should_skip(path: &Path) -> bool { @@ -72,15 +92,16 @@ pub fn should_skip(path: &Path) -> bool { } /// Run one JSON-encoded Ethereum blockchain test at the specified path. -pub async fn run_test(path: PathBuf) -> eyre::Result<()> { +pub async fn run_test(path: PathBuf) -> eyre::Result { let path = path.as_path(); let json_file = std::fs::read(path)?; let suites: Test = serde_json::from_reader(&*json_file)?; if should_skip(path) { - return Ok(()) + return Ok(TestOutcome::Skipped) } - info!(target: "reth::cli", ?path, "Running test suite"); + + debug!(target: "reth::cli", ?path, "Running test suite"); for (name, suite) in suites.0 { if matches!( @@ -189,7 +210,7 @@ pub async fn run_test(path: PathBuf) -> eyre::Result<()> { // Validate post state match suite.post_state { Some(RootOrState::Root(root)) => { - info!(target: "reth::cli", "Post-state root: #{root:?}") + debug!(target: "reth::cli", "Post-state root: #{root:?}") } Some(RootOrState::State(state)) => db.view(|tx| -> eyre::Result<()> { let mut cursor = tx.cursor_dup_read::()?; @@ -274,8 +295,8 @@ pub async fn run_test(path: PathBuf) -> eyre::Result<()> { } Ok(()) })??, - None => info!(target: "reth::cli", "No post-state"), + None => debug!(target: "reth::cli", "No post-state"), } } - Ok(()) + Ok(TestOutcome::Passed) }