feat(ci): reorganize e2e tests with dedicated nextest filter and CI workflow (#17290)

.config/nextest.toml

@@ -5,3 +5,9 @@ slow-timeout = { period = "30s", terminate-after = 4 }
 [[profile.default.overrides]]
 filter = "test(general_state_tests)"
 slow-timeout = { period = "1m", terminate-after = 10 }
+
+# E2E tests using the testsuite framework from crates/e2e-test-utils
+# These tests are located in tests/e2e-testsuite/ directories across various crates
+[[profile.default.overrides]]
+filter = "binary(e2e_testsuite)"
+slow-timeout = { period = "2m", terminate-after = 3 }

.github/workflows/e2e.yml

@@ -0,0 +1,46 @@
+# Runs e2e tests using the testsuite framework
+
+name: e2e
+
+on:
+  pull_request:
+  merge_group:
+  push:
+    branches: [main]
+
+env:
+  CARGO_TERM_COLOR: always
+  SEED: rustethereumethereumrust
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    name: e2e-testsuite
+    runs-on:
+      group: Reth
+    env:
+      RUST_BACKTRACE: 1
+    timeout-minutes: 90
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: taiki-e/install-action@nextest
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+      - name: Run e2e tests
+        run: |
+          cargo nextest run \
+            --locked --features "asm-keccak" \
+            --workspace \
+            --exclude 'example-*' \
+            --exclude 'exex-subscription' \
+            --exclude 'reth-bench' \
+            --exclude 'ef-tests' \
+            --exclude 'op-reth' \
+            --exclude 'reth' \
+            -E 'binary(e2e_testsuite)'
+

.github/workflows/integration.yml

@@ -47,7 +47,7 @@ jobs:
           cargo nextest run \
             --locked --features "asm-keccak ${{ matrix.network }}" \
             --workspace --exclude ef-tests \
-            -E "kind(test)"
+            -E "kind(test) and not binary(e2e_testsuite)"
       - if: matrix.network == 'optimism'
         name: Run tests
         run: |

.github/workflows/unit.yml

@@ -61,7 +61,7 @@ jobs:
             ${{ matrix.args }} --workspace \
             --exclude ef-tests --no-tests=warn \
             --partition hash:${{ matrix.partition }}/2 \
-            -E "!kind(test)"
+            -E "!kind(test) and not binary(e2e_testsuite)"
 
   state:
     name: Ethereum state tests

crates/e2e-test-utils/Cargo.toml

@@ -72,3 +72,7 @@ tokio-stream.workspace = true
 serde_json.workspace = true
 tracing.workspace = true
 derive_more.workspace = true
+
+[[test]]
+name = "e2e_testsuite"
+path = "tests/e2e-testsuite/main.rs"

crates/e2e-test-utils/src/testsuite/README.md

@@ -0,0 +1,106 @@
+# E2E Test Suite Framework
+
+This directory contains the framework for writing end-to-end (e2e) tests in Reth. The framework provides utilities for setting up test environments, performing actions, and verifying blockchain behavior.
+
+## Test Organization
+
+E2E tests using this framework follow a consistent structure across the codebase:
+
+### Directory Structure
+Each crate that requires e2e tests should organize them as follows:
+```
+<crate-name>/
+├── src/
+│   └── ... (implementation code)
+├── tests/
+│   └── e2e-testsuite/
+│       └── main.rs (or other test files)
+└── Cargo.toml
+```
+
+### Cargo.toml Configuration
+In your crate's `Cargo.toml`, define the e2e test binary:
+```toml
+[[test]]
+name = "e2e_testsuite"
+path = "tests/e2e-testsuite/main.rs"
+harness = true
+```
+
+**Important**: The test binary MUST be named `e2e_testsuite` to be properly recognized by the nextest filter and CI workflows.
+
+## Running E2E Tests
+
+### Run all e2e tests across the workspace
+```bash
+cargo nextest run --workspace \
+  --exclude 'example-*' \
+  --exclude 'exex-subscription' \
+  --exclude 'reth-bench' \
+  --exclude 'ef-tests' \
+  --exclude 'op-reth' \
+  --exclude 'reth' \
+  -E 'binary(e2e_testsuite)'
+```
+
+Note: The `--exclude` flags prevent compilation of crates that don't contain e2e tests (examples, benchmarks, binaries, and EF tests), significantly reducing build time.
+
+### Run e2e tests for a specific crate
+```bash
+cargo nextest run -p <crate-name> -E 'binary(e2e_testsuite)'
+```
+
+### Run with additional features
+```bash
+cargo nextest run --locked --features "asm-keccak" --workspace -E 'binary(e2e_testsuite)'
+```
+
+### Run a specific test
+```bash
+cargo nextest run --workspace -E 'binary(e2e_testsuite) and test(test_name)'
+```
+
+## Writing E2E Tests
+
+Tests use the framework components from this directory:
+
+```rust
+use reth_e2e_test_utils::{setup_import, Environment, TestBuilder};
+
+#[tokio::test]
+async fn test_example() -> eyre::Result<()> {
+    // Create test environment
+    let (mut env, mut handle) = TestBuilder::new()
+        .build()
+        .await?;
+
+    // Perform test actions...
+    
+    Ok(())
+}
+```
+
+## Framework Components
+
+- **Environment**: Core test environment managing nodes and network state
+- **TestBuilder**: Builder pattern for configuring test environments
+- **Actions** (`actions/`): Pre-built test actions like block production, reorgs, etc.
+- **Setup utilities**: Helper functions for common test scenarios
+
+## CI Integration
+
+E2E tests run in a dedicated GitHub Actions workflow (`.github/workflows/e2e.yml`) with:
+- Extended timeouts (2 minutes per test, with 3 retries)
+- Isolation from unit and integration tests
+- Parallel execution support
+
+## Nextest Configuration
+
+The framework uses custom nextest settings (`.config/nextest.toml`):
+```toml
+[[profile.default.overrides]]
+filter = "binary(e2e_testsuite)"
+slow-timeout = { period = "2m", terminate-after = 3 }
+```
+
+This ensures all e2e tests get appropriate timeouts for complex blockchain operations.

crates/e2e-test-utils/src/testsuite/mod.rs

@@ -20,9 +20,6 @@ use reth_rpc_builder::auth::AuthServerHandle;
 use std::sync::Arc;
 use url::Url;
 
-#[cfg(test)]
-mod examples;
-
 /// Client handles for both regular RPC and Engine API endpoints
 #[derive(Clone)]
 pub struct NodeClient {

crates/e2e-test-utils/tests/e2e-testsuite/main.rs

@@ -1,35 +1,41 @@
 //! Example tests using the test suite framework.
 
-use crate::testsuite::{
-    actions::{
-        Action, AssertChainTip, AssertMineBlock, CaptureBlock, CaptureBlockOnNode,
-        CompareNodeChainTips, CreateFork, MakeCanonical, ProduceBlocks, ReorgTo, SelectActiveNode,
-        UpdateBlockInfo,
-    },
-    setup::{NetworkSetup, Setup},
-    TestBuilder,
-};
 use alloy_primitives::{Address, B256};
 use alloy_rpc_types_engine::PayloadAttributes;
 use eyre::Result;
 use reth_chainspec::{ChainSpecBuilder, MAINNET};
+use reth_e2e_test_utils::{
+    test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp},
+    testsuite::{
+        actions::{
+            Action, AssertChainTip, AssertMineBlock, CaptureBlock, CaptureBlockOnNode,
+            CompareNodeChainTips, CreateFork, MakeCanonical, ProduceBlocks, ReorgTo,
+            SelectActiveNode, UpdateBlockInfo,
+        },
+        setup::{NetworkSetup, Setup},
+        Environment, TestBuilder,
+    },
+};
 use reth_node_api::TreeConfig;
 use reth_node_ethereum::{EthEngineTypes, EthereumNode};
 use std::sync::Arc;
+use tempfile::TempDir;
 use tracing::debug;
 
 #[tokio::test]
 async fn test_apply_with_import() -> Result<()> {
-    use crate::test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp};
-    use tempfile::TempDir;
-
     reth_tracing::init_test_tracing();
 
     // Create test chain spec
     let chain_spec = Arc::new(
         ChainSpecBuilder::default()
             .chain(MAINNET.chain)
-            .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+            .genesis(
+                serde_json::from_str(include_str!(
+                    "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                ))
+                .unwrap(),
+            )
             .london_activated()
             .shanghai_activated()
             .cancun_activated()
@@ -49,7 +55,7 @@ async fn test_apply_with_import() -> Result<()> {
         Setup::default().with_chain_spec(chain_spec).with_network(NetworkSetup::single_node());
 
     // Create environment and apply setup with import
-    let mut env = crate::testsuite::Environment::<EthEngineTypes>::default();
+    let mut env = Environment::<EthEngineTypes>::default();
     setup.apply_with_import::<EthereumNode>(&mut env, &rlp_path).await?;
 
     // Now run test actions on the environment with imported chain
@@ -126,7 +132,12 @@ async fn test_testsuite_assert_mine_block() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .paris_activated()
                 .build(),
         ))
@@ -163,7 +174,12 @@ async fn test_testsuite_produce_blocks() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .cancun_activated()
                 .build(),
         ))
@@ -187,7 +203,12 @@ async fn test_testsuite_create_fork() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .cancun_activated()
                 .build(),
         ))
@@ -212,7 +233,12 @@ async fn test_testsuite_reorg_with_tagging() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .cancun_activated()
                 .build(),
         ))
@@ -239,7 +265,12 @@ async fn test_testsuite_deep_reorg() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .cancun_activated()
                 .build(),
         ))
@@ -284,7 +315,12 @@ async fn test_testsuite_multinode_block_production() -> Result<()> {
         .with_chain_spec(Arc::new(
             ChainSpecBuilder::default()
                 .chain(MAINNET.chain)
-                .genesis(serde_json::from_str(include_str!("assets/genesis.json")).unwrap())
+                .genesis(
+                    serde_json::from_str(include_str!(
+                        "../../../../crates/e2e-test-utils/src/testsuite/assets/genesis.json"
+                    ))
+                    .unwrap(),
+                )
                 .cancun_activated()
                 .build(),
         ))

crates/engine/tree/Cargo.toml

@@ -139,3 +139,7 @@ test-utils = [
     "reth-node-ethereum/test-utils",
     "reth-evm-ethereum/test-utils",
 ]
+
+[[test]]
+name = "e2e_testsuite"
+path = "tests/e2e-testsuite/main.rs"

crates/engine/tree/src/tree/mod.rs

@@ -66,8 +66,6 @@ use tracing::*;
 
 mod block_buffer;
 mod cached_state;
-#[cfg(test)]
-mod e2e_tests;
 pub mod error;
 mod instrumented_state;
 mod invalid_block_hook;

crates/engine/tree/tests/e2e-testsuite/main.rs

@@ -1,6 +1,5 @@
 //! E2E test implementations using the e2e test framework for engine tree functionality.
 
-use crate::tree::TreeConfig;
 use eyre::Result;
 use reth_chainspec::{ChainSpecBuilder, MAINNET};
 use reth_e2e_test_utils::testsuite::{
@@ -12,6 +11,7 @@ use reth_e2e_test_utils::testsuite::{
     setup::{NetworkSetup, Setup},
     TestBuilder,
 };
+use reth_engine_tree::tree::TreeConfig;
 use reth_ethereum_engine_primitives::EthEngineTypes;
 use reth_node_ethereum::EthereumNode;
 use std::sync::Arc;

crates/optimism/node/Cargo.toml

@@ -119,3 +119,7 @@ test-utils = [
     "reth-trie-common/test-utils",
 ]
 reth-codec = ["reth-optimism-primitives/reth-codec"]
+
+[[test]]
+name = "e2e_testsuite"
+path = "tests/e2e-testsuite/main.rs"

crates/rpc/rpc-e2e-tests/Cargo.toml

@@ -37,3 +37,7 @@ reth-tracing.workspace = true
 reth-chainspec.workspace = true
 reth-node-ethereum.workspace = true
 alloy-genesis.workspace = true
+
+[[test]]
+name = "e2e_testsuite"
+path = "tests/e2e-testsuite/main.rs"