test: update to vitest 4 to use builtin bun support (#8599)

**Motivation**

Update the vitest to avoid using third party test pool. 

**Description**

- Use latest vitest
- Remove custom process pool which we developed to run our tests in Bun
runtime
- Migrate test configs to latest version
- Update types
- Switch to playwright from webdriverio for browser tests performance,
which was due for long.


**Steps to test or reproduce**

- Run all tests

.github/workflows/test.yml

@@ -175,10 +175,8 @@ jobs:
       - uses: "./.github/actions/setup-and-build"
         with:
           node: ${{ matrix.node }}
-      - name: Install Chrome browser
-        run: npx @puppeteer/browsers install chromedriver@stable --path /tmp
-      - name: Install Firefox browser
-        run: npx @puppeteer/browsers install firefox@stable --path /tmp
+      - name: Install
+        run: yarn playwright install
       - name: Browser tests
         run: |
           export DISPLAY=':99.0'

configs/vite.config.base.ts

@@ -1,5 +1,5 @@
 import {visualizer} from "rollup-plugin-visualizer";
-import {UserConfig, defineConfig} from "vite";
+import {defineConfig} from "vite";
 import {nodePolyfills} from "vite-plugin-node-polyfills";
 import topLevelAwait from "vite-plugin-top-level-await";
 import {blsBrowserPlugin} from "../scripts/vite/plugins/blsBrowserPlugin.js";

configs/vitest.config.browser.ts

@@ -1,6 +1,5 @@
-/// <reference types="@vitest/browser/providers/webdriverio" />
-
 import path from "node:path";
+import {playwright} from "@vitest/browser-playwright";
 import {nodePolyfills} from "vite-plugin-node-polyfills";
 import {defineProject} from "vitest/config";
 import {blsBrowserPlugin} from "../scripts/vite/plugins/blsBrowserPlugin.js";
@@ -21,29 +20,18 @@ export const browserTestProject = defineProject({
       headless: true,
       ui: false,
       screenshotFailures: false,
-      // Recommended provider is `playwright` but it's causing following error on CI
-      // Error: Failed to connect to the browser session "af5be85a-7f29-4299-b680-b07f0cfc2520" within the timeout.
-      // TODO: Debug the issue in later versions of playwright and vitest
-      provider: "webdriverio",
+      provider: playwright(),
       connectTimeout: 90_0000,
       instances: [
         // TODO: Add support for webkit when available
-        // Invalid types from webdriverio for capabilities
         {
           browser: "firefox",
           maxConcurrency: 1,
-          capabilities: {
-            browserVersion: "stable",
-          },
-        } as never,
-        // Invalid types from webdriverio for capabilities
+        },
         {
-          browser: "chrome",
+          browser: "chromium",
           maxConcurrency: 1,
-          capabilities: {
-            browserVersion: "stable",
-          },
-        } as never,
+        },
       ],
     },
   },

configs/vitest.config.e2e.ts

@@ -17,11 +17,8 @@ export const e2eMinimalProject = defineProject({
       LODESTAR_PRESET: "minimal",
     },
     pool: "forks",
-    poolOptions: {
-      forks: {
-        singleFork: true,
-      },
-    },
+    maxWorkers: 1,
+    isolate: true,
     sequence: {
       concurrent: false,
       shuffle: false,
@@ -43,11 +40,8 @@ export const e2eMainnetProject = defineProject({
       LODESTAR_PRESET: "mainnet",
     },
     pool: "forks",
-    poolOptions: {
-      forks: {
-        singleFork: true,
-      },
-    },
+    maxWorkers: 1,
+    isolate: true,
     sequence: {
       concurrent: false,
       shuffle: false,

configs/vitest.config.unit.ts

@@ -7,19 +7,12 @@ const setupFiles = [
   path.join(import.meta.dirname, "../scripts/vitest/setupFiles/lodestarPreset.ts"),
 ];
 
-const isBun = "bun" in process.versions;
-
 export const unitTestMinimalProject = defineProject({
   test: {
     name: "unit-minimal",
     include: ["**/test/unit-minimal/**/*.test.ts"],
     setupFiles,
-    // Current vitest `forks` pool is not yet supported with Bun
-    // so we conditionally switch to our custom pool which run tests in main process
-    pool: isBun ? "vitest-in-process-pool" : "forks",
-    poolOptions: {
-      forks: isBun ? {singleFork: true} : {},
-    },
+    pool: "forks",
     env: {
       LODESTAR_PRESET: "minimal",
     },
@@ -39,12 +32,7 @@ export const unitTestMainnetProject = defineProject({
     // for now I tried to identify such tests an increase the limit a bit higher
     testTimeout: 20_000,
     hookTimeout: 20_000,
-    // Current vitest `forks` pool is not yet supported with Bun
-    // so we conditionally switch to our custom pool which run tests in main process
-    pool: isBun ? "vitest-in-process-pool" : "forks",
-    poolOptions: {
-      forks: isBun ? {singleFork: true} : {},
-    },
+    pool: "forks",
     env: {
       LODESTAR_PRESET: "mainnet",
     },

package.json

@@ -47,8 +47,9 @@
     "@chainsafe/biomejs-config": "^1.0.0",
     "@types/node": "^22.18.6",
     "@types/react": "^19.1.12",
-    "@vitest/browser": "3.0.9",
-    "@vitest/coverage-v8": "3.0.9",
+    "@vitest/browser": "^4.0.7",
+    "@vitest/browser-playwright": "^4.0.7",
+    "@vitest/coverage-v8": "^4.0.7",
     "bun-types": "^1.2.21",
     "crypto-browserify": "^3.12.0",
     "dotenv": "^16.4.5",
@@ -60,6 +61,7 @@
     "node-gyp": "^9.4.0",
     "npm-run-all": "^4.1.5",
     "path-browserify": "^1.0.1",
+    "playwright": "^1.56.1",
     "prettier": "^3.2.5",
     "process": "^0.11.10",
     "rollup-plugin-visualizer": "^5.12.0",
@@ -70,20 +72,17 @@
     "typescript": "^5.9.2",
     "typescript-docs-verifier": "^3.0.1",
     "vite": "^6.0.11",
-    "vite-plugin-dts": "^4.5.0",
-    "vite-plugin-node-polyfills": "^0.23.0",
-    "vite-plugin-top-level-await": "^1.5.0",
-    "vitest": "3.0.9",
-    "vitest-in-process-pool": "2.0.1",
-    "vitest-when": "^0.6.1",
-    "wait-port": "^1.1.0",
-    "webdriverio": "^9.7.2"
+    "vite-plugin-dts": "^4.5.4",
+    "vite-plugin-node-polyfills": "^0.24.0",
+    "vite-plugin-top-level-await": "^1.6.0",
+    "vitest": "^4.0.7",
+    "vitest-when": "^0.9.0",
+    "wait-port": "^1.1.0"
   },
   "resolutions": {
     "dns-over-http-resolver": "^2.1.1",
     "loupe": "^2.3.6",
     "testcontainers/**/nan": "^2.19.0",
-    "vitest": "3.0.9",
     "elliptic": ">=6.6.1"
   }
 }

packages/beacon-node/test/mocks/beaconSyncMock.ts

@@ -6,7 +6,7 @@ export type MockedBeaconSync = Mocked<BeaconSync>;
 vi.mock("../../src/sync/index.js", async (importActual) => {
   const mod = await importActual<typeof import("../../src/sync/index.js")>();
 
-  const BeaconSync = vi.fn().mockImplementation(() => {
+  const BeaconSync = vi.fn().mockImplementation(function MockedBeaconSync() {
     const sync = {
       isSynced: vi.fn(),
     };

packages/beacon-node/test/mocks/mockedBeaconChain.ts

@@ -3,6 +3,7 @@ import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
 import {ChainForkConfig} from "@lodestar/config";
 import {config as defaultConfig} from "@lodestar/config/default";
 import {EpochDifference, ForkChoice, ProtoBlock} from "@lodestar/fork-choice";
+import {CachedBeaconStateAllForks} from "@lodestar/state-transition";
 import {Logger} from "@lodestar/utils";
 import {BeaconProposerCache} from "../../src/chain/beaconProposerCache.js";
 import {BeaconChain} from "../../src/chain/chain.js";
@@ -20,7 +21,7 @@ import {getMockedLogger} from "./loggerMock.js";
 
 export type MockedBeaconChain = Mocked<BeaconChain> & {
   logger: Mocked<Logger>;
-  getHeadState: Mock;
+  getHeadState: Mocked<CachedBeaconStateAllForks>;
   forkChoice: MockedForkChoice;
   executionEngine: Mocked<ExecutionEngineHttp>;
   executionBuilder: Mocked<ExecutionBuilderHttp>;
@@ -43,7 +44,7 @@ export type MockedBeaconChain = Mocked<BeaconChain> & {
 vi.mock("@lodestar/fork-choice", async (importActual) => {
   const mod = await importActual<typeof import("@lodestar/fork-choice")>();
 
-  const ForkChoice = vi.fn().mockImplementation(() => {
+  const ForkChoice = vi.fn().mockImplementation(function MockedForkChoice() {
     return {
       updateTime: vi.fn(),
       getJustifiedBlock: vi.fn(),
@@ -80,7 +81,7 @@ vi.mock("../../src/chain/lightClient/index.js");
 vi.mock("../../src/chain/opPools/index.js", async (importActual) => {
   const mod = await importActual<typeof import("../../src/chain/opPools/index.js")>();
 
-  const OpPool = vi.fn().mockImplementation(() => {
+  const OpPool = vi.fn().mockImplementation(function MockedOpPool() {
     return {
       hasSeenBlsToExecutionChange: vi.fn(),
       hasSeenVoluntaryExit: vi.fn(),
@@ -90,13 +91,13 @@ vi.mock("../../src/chain/opPools/index.js", async (importActual) => {
     };
   });
 
-  const AggregatedAttestationPool = vi.fn().mockImplementation(() => {
+  const AggregatedAttestationPool = vi.fn().mockImplementation(function MockedAggregatedAttestationPool() {
     return {
       getAttestationsForBlock: vi.fn(),
     };
   });
 
-  const SyncContributionAndProofPool = vi.fn().mockImplementation(() => {
+  const SyncContributionAndProofPool = vi.fn().mockImplementation(function MockedSyncContributionAndProofPool() {
     return {
       getAggregate: vi.fn(),
     };
@@ -113,63 +114,63 @@ vi.mock("../../src/chain/opPools/index.js", async (importActual) => {
 vi.mock("../../src/chain/chain.js", async (importActual) => {
   const mod = await importActual<typeof import("../../src/chain/chain.js")>();
 
-  const BeaconChain = vi
-    .fn()
-    .mockImplementation(({clock: clockParam, genesisTime, config}: MockedBeaconChainOptions) => {
-      const logger = getMockedLogger();
-      const clock =
-        clockParam === "real"
-          ? new Clock({config, genesisTime, signal: new AbortController().signal})
-          : getMockedClock();
+  const BeaconChain = vi.fn().mockImplementation(function MockedBeaconChain({
+    clock: clockParam,
+    genesisTime,
+    config,
+  }: MockedBeaconChainOptions) {
+    const logger = getMockedLogger();
+    const clock =
+      clockParam === "real" ? new Clock({config, genesisTime, signal: new AbortController().signal}) : getMockedClock();
 
-      return {
-        config,
-        opts: {},
-        genesisTime,
-        clock,
-        forkChoice: getMockedForkChoice(),
-        executionEngine: {
-          notifyForkchoiceUpdate: vi.fn(),
-          getPayload: vi.fn(),
-          getClientVersion: vi.fn(),
-        },
-        executionBuilder: {},
-        // @ts-expect-error
-        eth1: new Eth1ForBlockProduction(),
-        opPool: new OpPool(),
-        aggregatedAttestationPool: new AggregatedAttestationPool(config),
-        syncContributionAndProofPool: new SyncContributionAndProofPool(config, clock),
-        // @ts-expect-error
-        beaconProposerCache: new BeaconProposerCache(),
-        shufflingCache: new ShufflingCache(),
-        pubkey2index: new PubkeyIndexMap(),
-        index2pubkey: [],
-        produceCommonBlockBody: vi.fn(),
-        getProposerHead: vi.fn(),
-        produceBlock: vi.fn(),
-        produceBlindedBlock: vi.fn(),
-        getCanonicalBlockAtSlot: vi.fn(),
-        recomputeForkChoiceHead: vi.fn(),
-        predictProposerHead: vi.fn(),
-        getHeadStateAtCurrentEpoch: vi.fn(),
-        getHeadState: vi.fn(),
-        getStateBySlot: vi.fn(),
-        updateBuilderStatus: vi.fn(),
-        processBlock: vi.fn(),
-        regenStateForAttestationVerification: vi.fn(),
-        close: vi.fn(),
-        logger,
-        regen: new QueuedStateRegenerator({} as any),
-        lightClientServer: new LightClientServer({} as any, {} as any),
-        bls: {
-          verifySignatureSets: vi.fn().mockResolvedValue(true),
-          verifySignatureSetsSameMessage: vi.fn().mockResolvedValue([true]),
-          close: vi.fn().mockResolvedValue(true),
-          canAcceptWork: vi.fn().mockReturnValue(true),
-        },
-        emitter: new ChainEventEmitter(),
-      };
-    });
+    return {
+      config,
+      opts: {},
+      genesisTime,
+      clock,
+      forkChoice: getMockedForkChoice(),
+      executionEngine: {
+        notifyForkchoiceUpdate: vi.fn(),
+        getPayload: vi.fn(),
+        getClientVersion: vi.fn(),
+      },
+      executionBuilder: {},
+      // @ts-expect-error
+      eth1: new Eth1ForBlockProduction(),
+      opPool: new OpPool(),
+      aggregatedAttestationPool: new AggregatedAttestationPool(config),
+      syncContributionAndProofPool: new SyncContributionAndProofPool(config, clock),
+      // @ts-expect-error
+      beaconProposerCache: new BeaconProposerCache(),
+      shufflingCache: new ShufflingCache(),
+      pubkey2index: new PubkeyIndexMap(),
+      index2pubkey: [],
+      produceCommonBlockBody: vi.fn(),
+      getProposerHead: vi.fn(),
+      produceBlock: vi.fn(),
+      produceBlindedBlock: vi.fn(),
+      getCanonicalBlockAtSlot: vi.fn(),
+      recomputeForkChoiceHead: vi.fn(),
+      predictProposerHead: vi.fn(),
+      getHeadStateAtCurrentEpoch: vi.fn(),
+      getHeadState: vi.fn(),
+      getStateBySlot: vi.fn(),
+      updateBuilderStatus: vi.fn(),
+      processBlock: vi.fn(),
+      regenStateForAttestationVerification: vi.fn(),
+      close: vi.fn(),
+      logger,
+      regen: new QueuedStateRegenerator({} as any),
+      lightClientServer: new LightClientServer({} as any, {} as any),
+      bls: {
+        verifySignatureSets: vi.fn().mockResolvedValue(true),
+        verifySignatureSetsSameMessage: vi.fn().mockResolvedValue([true]),
+        close: vi.fn().mockResolvedValue(true),
+        canAcceptWork: vi.fn().mockReturnValue(true),
+      },
+      emitter: new ChainEventEmitter(),
+    };
+  });
 
   return {
     ...mod,

packages/beacon-node/test/mocks/mockedBeaconDb.ts

@@ -45,7 +45,7 @@ vi.mock("../../src/db/repositories/index.js");
 vi.mock("../../src/db/index.js", async (importActual) => {
   const mod = await importActual<typeof import("../../src/db/index.js")>();
 
-  const mockedBeaconDb = vi.fn().mockImplementation(() => {
+  const mockedBeaconDb = vi.fn().mockImplementation(function MockedBeaconDb() {
     return {
       block: vi.mocked(new BlockRepository({} as any, {} as any)),
       blockArchive: vi.mocked(new BlockArchiveRepository({} as any, {} as any)),

packages/beacon-node/test/mocks/mockedNetwork.ts

@@ -4,7 +4,7 @@ import {INetwork, Network} from "../../src/network/index.js";
 vi.mock("../../src/network/index.js", async (importActual) => {
   const mod = await importActual<typeof import("../../src/network/index.js")>();
 
-  const Network = vi.fn().mockImplementation(() => {
+  const Network = vi.fn().mockImplementation(function MockedNetwork() {
     return {};
   });
 

packages/beacon-node/test/unit/api/impl/events/events.test.ts

@@ -12,13 +12,13 @@ vi.mock("../../../../../src/chain/index.js", async (importActual) => {
 
   return {
     ...mod,
-    BeaconChain: vi.spyOn(mod, "BeaconChain").mockImplementation(() => {
+    BeaconChain: vi.spyOn(mod, "BeaconChain").mockImplementation(function MockedBeaconChain() {
       return {
         emitter: new ChainEventEmitter(),
         forkChoice: {
           getHead: vi.fn(),
         },
-      } as unknown as BeaconChain;
+      };
     }),
   };
 });

packages/beacon-node/test/unit/chain/validation/voluntaryExit.test.ts

@@ -66,7 +66,7 @@ describe("validate voluntary exit", () => {
     vi.spyOn(chainStub, "getHeadStateAtCurrentEpoch").mockResolvedValue(state);
     vi.spyOn(opPool, "hasSeenBlsToExecutionChange");
     vi.spyOn(opPool, "hasSeenVoluntaryExit");
-    vi.spyOn(chainStub.bls, "verifySignatureSets").mockResolvedValue(true);
+    chainStub.bls.verifySignatureSets.mockResolvedValue(true);
   });
 
   afterEach(() => {
@@ -165,7 +165,7 @@ describe("validate voluntary exit", () => {
 
     opPool.hasSeenVoluntaryExit.mockReturnValue(false);
 
-    vi.spyOn(chainStub.bls, "verifySignatureSets").mockResolvedValue(false);
+    chainStub.bls.verifySignatureSets.mockResolvedValue(false);
 
     await expectRejectedWithLodestarError(
       validateGossipVoluntaryExit(chainStub, signedVoluntaryExitInvalidSig),

packages/beacon-node/test/unit/db/api/repository.test.ts

@@ -12,7 +12,7 @@ vi.mock("@lodestar/db/controller/level", async (importOriginal) => {
 
   return {
     ...mod,
-    LevelDbController: vi.spyOn(mod, "LevelDbController").mockImplementation(() => {
+    LevelDbController: vi.spyOn(mod, "LevelDbController").mockImplementation(function MockedLevelDbController() {
       return {
         get: vi.fn(),
         put: vi.fn(),
@@ -21,7 +21,7 @@ vi.mock("@lodestar/db/controller/level", async (importOriginal) => {
         valuesStream: vi.fn(),
         batchDelete: vi.fn(),
         batchPut: vi.fn(),
-      } as unknown as LevelDbController;
+      };
     }),
   };
 });

packages/beacon-node/test/unit/sync/unknownBlock.test.ts

@@ -27,6 +27,8 @@ import {getRandPeerIdStr, getRandPeerSyncMeta} from "../../utils/peer.js";
 
 describe.skip(
   "sync by UnknownBlockSync",
+  // spacer comment to avoid unnecessary changes in git
+  {timeout: 20_000},
   () => {
     const logger = testLogger();
     const slotSec = 0.3;
@@ -258,8 +260,7 @@ describe.skip(
         syncService.close();
       });
     }
-  },
-  {timeout: 20_000}
+  }
 );
 
 describe("UnknownBlockSync", () => {

packages/cli/test/utils/runUtils.ts

@@ -1,5 +1,5 @@
 import fs from "node:fs";
-import {expect} from "vitest";
+import {DeeplyAllowMatchers, expect} from "vitest";
 import {apiTokenFileName} from "../../src/cmds/validator/keymanager/server.js";
 import {recursiveLookup} from "../../src/util/index.js";
 
@@ -23,7 +23,11 @@ export function expectDeepEquals<T>(a: T, b: T, message: string): void {
 /**
  * Similar to `expectDeepEquals` but only checks presence of all elements in array, irrespective of their order.
  */
-export function expectDeepEqualsUnordered<T>(a: T[], b: T[], message: string): void {
+export function expectDeepEqualsUnordered<T>(
+  a: DeeplyAllowMatchers<T>[],
+  b: DeeplyAllowMatchers<T>[],
+  message: string
+): void {
   expect(a).toEqualWithMessage(expect.arrayContaining(b), message);
   expect(b).toEqualWithMessage(expect.arrayContaining(a), message);
   expect(a).toHaveLength(b.length);

packages/spec-test-util/package.json

@@ -57,9 +57,9 @@
     "@lodestar/utils": "^1.36.0",
     "rimraf": "^4.4.1",
     "snappyjs": "^0.7.0",
-    "vitest": "3.0.9"
+    "vitest": "^4.0.7"
   },
   "peerDependencies": {
-    "vitest": "3.0.9"
+    "vitest": "^4.0.7"
   }
 }

packages/test-utils/package.json

@@ -57,12 +57,12 @@
     "axios": "^1.3.4",
     "testcontainers": "^10.2.1",
     "tmp": "^0.2.1",
-    "vitest": "3.0.9"
+    "vitest": "^4.0.7"
   },
   "devDependencies": {
     "@types/yargs": "^17.0.24"
   },
   "peerDependencies": {
-    "vitest": "3.0.9"
+    "vitest": "^4.0.7"
   }
 }

scripts/vite/plugins/esmCjsInteropPlugin.ts

@@ -0,0 +1,32 @@
+import { createRequire } from "node:module";
+import {UserConfig, ConfigEnv, Plugin} from "vite";
+
+const require = createRequire(import.meta.url);
+const prettyFormatCjsPath = require.resolve("../../../node_modules/pretty-format/build/index.js");
+
+const isBun = "bun" in process.versions;
+
+// This plugin is developed to overcome an esm resolution issue in Bun runtime. 
+// TODO: Should remove this plugin when following issue is resolved
+// https://github.com/oven-sh/bun/issues/24341
+export function esmCjsInteropPlugin(): Plugin {
+  return {
+    name: "esmCjsInteropPlugin",
+    config(_config: UserConfig, _env: ConfigEnv) {
+      if(!isBun) return {};
+
+      return {
+        test: {
+          server: {
+            deps: {
+              inline: ["pretty-format", "vitest-when"],
+            },
+          }
+        },
+        resolve: {
+          alias: [{find: /^pretty-format$/, replacement: prettyFormatCjsPath}],
+        },
+      };
+    },
+};
+}

types/vitest/index.d.ts

@@ -41,6 +41,6 @@ interface CustomAsymmetricMatchers<R = unknown> extends CustomMatchers<R> {
 }
 
 declare module "vitest" {
-    interface Assertion<T = any> extends CustomMatchers<T> {}
+  interface Assertion<T = any> extends CustomMatchers<T> {}
   interface AsymmetricMatchersContaining extends CustomAsymmetricMatchers {}
 }

vitest.config.ts

@@ -1,24 +1,23 @@
 import path from "node:path";
-import {ViteUserConfig, defineConfig} from "vitest/config";
+import {TestUserConfig, defineConfig} from "vitest/config";
 import {browserTestProject} from "./configs/vitest.config.browser.js";
 import {e2eMainnetProject, e2eMinimalProject} from "./configs/vitest.config.e2e.js";
 import {specProjectMainnet, specProjectMinimal} from "./configs/vitest.config.spec.js";
 import {typesTestProject} from "./configs/vitest.config.types.js";
 import {unitTestMainnetProject, unitTestMinimalProject} from "./configs/vitest.config.unit.js";
+import {esmCjsInteropPlugin} from "./scripts/vite/plugins/esmCjsInteropPlugin.js";
 
-const isBun = "bun" in process.versions;
-
-export function getReporters(): ViteUserConfig["test"]["reporters"] {
-  if (isBun) return [["default", {summary: false}]];
-  if (process.env.GITHUB_ACTIONS) return ["verbose", "hanging-process", "github-actions"];
+export function getReporters(): TestUserConfig["reporters"] {
+  if (process.env.GITHUB_ACTIONS) return ["tree", "hanging-process", "github-actions"];
   if (process.env.TEST_COMPACT_OUTPUT) return ["basic", "hanging-process"];
 
-  return ["verbose", "hanging-process"];
+  return ["tree", "hanging-process"];
 }
 
 export default defineConfig({
+  plugins: [esmCjsInteropPlugin()],
   test: {
-    workspace: [
+    projects: [
       {
         extends: true,
         ...unitTestMinimalProject,
@@ -77,9 +76,8 @@ export default defineConfig({
     onConsoleLog: () => !process.env.TEST_QUIET_CONSOLE,
     coverage: {
       enabled: false,
+      include: ["packages/**/src/**.{ts}"],
       clean: true,
-      all: false,
-      extension: [".ts"],
       provider: "v8",
       reporter: [["lcovonly", {file: "lcov.info"}], ["text"]],
       reportsDirectory: "./coverage",