add test retry handling and cache management for E2E tests

tools/e2e-tests/helpers.js

@@ -9,6 +9,73 @@ const rimraf = require('rimraf');
 const REPO_ROOT = path.resolve(__dirname, '../..');
 const METEOR_EXECUTABLE = path.join(REPO_ROOT, 'meteor');
 
+/**
+ * Returns true when the current Jest test is a retry attempt.
+ */
+export function isRetryAttempt() {
+  return Boolean(globalThis.__e2eIsRetryAttempt);
+}
+
+/**
+ * Snapshot file contents so they can be restored after a test mutates them.
+ * @param {string} baseDir - Base directory for relative paths
+ * @param {string[]} relPaths - Relative paths to snapshot
+ * @returns {Promise<Map<string, {content: string|null, existed: boolean}>>}
+ */
+export async function snapshotFiles(baseDir, relPaths = []) {
+  const snapshot = new Map();
+  for (const relPath of relPaths) {
+    if (!relPath) continue;
+    const fullPath = path.join(baseDir, relPath);
+    if (await fs.pathExists(fullPath)) {
+      const content = await fs.readFile(fullPath, 'utf8');
+      snapshot.set(fullPath, { content, existed: true });
+    } else {
+      snapshot.set(fullPath, { content: null, existed: false });
+    }
+  }
+  return snapshot;
+}
+
+/**
+ * Restore files captured by snapshotFiles to their original state.
+ * @param {Map<string, {content: string|null, existed: boolean}>} snapshot
+ */
+export async function restoreFiles(snapshot) {
+  if (!snapshot || snapshot.size === 0) return;
+  for (const [fullPath, entry] of snapshot.entries()) {
+    if (entry.existed) {
+      await fs.writeFile(fullPath, entry.content, 'utf8');
+    } else if (await fs.pathExists(fullPath)) {
+      await fs.remove(fullPath);
+    }
+  }
+}
+
+/**
+ * Remove build artifacts and caches under a Meteor app directory.
+ * @param {string} appDir - Directory containing the Meteor app
+ */
+export async function clearBuildArtifacts(appDir) {
+  if (!appDir) return;
+  const targets = [
+    '_build',
+    '.meteor/local/build',
+    '.meteor/local/bundler-cache',
+    '.meteor/local/plugin-cache',
+    'node_modules/.cache/rspack',
+    'node_modules/.cache/meteor',
+  ];
+  for (const target of targets) {
+    const fullPath = path.join(appDir, target);
+    try {
+      await fs.remove(fullPath);
+    } catch (err) {
+      console.log(`Could not remove ${fullPath}: ${err.message}`);
+    }
+  }
+}
+
 /**
  * Helper function to set up a Meteor app in a temporary directory
  * Copies the app and runs npm install

tools/e2e-tests/jest.setup.js

@@ -23,9 +23,36 @@ if (process.env.CI) {
   });
 }
 
-// This runs before each test
+// Retries are only enabled on CI — local runs fail fast so flakes surface.
+if (process.env.CI) {
+  jest.retryTimes(2, { logErrorsBeforeRetry: true });
+}
+
+// Track attempts per test — Jest 29 doesn't expose currentTestRetryAttempt.
+const attemptCounts = new Map();
+
 beforeEach(() => {
-  const name = expect.getState().currentTestName;
-  // e.g. a bright cyan arrow and test name
-  console.log(chalk.cyan(`▶ ${name}`));
+  const name = expect.getState().currentTestName || '<unknown>';
+  const attemptIndex = attemptCounts.get(name) ?? 0;
+  attemptCounts.set(name, attemptIndex + 1);
+  globalThis.__e2eIsRetryAttempt = attemptIndex > 0;
+
+  const attemptLabel = attemptIndex > 0 ? chalk.yellow(` (retry ${attemptIndex})`) : '';
+  console.log(chalk.cyan(`▶ ${name}`) + attemptLabel);
+});
+
+// E2E_FORCE_FLAKY_TEST: force matching tests to fail on the first attempt
+// to validate retry isolation. Runs in afterEach so the test body completes
+// (including any mutations) before the forced failure.
+afterEach(() => {
+  const flakyPattern = process.env.E2E_FORCE_FLAKY_TEST;
+  if (!flakyPattern) return;
+  const name = expect.getState().currentTestName || '<unknown>';
+  const attemptsSoFar = attemptCounts.get(name) ?? 0;
+  if (attemptsSoFar === 1 && name.includes(flakyPattern)) {
+    throw new Error(
+      `[E2E_FORCE_FLAKY_TEST] Forcing first-attempt failure for "${name}" ` +
+      `(matches pattern "${flakyPattern}"). Retry should pass if isolation works.`
+    );
+  }
 });

tools/e2e-tests/test-helpers.js

@@ -7,13 +7,17 @@ import {
   appendFileContent,
   buildMeteorApp,
   cleanupTempDir,
+  clearBuildArtifacts,
   createMeteorApp,
+  isRetryAttempt,
   killMeteorProcess,
   killProcessByPort,
+  restoreFiles,
   runMeteorApp,
   runMeteorCommand,
   runMeteorTests,
   setupMeteorApp,
+  snapshotFiles,
   wait,
   waitForMeteorOutput,
   waitForPlaywrightConsole
@@ -111,6 +115,11 @@ export function testMeteorBundler(options) {
     beforeEach(async () => {
       // Ensure any process on the port is killed
       await killProcessByPort([port, devServerPortStr]);
+
+      // On retry, purge build caches so the next attempt starts clean.
+      if (isRetryAttempt() && tempDir) {
+        await clearBuildArtifacts(tempDir);
+      }
     });
 
     afterEach(async () => {
@@ -235,6 +244,17 @@ export function testMeteorRspackBundler(options) {
     let tempDir;
     let appDir;
     let previousRspackDevServerPort;
+    let fileSnapshot;
+
+    // Paths the rspack bundler generator mutates via appendFileContent. Snapshotted
+    // in beforeEach and restored in afterEach so retries see pristine source files.
+    const mutatedPaths = [
+      filePaths.client,
+      filePaths.server,
+      filePaths.test,
+      filePaths.testClient,
+      filePaths.testServer,
+    ].filter(Boolean);
 
     beforeAll(async () => {
       // Route this test's rspack dev server to the configured port so it doesn't
@@ -324,6 +344,16 @@ export function testMeteorRspackBundler(options) {
     beforeEach(async () => {
       // Ensure any process on the port is killed
       await killProcessByPort([port, devServerPortStr]);
+
+      // On retry, purge build artifacts so the next attempt recompiles from a
+      // clean slate (guards against caches that encode the failed attempt's state).
+      if (isRetryAttempt() && appDir) {
+        await clearBuildArtifacts(appDir);
+      }
+
+      // Capture mutable source files so afterEach can restore them. This makes
+      // retries see pristine files even after appendFileContent calls.
+      fileSnapshot = tempDir ? await snapshotFiles(tempDir, mutatedPaths) : null;
     });
 
     afterEach(async () => {
@@ -331,6 +361,13 @@ export function testMeteorRspackBundler(options) {
         await killMeteorProcess(meteorProcess);
         meteorProcess = null;
       }
+
+      // Restore mutated files regardless of pass/fail — idempotent on green runs,
+      // essential on retries.
+      if (fileSnapshot) {
+        await restoreFiles(fileSnapshot);
+        fileSnapshot = null;
+      }
     });
 
     test(`"meteor run" / should run and rebuild the app with Rspack`, async () => {
@@ -818,6 +855,20 @@ export function testMeteorRspackBundler(options) {
         : '';
       const localDirSuffix = meteorLocalDirName ? `-${meteorLocalDirName}` : '';
 
+      // On retry, the prior attempt deleted the artifacts this test asserts on.
+      // Re-seed by running meteor briefly so the assertFileExist checks below hold.
+      if (isRetryAttempt()) {
+        console.log('[retry] re-seeding build artifacts for meteor reset test');
+        const seedResult = await runMeteorApp(tempDir, port, {
+          waitForOutput: "=> App running at",
+          isMonorepo,
+          skipWaitOn: skipClient,
+          env: { ...env, ...(env.meteorRun || {}) },
+        });
+        await killMeteorProcess(seedResult.meteorProcess);
+        await killProcessByPort([port, devServerPortStr]);
+      }
+
       // Verify build artifacts exist from previous tests
       await assertFileExist(appDir, buildDir);
       await assertFileExist(appDir, 'node_modules/.cache/rspack');
@@ -972,6 +1023,13 @@ export function testMeteorSkeleton(options) {
     beforeEach(async () => {
       // Ensure any process on the port is killed
       await killProcessByPort([port, devServerPortStr]);
+
+      // On retry, purge caches left by the failing attempt so the next one
+      // recompiles from scratch. Skip when tempDir isn't set yet (e.g. retry
+      // of the "meteor create" test, which allocates its own tempDir).
+      if (isRetryAttempt() && tempDir) {
+        await clearBuildArtifacts(tempDir);
+      }
     });
 
     afterEach(async () => {
@@ -1183,6 +1241,18 @@ export function testMeteorSkeleton(options) {
         : '';
       const localDirSuffix = meteorLocalDirName ? `-${meteorLocalDirName}` : '';
 
+      // On retry, the prior attempt deleted the artifacts this test asserts on.
+      // Re-seed by running meteor briefly when the skeleton produces build caches.
+      if (isRetryAttempt() && !skipBuildCacheCheck) {
+        console.log('[retry] re-seeding build artifacts for meteor reset test');
+        const seedResult = await runMeteorApp(tempDir, port, {
+          waitForOutput: "=> App running at",
+          env: env.meteorRun,
+        });
+        await killMeteorProcess(seedResult.meteorProcess);
+        await killProcessByPort([port, devServerPortStr]);
+      }
+
       // Verify build artifacts exist from previous tests
       if (!skipBuildCacheCheck) {
         await assertFileExist(tempDir, "_build");