ci: run main and remote woa tests separately

azure-pipelines-woa.yml

@@ -53,6 +53,16 @@ steps:
   env:
     APPVEYOR_TOKEN: $(APPVEYOR_TOKEN)
 
+- powershell: |
+    $localArtifactPath = "$pwd\src\pdb.zip"
+    $serverArtifactPath = "$env:APPVEYOR_URL/buildjobs/$env:APPVEYOR_JOB_ID/artifacts/pdb.zip"
+    Invoke-RestMethod -Method Get -Uri $serverArtifactPath -OutFile $localArtifactPath -Headers @{ "Authorization" = "Bearer $env:APPVEYOR_TOKEN" }
+    cd src
+    & "${env:ProgramFiles(x86)}\7-Zip\7z.exe" x -y pdb.zip
+  displayName: 'Download pdb files for detailed stacktraces'
+  env:
+    APPVEYOR_TOKEN: $(APPVEYOR_TOKEN)
+
 - powershell: |
     New-Item src\out\Default\gen\node_headers\Release -Type directory
     Copy-Item -path src\out\Default\electron.lib -destination src\out\Default\gen\node_headers\Release\node.lib
@@ -63,15 +73,30 @@ steps:
     set npm_config_nodedir=%cd%\out\Default\gen\node_headers
     set npm_config_arch=arm64
     cd electron
-    # CalculateNativeWinOcclusion is disabled due to https://bugs.chromium.org/p/chromium/issues/detail?id=1139022
-    node script/yarn test -- --enable-logging --verbose --disable-features=CalculateNativeWinOcclusion
-  displayName: 'Run Electron tests'
+    node script/yarn test --runners=main --runTestFilesSeperately --enable-logging --disable-features=CalculateNativeWinOcclusion
+  displayName: 'Run Electron Main process tests'
+  env:
+    ELECTRON_ENABLE_STACK_DUMPING: true
+    ELECTRON_OUT_DIR: Default
+    IGNORE_YARN_INSTALL_ERROR: 1
+    ELECTRON_TEST_RESULTS_DIR: junit
+    MOCHA_MULTI_REPORTERS: 'mocha-junit-reporter, tap'
+    MOCHA_REPORTER: mocha-multi-reporters
+
+- script: |
+    cd src
+    set npm_config_nodedir=%cd%\out\Default\gen\node_headers
+    set npm_config_arch=arm64
+    cd electron
+    node script/yarn test --runners=remote --enable-logging --disable-features=CalculateNativeWinOcclusion
+  displayName: 'Run Electron Remote based tests'
   env:
     ELECTRON_OUT_DIR: Default
     IGNORE_YARN_INSTALL_ERROR: 1
     ELECTRON_TEST_RESULTS_DIR: junit
     MOCHA_MULTI_REPORTERS: 'mocha-junit-reporter, tap'
     MOCHA_REPORTER: mocha-multi-reporters
+  condition: always()    
 
 - task: PublishTestResults@2
   displayName: 'Publish Test Results'

script/spec-runner.js

@@ -13,7 +13,7 @@ const fail = '✗'.red;
 
 const args = require('minimist')(process.argv, {
   string: ['runners', 'target'],
-  boolean: ['buildNativeTests'],
+  boolean: ['buildNativeTests', 'runTestFilesSeperately'],
   unknown: arg => unknownFlags.push(arg)
 });
 
@@ -123,24 +123,55 @@ async function runElectronTests () {
   }
 }
 
-async function runRemoteBasedElectronTests () {
+async function runTestUsingElectron (specDir, testName) {
   let exe = path.resolve(BASE, utils.getElectronExec());
-  const runnerArgs = ['electron/spec', ...unknownArgs.slice(2)];
+  const runnerArgs = [`electron/${specDir}`, ...unknownArgs.slice(2)];
   if (process.platform === 'linux') {
     runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe);
     exe = 'python3';
   }
-
-  const { status } = childProcess.spawnSync(exe, runnerArgs, {
+  const { status, signal } = childProcess.spawnSync(exe, runnerArgs, {
     cwd: path.resolve(__dirname, '../..'),
     stdio: 'inherit'
   });
   if (status !== 0) {
-    const textStatus = process.platform === 'win32' ? `0x${status.toString(16)}` : status.toString();
-    console.log(`${fail} Electron tests failed with code ${textStatus}.`);
+    if (status) {
+      const textStatus = process.platform === 'win32' ? `0x${status.toString(16)}` : status.toString();
+      console.log(`${fail} Electron tests failed with code ${textStatus}.`);
+    } else {
+      console.log(`${fail} Electron tests failed with kill signal ${signal}.`);
+    }
     process.exit(1);
   }
-  console.log(`${pass} Electron remote process tests passed.`);
+  console.log(`${pass} Electron ${testName} process tests passed.`);
+}
+
+const specFilter = (file) => {
+  if (!/-spec\.[tj]s$/.test(file)) {
+    return false;
+  } else {
+    return true;
+  }
+};
+
+async function runTests (specDir, testName) {
+  if (args.runTestFilesSeperately) {
+    const getFiles = require('../spec/static/get-files');
+    const testFiles = await getFiles(path.resolve(__dirname, `../${specDir}`), { filter: specFilter });
+    const baseElectronDir = path.resolve(__dirname, '..');
+    unknownArgs.splice(unknownArgs.length, 0, '--files', '');
+    testFiles.sort().forEach(async (file) => {
+      unknownArgs.splice((unknownArgs.length - 1), 1, path.relative(baseElectronDir, file));
+      console.log(`Running tests for ${unknownArgs[unknownArgs.length - 1]}`);
+      await runTestUsingElectron(specDir, testName);
+    });
+  } else {
+    await runTestUsingElectron(specDir, testName);
+  }
+}
+
+async function runRemoteBasedElectronTests () {
+  await runTests('spec', 'remote');
 }
 
 async function runNativeElectronTests () {
@@ -195,27 +226,7 @@ async function runNativeElectronTests () {
 }
 
 async function runMainProcessElectronTests () {
-  let exe = path.resolve(BASE, utils.getElectronExec());
-  const runnerArgs = ['electron/spec-main', ...unknownArgs.slice(2)];
-  if (process.platform === 'linux') {
-    runnerArgs.unshift(path.resolve(__dirname, 'dbus_mock.py'), exe);
-    exe = 'python3';
-  }
-
-  const { status, signal } = childProcess.spawnSync(exe, runnerArgs, {
-    cwd: path.resolve(__dirname, '../..'),
-    stdio: 'inherit'
-  });
-  if (status !== 0) {
-    if (status) {
-      const textStatus = process.platform === 'win32' ? `0x${status.toString(16)}` : status.toString();
-      console.log(`${fail} Electron tests failed with code ${textStatus}.`);
-    } else {
-      console.log(`${fail} Electron tests failed with kill signal ${signal}.`);
-    }
-    process.exit(1);
-  }
-  console.log(`${pass} Electron main process tests passed.`);
+  await runTests('spec-main', 'main');
 }
 
 async function installSpecModules (dir) {