build: add apple silicon support (#24545)

* chore: add patches to prevent installation of non-arm pip packages

* chore: add patches for apple-silicon

* build: add apple silicon build

* ci: add testing of new arm binary

* chore: remove / update for upstreamed patches

* Skip content tracing on macos on arm

* build: ensure that spec native modules are rebuilt for arm64 on apple-silicon

* chore: fix patches

* chore: fix broken patch

* chore: fix arm64 DCHECK

* build: add MAS arm64 build

* build: disable arm2 tests

* chore: update patches

* build: actually build MAS version of apple silicon app

Co-authored-by: John Kleinschmidt <jkleinsc@github.com>

script/release/ci-release-build.js

@@ -6,6 +6,7 @@ const request = require('request');
 const BUILD_APPVEYOR_URL = 'https://ci.appveyor.com/api/builds';
 const CIRCLECI_PIPELINE_URL = 'https://circleci.com/api/v2/project/gh/electron/electron/pipeline';
 const VSTS_URL = 'https://github.visualstudio.com/electron/_apis/build';
+const DEVOPS_URL = 'https://dev.azure.com/electron-ci/electron/_apis/build';
 const CIRCLECI_WAIT_TIME = process.env.CIRCLECI_WAIT_TIME || 30000;
 
 const appVeyorJobs = {
@@ -30,6 +31,7 @@ const circleCIPublishWorkflows = [
 
 const vstsArmJobs = [
   'electron-arm-testing',
+  'electron-arm2-testing',
   'electron-arm64-testing',
   'electron-woa-testing'
 ];
@@ -249,45 +251,43 @@ function buildCircleCI (targetBranch, options) {
 }
 
 async function buildVSTS (targetBranch, options) {
-  if (options.armTest) {
-    assert(vstsArmJobs.includes(options.job), `Unknown VSTS CI arm test job name: ${options.job}. Valid values are: ${vstsArmJobs}.`);
+  assert(options.armTest, `${options.ci} only works with the --armTest option.`);
+  assert(vstsArmJobs.includes(options.job), `Unknown VSTS CI arm test job name: ${options.job}. Valid values are: ${vstsArmJobs}.`);
+
+  console.log(`Triggering VSTS to run build on branch: ${targetBranch}.`);
+  const environmentVariables = {};
+
+  if (options.circleBuildNum) {
+    environmentVariables.CIRCLE_BUILD_NUM = options.circleBuildNum;
+  } else if (options.appveyorJobId) {
+    environmentVariables.APPVEYOR_JOB_ID = options.appveyorJobId;
   }
 
-  console.log(`Triggering VSTS to run build on branch: ${targetBranch} with release flag.`);
-  const environmentVariables = {
-    ELECTRON_RELEASE: 1
-  };
-
-  if (options.armTest) {
-    if (options.circleBuildNum) {
-      environmentVariables.CIRCLE_BUILD_NUM = options.circleBuildNum;
-    } else if (options.appveyorJobId) {
-      environmentVariables.APPVEYOR_JOB_ID = options.appveyorJobId;
-    }
-  } else {
-    if (!options.ghRelease) {
-      environmentVariables.UPLOAD_TO_S3 = 1;
-    }
+  let vstsURL = VSTS_URL;
+  let vstsToken = process.env.VSTS_TOKEN;
+  if (options.ci === 'DevOps') {
+    vstsURL = DEVOPS_URL;
+    vstsToken = process.env.DEVOPS_TOKEN;
   }
-
   const requestOpts = {
-    url: `${VSTS_URL}/definitions?api-version=4.1`,
+    url: `${vstsURL}/definitions?api-version=4.1`,
     auth: {
       user: '',
-      password: process.env.VSTS_TOKEN
+      password: vstsToken
     },
     headers: {
       'Content-Type': 'application/json'
     }
   };
+  jobRequestedCount++;
   const vstsResponse = await makeRequest(requestOpts, true).catch(err => {
     console.log('Error calling VSTS to get build definitions:', err);
   });
-  const buildsToRun = vstsResponse.value.filter(build => build.name === options.job);
-  buildsToRun.forEach((build) => callVSTSBuild(build, targetBranch, environmentVariables));
+  const buildToRun = vstsResponse.value.find(build => build.name === options.job);
+  callVSTSBuild(buildToRun, targetBranch, environmentVariables, vstsURL, vstsToken);
 }
 
-async function callVSTSBuild (build, targetBranch, environmentVariables) {
+async function callVSTSBuild (build, targetBranch, environmentVariables, vstsURL, vstsToken) {
   const buildBody = {
     definition: build,
     sourceBranch: targetBranch,
@@ -297,10 +297,10 @@ async function callVSTSBuild (build, targetBranch, environmentVariables) {
     buildBody.parameters = JSON.stringify(environmentVariables);
   }
   const requestOpts = {
-    url: `${VSTS_URL}/builds?api-version=4.1`,
+    url: `${vstsURL}/builds?api-version=4.1`,
     auth: {
       user: '',
-      password: process.env.VSTS_TOKEN
+      password: vstsToken
     },
     headers: {
       'Content-Type': 'application/json'
@@ -308,7 +308,6 @@ async function callVSTSBuild (build, targetBranch, environmentVariables) {
     body: JSON.stringify(buildBody),
     method: 'POST'
   };
-  jobRequestedCount++;
   const vstsResponse = await makeRequest(requestOpts, true).catch(err => {
     console.log(`Error calling VSTS for job ${build.name}`, err);
   });
@@ -326,6 +325,7 @@ function runRelease (targetBranch, options) {
         buildAppVeyor(targetBranch, options);
         break;
       }
+      case 'DevOps':
       case 'VSTS': {
         buildVSTS(targetBranch, options);
         break;
@@ -351,7 +351,7 @@ if (require.main === module) {
   const targetBranch = args._[0];
   if (args._.length < 1) {
     console.log(`Trigger CI to build release builds of electron.
-    Usage: ci-release-build.js [--job=CI_JOB_NAME] [--ci=CircleCI|AppVeyor|VSTS]
+    Usage: ci-release-build.js [--job=CI_JOB_NAME] [--ci=CircleCI|AppVeyor|VSTS|DevOps]
     [--ghRelease] [--armTest] [--circleBuildNum=xxx] [--appveyorJobId=xxx] TARGET_BRANCH
     `);
     process.exit(0);

script/spec-runner.js

@@ -2,7 +2,7 @@
 
 const childProcess = require('child_process');
 const crypto = require('crypto');
-const fs = require('fs');
+const fs = require('fs-extra');
 const { hashElement } = require('folder-hash');
 const path = require('path');
 const unknownFlags = [];
@@ -225,6 +225,9 @@ async function installSpecModules (dir) {
     npm_config_nodedir: nodeDir,
     npm_config_msvs_version: '2019'
   });
+  if (fs.existsSync(path.resolve(dir, 'node_modules'))) {
+    await fs.remove(path.resolve(dir, 'node_modules'));
+  }
   const { status } = childProcess.spawnSync(NPX_CMD, [`yarn@${YARN_VERSION}`, 'install', '--frozen-lockfile'], {
     env,
     cwd: dir,
@@ -234,6 +237,19 @@ async function installSpecModules (dir) {
     console.log(`${fail} Failed to yarn install in '${dir}'`);
     process.exit(1);
   }
+
+  // TODO(MarshallOfSound): Remove once node-gyp supports arm64
+  if (process.platform === 'darwin' && process.env.npm_config_arch === 'arm64') {
+    for (const nodeModule of fs.readdirSync(path.resolve(dir, 'node_modules'))) {
+      if (fs.existsSync(path.resolve(dir, 'node_modules', nodeModule, 'binding.gyp'))) {
+        childProcess.spawnSync(NPX_CMD, ['https://github.com/MarshallOfSound/node-gyp/archive/apple-silicon.tar.gz', 'clean', 'configure', 'build', '--arch=arm64'], {
+          env,
+          cwd: path.resolve(dir, 'node_modules', nodeModule),
+          stdio: 'inherit'
+        });
+      }
+    }
+  }
 }
 
 function getSpecHash () {
@@ -244,6 +260,7 @@ function getSpecHash () {
       hasher.update(fs.readFileSync(path.resolve(__dirname, '../spec-main/package.json')));
       hasher.update(fs.readFileSync(path.resolve(__dirname, '../spec/yarn.lock')));
       hasher.update(fs.readFileSync(path.resolve(__dirname, '../spec-main/yarn.lock')));
+      hasher.update(fs.readFileSync(path.resolve(__dirname, '../script/spec-runner.js')));
       return hasher.digest('hex');
     })(),
     (async () => {