build: the win32 tests should build ia32 native modules

.circleci/config.yml

@@ -11,9 +11,13 @@ parameters:
 
   run-build-linux:
     type: boolean
-    default: true
+    default: false
 
   run-build-mac:
+    type: boolean
+    default: false
+
+  run-build-win:
     type: boolean
     default: true
 
@@ -72,6 +76,18 @@ machine-mac-large: &machine-mac-large
   macos:
     xcode: "10.3.0"
 
+machine-windows-medium: &machine-windows-medium
+  machine:
+    image: 'windows-server-2019-vs2019:stable'
+    resource_class: windows.medium
+    shell: bash.exe
+
+machine-windows-xlarge: &machine-windows-xlarge
+  machine:
+    image: 'windows-server-2019-vs2019:stable'
+    resource_class: windows.xlarge
+    shell: bash.exe
+
 # Build configurations options.
 env-testing-build: &env-testing-build
   GN_CONFIG: //electron/build/args/testing.gn
@@ -159,6 +175,9 @@ env-macos-build: &env-macos-build
   # Disable pre-compiled headers to reduce out size, only useful for rebuilds
   GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false'
 
+env-windows-build: &env-windows-build
+  NINJA_SUFFIX: '.exe'
+
 # Individual (shared) steps.
 step-maybe-notify-slack-failure: &step-maybe-notify-slack-failure
   run:
@@ -195,7 +214,12 @@ step-depot-tools-get: &step-depot-tools-get
 step-depot-tools-add-to-path: &step-depot-tools-add-to-path
   run:
     name: Add depot tools to PATH
-    command: echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV
+    command: |
+      if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+        echo 'export PATH="'"$PWD"'/depot_tools:$PATH"' >> $BASH_ENV
+      else
+        echo 'export PATH="$PATH:'"$PWD"'/depot_tools"' >> $BASH_ENV
+      fi
 
 step-gclient-sync: &step-gclient-sync
   run:
@@ -222,6 +246,9 @@ step-setup-env-for-build: &step-setup-env-for-build
       if [ "$USE_SCCACHE" == "true" ]; then
         # https://github.com/mozilla/sccache
         SCCACHE_PATH="$PWD/src/electron/external_binaries/sccache"
+        if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+          SCCACHE_PATH="C:\\\\Users\\circleci\\project\\src\\electron\\external_binaries\\sccache.exe"
+        fi
         echo 'export SCCACHE_PATH="'"$SCCACHE_PATH"'"' >> $BASH_ENV
         if [ "$CIRCLE_PR_NUMBER" != "" ]; then
           #if building a fork set readonly access to sccache
@@ -273,9 +300,9 @@ step-delete-git-directories: &step-delete-git-directories
 # we are on a macOS machine to get the correct state
 step-install-npm-deps-on-mac: &step-install-npm-deps-on-mac
   run:
-    name: Install node_modules on MacOS
+    name: Install node_modules for cross-OS checkout
     command: |
-      if [ "`uname`" == "Darwin" ]; then
+      if [ "`uname`" != "Linux" ]; then
         cd src/electron
         node script/yarn install
       fi
@@ -284,9 +311,9 @@ step-install-npm-deps-on-mac: &step-install-npm-deps-on-mac
 # and the expected state on macOS
 step-fix-sync-on-mac: &step-fix-sync-on-mac
   run:
-    name: Fix Sync on macOS
+    name: Fix Sync for cross-OS checkout
     command: |
-      if [ "`uname`" == "Darwin" ]; then
+      if [ "`uname`" != "Linux" ]; then
         # Fix Clang Install (wrong binary)
         rm -rf src/third_party/llvm-build
         python src/tools/clang/scripts/update.py
@@ -396,6 +423,13 @@ step-electron-dist-build: &step-electron-dist-build
             else
               target_cpu="$TARGET_ARCH"
             fi
+          elif [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+            target_os=win
+            if [ x"$TARGET_ARCH" == x ]; then
+              target_cpu=x64
+            else
+              target_cpu="$TARGET_ARCH"
+            fi
           else
             echo "Unknown system: `uname`"
             exit 1
@@ -429,6 +463,10 @@ step-nodejs-headers-build: &step-nodejs-headers-build
     command: |
       cd src
       ninja -C out/Default third_party/electron_node:headers
+      if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+        mkdir -p out/Default/gen/node_headers/Release
+        cp out/Default/electron.lib out/Default/gen/node_headers/Release/node.lib
+      fi
 
 step-nodejs-headers-store: &step-nodejs-headers-store
   store_artifacts:
@@ -527,7 +565,7 @@ step-setup-linux-for-headless-testing: &step-setup-linux-for-headless-testing
   run:
     name: Setup for headless testing
     command: |
-      if [ "`uname`" != "Darwin" ]; then
+      if [ "`uname`" == "Linux" ]; then
         sh -e /etc/init.d/xvfb start
       fi
 
@@ -545,7 +583,7 @@ step-mksnapshot-build: &step-mksnapshot-build
     command: |
       cd src
       ninja -C out/Default electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES
-      if [ "`uname`" != "Darwin" ]; then
+      if [ "`uname`" == "Linux" ]; then
         if [ "$TARGET_ARCH" == "arm" ]; then
           electron/script/strip-binaries.py --file $PWD/out/Default/clang_x86_v8_arm/mksnapshot
         elif [ "$TARGET_ARCH" == "arm64" ]; then
@@ -659,7 +697,7 @@ step-ninja-summary: &step-ninja-summary
   run:
     name: Print ninja summary
     command: |
-      python depot_tools/post_build_ninja_summary.py -C src/out/Default
+      vpython depot_tools/post_build_ninja_summary.py -C src/out/Default
 
 step-ninja-report: &step-ninja-report
   store_artifacts:
@@ -703,7 +741,7 @@ step-restore-out-cache: &step-restore-out-cache
     paths:
       - ./src/out/Default
     keys:
-      - v7-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }}
+      - v8-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }}
     name: Restoring out cache
 
 step-set-git-cache-path: &step-set-git-cache-path
@@ -727,13 +765,19 @@ step-save-out-cache: &step-save-out-cache
   save_cache:
     paths:
       - ./src/out/Default
-    key: v7-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }}
+    key: v8-out-cache-{{ checksum "src/electron/.depshash" }}-{{ checksum "src/electron/.depshash-target" }}
     name: Persisting out cache
 
 step-run-electron-only-hooks: &step-run-electron-only-hooks
   run:
     name: Run Electron Only Hooks
-    command: gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
+    command: |
+      # On windows we have to run gclient through cmd.exe
+      if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+        cmd.exe /c "gclient runhooks --spec=\"solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]\""
+      else
+        gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]"
+      fi
 
 step-generate-deps-hash-cleanly: &step-generate-deps-hash-cleanly
   run:
@@ -880,7 +924,6 @@ steps-checkout-and-save-cache: &steps-checkout-and-save-cache
         name: Wipe Electron
         command: rm -rf src/electron
     - *step-checkout-electron
-    - *step-run-electron-only-hooks
     - *step-generate-deps-hash-cleanly
     - *step-mark-sync-done
     - *step-minimize-workspace-size-from-checkout
@@ -1208,19 +1251,34 @@ commands:
       - run:
           name: Prepare for cross-OS sync restore
           command: |
-            sudo mkdir -p /portal
+            if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
-            sudo chown -R $(id -u):$(id -g) /portal
+              mkdir -p /portal
+              chown -R $(id -u):$(id -g) /portal
+            else
+              sudo mkdir -p /portal
+              sudo chown -R $(id -u):$(id -g) /portal
+            fi
       - *step-maybe-restore-src-cache
       - run:
-          name: Fix the src cache restore point on macOS
+          name: Fix the src cache restore point for cross-OS cache restore
           command: |
             if [ -d "/portal/src" ]; then
               echo Relocating Cache
               rm -rf src
               mv /portal/src ./
             fi
+            # Handle cache restore point on windows
+            if [ -d "/c/portal/src" ]; then
+              echo Relocating Cache on Windows
+              rm -rf src
+              mv /c/portal/src ./
+            fi
   checkout-from-cache:
     steps:
+      - run:
+          name: Ensure LF on Windows
+          command: |
+            git config --global core.autocrlf false
       - *step-checkout-electron
       - *step-maybe-early-exit-doc-only-change
       - *step-depot-tools-get
@@ -1372,14 +1430,19 @@ commands:
             - run:
                 name: Remove the big things on macOS, this seems to be better on average
                 command: |
-                  if [ "`uname`" == "Darwin" ]; then
+                  if [ "`uname`" == "Darwin" ] || [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
                     mkdir -p src/out/Default
                     cd src/out/Default
-                    find . -type f -size +50M -delete
+                    FIND_CMD=find
+                    # For some reason "find" maps to the windows one not the bash one
+                    if [ "`uname|cut -c 1-7`" == "MSYS_NT" ]; then
+                      FIND_CMD='/c/Program Files/Git/usr/bin/find'
+                    fi
+                    "$FIND_CMD" . -type f -size +50M -delete
                     mkdir -p gen/electron
                     cd gen/electron
                     # These files do not seem to like being in a cache, let us remove them
-                    find . -type f -name '*_pkg_info' -delete
+                    "$FIND_CMD" . -type f -name '*_pkg_info' -delete
                   fi
             - *step-save-out-cache
 
@@ -1471,6 +1534,27 @@ jobs:
       GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
     <<: *steps-checkout-and-save-cache
 
+  win-checkout-fast:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True --custom-var=host_os=win'
+    steps:
+      - electron-build:
+          persist: false
+          build: false
+          checkout: true
+          persist-checkout: true
+
+  win-checkout-and-save-cache:
+    <<: *machine-linux-2xlarge
+    environment:
+      <<: *env-linux-2xlarge
+      <<: *env-testing-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True --custom-var=host_os=win'
+    <<: *steps-checkout-and-save-cache
+
   # Layer 2: Builds.
   linux-x64-testing:
     <<: *machine-linux-2xlarge
@@ -1791,6 +1875,39 @@ jobs:
       UPLOAD_TO_S3: << pipeline.parameters.upload-to-s3 >>
     <<: *steps-electron-build-for-publish
 
+  win32-testing:
+    <<: *machine-windows-xlarge
+    environment:
+      <<: *env-mac-large
+      <<: *env-testing-build
+      <<: *env-enable-sccache
+      <<: *env-ninja-status
+      <<: *env-windows-build
+      <<: *env-ia32
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True --custom-var=host_os=win'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: false
+          checkout-and-assume-cache: true
+          attach: false
+
+  win64-testing:
+    <<: *machine-windows-xlarge
+    environment:
+      <<: *env-mac-large
+      <<: *env-testing-build
+      <<: *env-enable-sccache
+      <<: *env-ninja-status
+      <<: *env-windows-build
+      GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True --custom-var=host_os=win'
+    steps:
+      - electron-build:
+          persist: true
+          checkout: false
+          checkout-and-assume-cache: true
+          attach: false
+
   # Layer 3: Tests.
   linux-x64-unittests:
     <<: *machine-linux-2xlarge
@@ -1995,6 +2112,23 @@ jobs:
       <<: *env-send-slack-notifications
     <<: *steps-verify-mksnapshot
 
+  win32-testing-tests:
+    <<: *machine-windows-medium
+    environment:
+      <<: *env-mac-large
+      <<: *env-stack-dumping
+      <<: *env-ia32
+    parallelism: 2
+    <<: *steps-tests
+
+  win64-testing-tests:
+    <<: *machine-windows-medium
+    environment:
+      <<: *env-mac-large
+      <<: *env-stack-dumping
+    parallelism: 2
+    <<: *steps-tests
+
   # Layer 4: Summary.
   linux-x64-release-summary:
     <<: *machine-linux-medium
@@ -2164,6 +2298,26 @@ workflows:
           requires:
             - mas-testing
 
+  build-win:
+    when: << pipeline.parameters.run-build-win >>
+    jobs:
+      - win-checkout-fast
+      - win-checkout-and-save-cache
+
+      - win32-testing:
+          requires:
+            - win-checkout-and-save-cache
+      - win64-testing:
+          requires:
+            - win-checkout-and-save-cache
+
+      - win32-testing-tests:
+          requires:
+            - win32-testing
+      - win64-testing-tests:
+          requires:
+            - win64-testing
+
   nightly-linux-release-test:
     triggers:
       - schedule:

script/external-binaries.json

@@ -33,7 +33,7 @@
       "platform": "linux"
     },
     {
-      "url": "sccache-win32-x64.zip",
+      "url": "sccache-win32-x64-v2.zip",
       "platform": "win32"
     }
   ]

script/generate-deps-hash.js

@@ -38,7 +38,7 @@ for (const file of filesToHash) {
 const extraArgs = process.env.GCLIENT_EXTRA_ARGS || 'no_extra_args'
 hasher.update(extraArgs)
 
-const effectivePlatform = extraArgs.includes('host_os=mac') ? 'darwin' : process.platform
+const effectivePlatform = extraArgs.includes('host_os=mac') ? 'darwin' : (extraArgs.includes('host_os=win') ? 'win32' : process.platform)
 
 // Write the hash to disk
 fs.writeFileSync(path.resolve(__dirname, '../.depshash'), hasher.digest('hex'))