Merge remote-tracking branch 'origin/main' into win-cross

This commit is contained in:
Samuel Attard
2024-06-14 12:13:09 -05:00
17 changed files with 237 additions and 51 deletions

View File

@@ -253,6 +253,25 @@ step-depot-tools-get: &step-depot-tools-get
sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
else
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
# Remove swift-format dep from cipd on macOS until we send a patch upstream.
cd depot_tools
cat > gclient.diff \<< 'EOF'
diff --git a/gclient.py b/gclient.py
index c305c248..e6e0fbdc 100755
--- a/gclient.py
+++ b/gclient.py
@@ -783,7 +783,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
not condition or "non_git_source" not in condition):
continue
cipd_root = self.GetCipdRoot()
- for package in dep_value.get('packages', []):
+ packages = dep_value.get('packages', [])
+ for package in (x for x in packages if "infra/3pp/tools/swift-format" not in x.get('package')):
deps_to_add.append(
CipdDependency(parent=self,
name=name,
EOF
git apply --3way gclient.diff
fi
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools

View File

@@ -27,7 +27,12 @@ runs:
shell: bash
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
# Remove swift-format dep from cipd on macOS until we send a patch upstream.
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update

View File

@@ -8,17 +8,60 @@ on:
description: 'SHA for electron/build image'
default: 'cf814a4d2501e8e843caea071a6b70a48e78b855'
required: true
# push
skip-macos:
type: boolean
description: 'Skip macOS builds'
default: false
required: false
skip-linux:
type: boolean
description: 'Skip Linux builds'
default: false
required: false
skip-lint:
type: boolean
description: 'Skip lint check'
default: false
required: false
# push:
# pull_request:
jobs:
changes:
runs-on: ubuntu-latest
permissions:
pull-requests: read
outputs:
docs: ${{ steps.filter.outputs.docs }}
src: ${{ steps.filter.outputs.src }}
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 #v4.0.2
- uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
id: filter
with:
filters: |
docs:
- 'docs/**'
src:
- '!docs/**'
# Lint Jobs
lint:
if: ${{ !inputs.skip-lint }}
uses: ./.github/workflows/pipeline-electron-lint.yml
with:
container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root"}'
secrets: inherit
# Docs Only Jobs
docs-only:
needs: changes
if: ${{ needs.changes.outputs.docs == 'true' && needs.changes.outputs.src == 'false'}}
uses: ./.github/workflows/pipeline-electron-docs-only.yml
with:
container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root"}'
secrets: inherit
# Checkout Jobs
checkout-windows:
runs-on: aks-linux-large
@@ -40,7 +83,9 @@ jobs:
- name: Checkout & Sync & Save
uses: ./src/electron/.github/actions/checkout
checkout-macos:
needs: changes
if: false
# if: ${{ needs.changes.outputs.src == 'true' && !inputs.skip-macos}}
runs-on: aks-linux-large
container:
image: ghcr.io/electron/build:${{ inputs.build-image-sha }}
@@ -60,8 +105,11 @@ jobs:
uses: ./src/electron/.github/actions/checkout
with:
generate-sas-token: 'true'
checkout-linux:
needs: changes
if: false
# if: ${{ needs.changes.outputs.src == 'true' && !inputs.skip-linux}}
runs-on: aks-linux-large
container:
image: ghcr.io/electron/build:${{ inputs.build-image-sha }}
@@ -116,6 +164,7 @@ jobs:
build-runs-on: aks-linux-large
test-runs-on: aks-linux-medium
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
test-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root --privileged --init"}'
target-platform: linux
target-arch: x64
is-release: false
@@ -144,8 +193,9 @@ jobs:
needs: checkout-linux
with:
build-runs-on: aks-linux-large
test-runs-on: aks-linux-medium
test-runs-on: aks-linux-arm-medium
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
test-container: '{"image":"ghcr.io/electron/test:arm32v7-${{ inputs.build-image-sha }}","options":"--user root --privileged --init"}'
target-platform: linux
target-arch: arm
is-release: false
@@ -159,8 +209,9 @@ jobs:
needs: checkout-linux
with:
build-runs-on: aks-linux-large
test-runs-on: aks-linux-medium
test-runs-on: aks-linux-arm-medium
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
test-container: '{"image":"ghcr.io/electron/test:arm64v8-${{ inputs.build-image-sha }}","options":"--user root --privileged --init"}'
target-platform: linux
target-arch: arm64
is-release: false

14
.github/workflows/config/gclient.diff vendored Normal file
View File

@@ -0,0 +1,14 @@
diff --git a/gclient.py b/gclient.py
index 59e2b4c5197928bdba1ef69bdbe637d7dfe471c1..b4bae5e48c83c84bd867187afaf40eed16e69851 100755
--- a/gclient.py
+++ b/gclient.py
@@ -783,7 +783,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings):
not condition or "non_git_source" not in condition):
continue
cipd_root = self.GetCipdRoot()
- for package in dep_value.get('packages', []):
+ packages = dep_value.get('packages', [])
+ for package in (x for x in packages if "infra/3pp/tools/swift-format" not in x.get('package')):
deps_to_add.append(
CipdDependency(parent=self,
name=name,

View File

@@ -41,6 +41,7 @@ jobs:
uses: ./.github/workflows/pipeline-segment-electron-build.yml
needs: checkout-linux
with:
environment: production-release
build-runs-on: aks-linux-large
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
target-platform: linux
@@ -55,6 +56,7 @@ jobs:
uses: ./.github/workflows/pipeline-segment-electron-build.yml
needs: checkout-linux
with:
environment: production-release
build-runs-on: aks-linux-large
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
target-platform: linux
@@ -69,6 +71,7 @@ jobs:
uses: ./.github/workflows/pipeline-segment-electron-build.yml
needs: checkout-linux
with:
environment: production-release
build-runs-on: aks-linux-large
build-container: '{"image":"ghcr.io/electron/build:${{ inputs.build-image-sha }}","options":"--user root","volumes":["/mnt/cross-instance-cache:/mnt/cross-instance-cache"]}'
target-platform: linux

View File

@@ -44,6 +44,7 @@ jobs:
uses: ./.github/workflows/pipeline-segment-electron-build.yml
needs: checkout-macos
with:
environment: production-release
build-runs-on: macos-14-xlarge
target-platform: macos
target-arch: x64
@@ -57,6 +58,7 @@ jobs:
uses: ./.github/workflows/pipeline-segment-electron-build.yml
needs: checkout-macos
with:
environment: production-release
build-runs-on: macos-14-xlarge
target-platform: macos
target-arch: arm64

View File

@@ -24,6 +24,11 @@ on:
description: 'JSON container information for aks runs-on'
required: false
default: '{"image":null}'
test-container:
type: string
description: 'JSON container information for testing'
required: false
default: '{"image":null}'
is-release:
description: 'Whether this build job is a release job'
required: true
@@ -47,7 +52,7 @@ on:
concurrency:
group: electron-build-and-test-and-nan-${{ inputs.target-platform }}-${{ inputs.target-arch }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
jobs:
build:
@@ -69,8 +74,7 @@ jobs:
target-arch: ${{ inputs.target-arch }}
target-platform: ${{ inputs.target-platform }}
test-runs-on: ${{ inputs.test-runs-on }}
test-container: ${{ inputs.build-container }}
gn-build-type: ${{ inputs.gn-build-type }}
test-container: ${{ inputs.test-container }}
secrets: inherit
nn-test:
uses: ./.github/workflows/pipeline-segment-node-nan-test.yml
@@ -79,6 +83,6 @@ jobs:
target-arch: ${{ inputs.target-arch }}
target-platform: ${{ inputs.target-platform }}
test-runs-on: ${{ inputs.test-runs-on }}
test-container: ${{ inputs.build-container }}
test-container: ${{ inputs.test-container }}
gn-build-type: ${{ inputs.gn-build-type }}
secrets: inherit

View File

@@ -24,6 +24,11 @@ on:
description: 'JSON container information for aks runs-on'
required: false
default: '{"image":null}'
test-container:
type: string
description: 'JSON container information for testing'
required: false
default: '{"image":null}'
is-release:
description: 'Whether this build job is a release job'
required: true
@@ -47,7 +52,7 @@ on:
concurrency:
group: electron-build-and-test-${{ inputs.target-platform }}-${{ inputs.target-arch }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
jobs:
build:
@@ -64,12 +69,10 @@ jobs:
secrets: inherit
test:
uses: ./.github/workflows/pipeline-segment-electron-test.yml
if: ${{ inputs.target-platform == 'macos' || inputs.target-arch == 'x64' }}
needs: build
with:
target-arch: ${{ inputs.target-arch }}
target-platform: ${{ inputs.target-platform }}
test-runs-on: ${{ inputs.test-runs-on }}
test-container: ${{ inputs.build-container }}
gn-build-type: ${{ inputs.gn-build-type }}
test-container: ${{ inputs.test-container }}
secrets: inherit

View File

@@ -0,0 +1,43 @@
name: Electron Docs Compile
on:
workflow_call:
inputs:
container:
required: true
description: 'Container to run the docs-only ts compile in'
type: string
concurrency:
group: electron-docs-only-${{ github.ref }}
cancel-in-progress: true
jobs:
docs-only:
name: Docs Only Compile
runs-on: aks-linux-medium
timeout-minutes: 20
container: ${{ fromJSON(inputs.container) }}
steps:
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
path: src/electron
fetch-depth: 0
- name: Install Dependencies
run: |
cd src/electron
node script/yarn install
- name: Run TS/JS compile
shell: bash
run: |
cd src/electron
node script/yarn create-typescript-definitions
node script/yarn tsc -p tsconfig.default_app.json --noEmit
for f in build/webpack/*.js
do
out="${f:29}"
if [ "$out" != "base.js" ]; then
node script/yarn webpack --config $f --output-filename=$out --output-path=./.tmp --env mode=development
fi
done

View File

@@ -10,7 +10,7 @@ on:
concurrency:
group: electron-lint-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
jobs:
lint:

View File

@@ -3,6 +3,10 @@ name: Pipeline Segment - Electron Build
on:
workflow_call:
inputs:
environment:
description: using the production or testing environment
required: false
type: string
target-platform:
type: string
description: 'Platform to run on, can be macos, linux or windows'
@@ -44,7 +48,7 @@ on:
concurrency:
group: electron-build-${{ inputs.target-platform }}-${{ inputs.target-arch }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
env:
AZURE_AKS_CACHE_STORAGE_ACCOUNT: ${{ secrets.AZURE_AKS_CACHE_STORAGE_ACCOUNT }}
@@ -64,6 +68,7 @@ jobs:
build:
runs-on: ${{ inputs.build-runs-on }}
container: ${{ fromJSON(inputs.build-container) }}
environment: ${{ inputs.environment }}
env:
TARGET_ARCH: ${{ inputs.target-arch }}
steps:
@@ -96,9 +101,9 @@ jobs:
if: ${{ inputs.target-platform == 'linux' }}
run: |
if [ "${{ inputs.target-arch }}" = "arm" ]; then
GN_EXTRA_ARGS='build_tflite_with_xnnpack=false'
GN_EXTRA_ARGS='target_cpu="arm" build_tflite_with_xnnpack=false'
elif [ "${{ inputs.target-arch }}" = "arm64" ]; then
GN_EXTRA_ARGS='fatal_linker_warnings=false enable_linux_installer=false'
GN_EXTRA_ARGS='target_cpu="arm64" fatal_linker_warnings=false enable_linux_installer=false'
fi
echo "GN_EXTRA_ARGS=$GN_EXTRA_ARGS" >> $GITHUB_ENV
- name: Get Depot Tools
@@ -116,6 +121,9 @@ jobs:
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
if [ "`uname`" = "Linux" ]; then
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
fi
touch .disable_auto_update
- name: Add Depot Tools to PATH
run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH

View File

@@ -20,16 +20,10 @@ on:
description: 'JSON container information for aks runs-on'
required: false
default: '{"image":null}'
gn-build-type:
description: 'The gn build type - testing or release'
required: true
type: string
default: testing
concurrency:
group: electron-test-${{ inputs.target-platform }}-${{ inputs.target-arch }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
env:
ELECTRON_OUT_DIR: Default
ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
@@ -42,16 +36,11 @@ jobs:
fail-fast: false
matrix:
build-type: ${{ inputs.target-platform == 'macos' && fromJSON('["darwin","mas"]') || fromJSON('["linux"]') }}
shard: ${{ inputs.target-platform == 'macos' && fromJSON('[1]') || fromJSON('[1, 2, 3]') }}
env:
BUILD_TYPE: ${{ matrix.build-type }}
TARGET_ARCH: ${{ inputs.target-arch }}
steps:
- name: Load Build Tools
run: |
export BUILD_TOOLS_SHA=2f67e10b9b6b5700b1c7940df412b0345257d9ae
npm i -g @electron/build-tools
e auto-update disable
e init --root=$(pwd) --out=Default ${{ inputs.gn-build-type }} --import ${{ inputs.gn-build-type }} --target-cpu ${{ inputs.target-arch }}
- name: Checkout Electron
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29
with:
@@ -70,6 +59,9 @@ jobs:
sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
else
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
# Remove swift-format dep from cipd on macOS until we send a patch upstream.
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
fi
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
@@ -100,9 +92,6 @@ jobs:
# sudo security authorizationdb write com.apple.trust-settings.admin allow
# cd src/electron
# ./script/codesign/generate-identity.sh
- name: Setup for headless testing
if: ${{ inputs.target-platform == 'linux' }}
run: sh -e /etc/init.d/xvfb start
- name: Run Electron Tests
env:
MOCHA_REPORTER: mocha-multi-reporters
@@ -110,9 +99,21 @@ jobs:
MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap
ELECTRON_DISABLE_SECURITY_WARNINGS: 1
ELECTRON_SKIP_NATIVE_MODULE_TESTS: true
DISPLAY: ':99.0'
run: |
cd src/electron
node script/yarn test --runners=main --trace-uncaught --enable-logging
# Get which tests are on this shard
tests_files=$(node script/split-tests ${{ matrix.shard }} ${{ inputs.target-platform == 'macos' && 1 || 3 }})
# Run tests
if [ "`uname`" = "Darwin" ]; then
node script/yarn test --runners=main --trace-uncaught --enable-logging
else
chown :builduser .. && chmod g+w ..
chown -R :builduser . && chmod -R g+w .
chmod 4755 ../out/Default/chrome-sandbox
runuser -u builduser -- xvfb-run script/actions/run-tests.sh script/yarn test --runners=main --trace-uncaught --enable-logging --files $tests_files
fi
- name: Wait for active SSH sessions
if: always() && !cancelled()
run: |

View File

@@ -28,7 +28,7 @@ on:
concurrency:
group: electron-node-nan-test-${{ inputs.target-platform }}-${{ inputs.target-arch }}-${{ github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.ref != 'refs/heads/main' && !endsWith(github.ref, '-x-y') }}
env:
ELECTRON_OUT_DIR: Default
@@ -37,10 +37,11 @@ env:
jobs:
node-tests:
name: Run Node.js Tests
runs-on: aks-linux-medium
runs-on: aks-linux-medium-plus
timeout-minutes: 20
env:
TARGET_ARCH: ${{ inputs.target-arch }}
BUILD_TYPE: linux
container: ${{ fromJSON(inputs.test-container) }}
steps:
- name: Load Build Tools
@@ -63,6 +64,8 @@ jobs:
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
@@ -71,8 +74,8 @@ jobs:
- name: Download Generated Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
name: generated_artifacts_linux_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_linux_${{ env.TARGET_ARCH }}
name: generated_artifacts_${{ env.BUILD_TYPE }}_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_${{ env.BUILD_TYPE }}_${{ env.TARGET_ARCH }}
- name: Download Src Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
@@ -103,6 +106,7 @@ jobs:
timeout-minutes: 20
env:
TARGET_ARCH: ${{ inputs.target-arch }}
BUILD_TYPE: linux
container: ${{ fromJSON(inputs.test-container) }}
steps:
- name: Load Build Tools
@@ -125,6 +129,8 @@ jobs:
run: |
git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja
cd depot_tools
git apply --3way ../src/electron/.github/workflows/config/gclient.diff
# Ensure depot_tools does not update.
test -d depot_tools && cd depot_tools
touch .disable_auto_update
@@ -133,8 +139,8 @@ jobs:
- name: Download Generated Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:
name: generated_artifacts_linux_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_linux_${{ env.TARGET_ARCH }}
name: generated_artifacts_${{ env.BUILD_TYPE }}_${{ env.TARGET_ARCH }}
path: ./generated_artifacts_${{ env.BUILD_TYPE }}_${{ env.TARGET_ARCH }}
- name: Download Src Artifacts
uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e
with:

View File

@@ -1,18 +1,5 @@
#!/bin/bash
if [ "`uname`" == "Darwin" ]; then
if [ -z "$MAS_BUILD" ]; then
BUILD_TYPE="darwin"
else
BUILD_TYPE="mas"
fi
elif [ "`uname`" == "Linux" ]; then
BUILD_TYPE="linux"
else
echo "Unsupported platform"
exit 1
fi
GENERATED_ARTIFACTS="generated_artifacts_${BUILD_TYPE}_${TARGET_ARCH}"
SRC_ARTIFACTS="src_artifacts_${BUILD_TYPE}_${TARGET_ARCH}"

7
script/actions/run-tests.sh Executable file
View File

@@ -0,0 +1,7 @@
#!/bin/bash
set -euo pipefail
export DISPLAY=:99
Xvfb :99 -screen 0 1024x768x16 -ac &
XVFB_PID=$!
node "$@"

View File

@@ -5,6 +5,7 @@
"parallel/test-child-process-fork-exec-path",
"parallel/test-code-cache",
"parallel/test-cluster-primary-error",
"parallel/test-cluster-primary-kill",
"parallel/test-crypto-aes-wrap",
"parallel/test-crypto-authenticated-stream",
"parallel/test-crypto-des3-wrap",

32
script/split-tests.js Normal file
View File

@@ -0,0 +1,32 @@
const fs = require('node:fs');
const glob = require('glob');
const currentShard = parseInt(process.argv[2], 10);
const shardCount = parseInt(process.argv[3], 10);
const specFiles = glob.sync('spec/*-spec.ts');
const buckets = [];
for (let i = 0; i < shardCount; i++) {
buckets.push([]);
}
const testsInSpecFile = Object.create(null);
for (const specFile of specFiles) {
const testContent = fs.readFileSync(specFile, 'utf8');
testsInSpecFile[specFile] = testContent.split('it(').length;
}
specFiles.sort((a, b) => {
return testsInSpecFile[b] - testsInSpecFile[a];
});
let shard = 0;
for (const specFile of specFiles) {
buckets[shard].push(specFile);
shard++;
if (shard === shardCount) shard = 0;
}
console.log(buckets[currentShard - 1].join(' '));