github/electron
1
1
Fork 0
mirror of https://github.com/electron/electron.git synced 2026-04-10 03:01:51 -04:00
Code Issues Packages Projects Releases Wiki Activity

ci: enable goma for all testing builds (#21992) (#22203)

Browse Source 
(cherry picked from commit e7982623ec)
This commit is contained in:
John Kleinschmidt
2020-02-18 12:03:33 -05:00
committed by GitHub
parent 44ee90e5cf
commit 0e9727e8d5
3 changed files with 86 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
.circleci/config.yml
View File

@@ -236,15 +236,6 @@ step-setup-env-for-build: &step-setup-env-for-build
command: |
# To find `gn` executable.
echo 'export CHROMIUM_BUILDTOOLS_PATH="'"$PWD"'/src/buildtools"' >> $BASH_ENV
# Setup Goma if applicable
if [ ! -z "$RAW_GOMA_AUTH" ] && [ "`uname`" == "Linux" ]; then
export USE_SCCACHE="false"
echo 'export USE_SCCACHE=false' >> $BASH_ENV
echo 'export USE_GOMA=true' >> $BASH_ENV
echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
src/electron/external_binaries/goma/goma_ctl.py ensure_start
fi
if [ "$USE_SCCACHE" == "true" ]; then
# https://github.com/mozilla/sccache
@@ -257,6 +248,29 @@ step-setup-env-for-build: &step-setup-env-for-build
fi
fi
step-setup-goma-for-build: &step-setup-goma-for-build
run:
name: Setup Goma
command: |
echo 'export USE_GOMA=true' >> $BASH_ENV
if [ "`uname`" == "Linux" ]; then
echo 'export NUMBER_OF_NINJA_PROCESSES=300' >> $BASH_ENV
else
echo 'export NUMBER_OF_NINJA_PROCESSES=25' >> $BASH_ENV
fi
if [ ! -z "$RAW_GOMA_AUTH" ]; then
echo $RAW_GOMA_AUTH > ~/.goma_oauth2_config
fi
git clone https://github.com/electron/build-tools.git
cd build-tools
npm install
mkdir third_party
node -e "require('./src/utils/goma.js').downloadAndPrepare()"
node -e "require('./src/utils/goma.js').ensure()"
echo 'export GN_GOMA_FILE='`node -e "console.log(require('./src/utils/goma.js').gnFilePath)"` >> $BASH_ENV
echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV
cd ..
step-restore-brew-cache: &step-restore-brew-cache
restore_cache:
paths:
@@ -347,7 +361,7 @@ step-gn-gen-default: &step-gn-gen-default
command: |
cd src
if [ "$USE_GOMA" == "true" ]; then
gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"//electron/build/args/goma.gn\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
gn gen out/Default --args="import(\"$GN_CONFIG\") import(\"$GN_GOMA_FILE\") $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
else
gn gen out/Default --args="import(\"$GN_CONFIG\") cc_wrapper=\"$SCCACHE_PATH\" $GN_EXTRA_ARGS $GN_BUILDFLAG_ARGS"
fi
@@ -616,7 +630,7 @@ step-show-sccache-stats: &step-show-sccache-stats
$SCCACHE_PATH -s
fi
if [ "$USE_GOMA" == "true" ]; then
src/electron/external_binaries/goma/goma_ctl.py stat
$LOCAL_GOMA_DIR/goma_ctl.py stat
fi
step-mksnapshot-build: &step-mksnapshot-build
@@ -986,69 +1000,6 @@ steps-electron-gn-check: &steps-electron-gn-check
- *step-gn-gen-default
- *step-gn-check
steps-electron-build: &steps-electron-build
steps:
- attach_workspace:
at: .
- *step-maybe-early-exit-doc-only-change
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-restore-brew-cache
- *step-get-more-space-on-mac
- *step-install-npm-deps-on-mac
- *step-fix-sync-on-mac
- *step-delete-git-directories
- *step-gn-gen-default
# Electron app
- *step-electron-build
- *step-maybe-electron-dist-strip
- *step-electron-dist-build
- *step-electron-dist-store
- *step-ninja-summary
# Native test targets
- *step-native-unittests-build
- *step-native-unittests-store
# Node.js headers
- *step-nodejs-headers-build
- *step-nodejs-headers-store
- *step-show-sccache-stats
# mksnapshot
- *step-mksnapshot-build
- *step-mksnapshot-store
- *step-maybe-cross-arch-snapshot
- *step-maybe-cross-arch-snapshot-store
# chromedriver
- *step-electron-maybe-chromedriver-gn-gen
- *step-electron-chromedriver-build
- *step-electron-chromedriver-store
# ffmpeg
- *step-ffmpeg-gn-gen
- *step-ffmpeg-build
- *step-ffmpeg-store
# hunspell
- *step-hunspell-build
- *step-hunspell-store
# Save all data needed for a further tests run.
- *step-persist-data-for-tests
- *step-maybe-generate-breakpad-symbols
- *step-maybe-zip-symbols
- *step-symbols-store
# Trigger tests on arm hardware if needed
- *step-maybe-trigger-arm-test
- *step-maybe-notify-slack-failure
steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-change
steps:
# Checkout - Copied ffrom steps-checkout
@@ -1392,10 +1343,10 @@ commands:
steps:
- *step-depot-tools-add-to-path
- *step-setup-env-for-build
- *step-setup-goma-for-build
- *step-get-more-space-on-mac
- *step-fix-sync-on-mac
- *step-delete-git-directories
- *step-gn-gen-default
# Electron app
- when:
@@ -1667,7 +1618,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
@@ -1681,7 +1631,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-disable-run-as-node
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1703,7 +1652,6 @@ jobs:
environment:
<<: *env-linux-medium
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -1712,10 +1660,13 @@ jobs:
environment:
<<: *env-linux-2xlarge-release
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: true
checkout: true
linux-x64-publish:
<<: *machine-linux-2xlarge
@@ -1748,7 +1699,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-ia32
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
@@ -1763,7 +1713,6 @@ jobs:
<<: *env-linux-medium
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -1773,10 +1722,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-ia32
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: true
checkout: true
linux-ia32-publish:
<<: *machine-linux-2xlarge
@@ -1813,7 +1765,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-arm
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
TRIGGER_ARM_TEST: true
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1829,7 +1780,6 @@ jobs:
<<: *env-linux-medium
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -1839,10 +1789,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-arm
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: false
checkout: true
linux-arm-publish:
<<: *machine-linux-2xlarge
@@ -1879,7 +1832,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-arm64
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
TRIGGER_ARM_TEST: true
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
@@ -1903,7 +1855,6 @@ jobs:
<<: *env-linux-medium
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -1913,10 +1864,13 @@ jobs:
<<: *env-linux-2xlarge-release
<<: *env-arm64
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- electron-build:
persist: false
checkout: true
linux-arm64-publish:
<<: *machine-linux-2xlarge
@@ -1950,7 +1904,6 @@ jobs:
environment:
<<: *env-mac-large
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-macos-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@@ -1973,7 +1926,6 @@ jobs:
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -1982,9 +1934,14 @@ jobs:
environment:
<<: *env-mac-large
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- electron-build:
persist: true
checkout: false
checkout-and-assume-cache: true
attach: false
osx-publish:
<<: *machine-mac-large
@@ -2017,7 +1974,6 @@ jobs:
<<: *env-mac-large
<<: *env-mas
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *env-macos-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
@@ -2041,7 +1997,6 @@ jobs:
environment:
<<: *env-machine-mac
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-send-slack-notifications
<<: *steps-chromedriver-build
@@ -2051,10 +2006,15 @@ jobs:
<<: *env-mac-large
<<: *env-mas
<<: *env-release-build
<<: *env-enable-sccache
<<: *env-ninja-status
<<: *steps-electron-build
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- electron-build:
persist: true
checkout: false
checkout-and-assume-cache: true
attach: false
mas-publish:
<<: *machine-mac-large
environment:
@@ -2088,7 +2048,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
@@ -2097,7 +2056,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--only-disabled-tests'
<<: *steps-native-tests
@@ -2107,7 +2065,6 @@ jobs:
environment:
<<: *env-linux-2xlarge
<<: *env-unittests
<<: *env-enable-sccache
<<: *env-headless-testing
TESTS_ARGS: '--include-disabled-tests'
<<: *steps-native-tests
@@ -2118,7 +2075,6 @@ jobs:
<<: *env-linux-2xlarge
<<: *env-browsertests
<<: *env-testing-build
<<: *env-enable-sccache
<<: *env-headless-testing
<<: *steps-native-tests
@@ -2528,10 +2484,9 @@ workflows:
- *chromium-upgrade-branches
jobs:
- linux-checkout-fast
- linux-checkout-and-save-cache
- linux-x64-release:
requires:
- linux-checkout-fast
- linux-x64-release
- linux-x64-release-tests:
requires:
- linux-x64-release
@@ -2555,9 +2510,7 @@ workflows:
- linux-x64-verify-ffmpeg
- linux-x64-chromedriver
- linux-ia32-release:
requires:
- linux-checkout-fast
- linux-ia32-release
- linux-ia32-release-tests:
requires:
- linux-ia32-release
@@ -2581,9 +2534,7 @@ workflows:
- linux-ia32-verify-ffmpeg
- linux-ia32-chromedriver
- linux-arm-release:
requires:
- linux-checkout-fast
- linux-arm-release
- linux-arm-chromedriver:
requires:
- linux-checkout-fast
@@ -2592,10 +2543,7 @@ workflows:
- linux-arm-release
- linux-arm-chromedriver
- linux-arm64-release:
requires:
- linux-checkout-fast
- linux-arm64-release
- linux-arm64-chromedriver:
requires:
- linux-checkout-fast
@@ -2615,10 +2563,11 @@ workflows:
- *chromium-upgrade-branches
jobs:
- mac-checkout-fast
- mac-checkout-and-save-cache
- osx-release:
requires:
- mac-checkout-fast
- mac-checkout-and-save-cache
- osx-release-tests:
requires:
- osx-release
@@ -2644,7 +2593,7 @@ workflows:
- mas-release:
requires:
- mac-checkout-fast
- mac-checkout-and-save-cache
- mas-release-tests:
requires:
- mas-release

24
appveyor.yml
View File

@@ -118,19 +118,29 @@ build_script:
}
}
- ps: >-
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
.\src\electron\script\start-goma.ps1 -gomaDir "$pwd\src\electron\external_binaries\goma"
if ($env:GN_CONFIG -ne 'release') {
if (Test-Path 'env:RAW_GOMA_AUTH') {
$env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
$env:RAW_GOMA_AUTH | Set-Content $env:GOMA_OAUTH2_CONFIG_FILE
}
git clone https://github.com/electron/build-tools.git
cd build-tools
npm install
mkdir third_party
node -e "require('./src/utils/goma.js').downloadAndPrepare()"
$env:GN_GOMA_FILE = node -e "console.log(require('./src/utils/goma.js').gnFilePath)"
$env:LOCAL_GOMA_DIR = node -e "console.log(require('./src/utils/goma.js').dir)"
cd ..
.\src\electron\script\start-goma.ps1 -gomaDir $env:LOCAL_GOMA_DIR
}
- cd src
- set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn
- if DEFINED RAW_GOMA_AUTH (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"//electron/build/args/goma.gn\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
- if DEFINED GN_GOMA_FILE (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% ") else (gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") %GN_EXTRA_ARGS% cc_wrapper=\"%SCCACHE_PATH%\"")
- gn check out/Default //electron:electron_lib
- gn check out/Default //electron:electron_app
- gn check out/Default //electron:manifests
- gn check out/Default //electron/shell/common/api:mojo
- if DEFINED RAW_GOMA_AUTH (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if DEFINED GN_GOMA_FILE (ninja -j 300 -C out/Default electron:electron_app) else (ninja -C out/Default electron:electron_app)
- if "%GN_CONFIG%"=="testing" ( python C:\depot_tools\post_build_ninja_summary.py -C out\Default )
- gn gen out/ffmpeg "--args=import(\"//electron/build/args/ffmpeg.gn\") %GN_EXTRA_ARGS%"
- ninja -C out/ffmpeg electron:electron_ffmpeg_zip
@@ -140,7 +150,7 @@ build_script:
- ninja -C out/Default electron:hunspell_dictionaries_zip
- ninja -C out/Default electron:electron_chromedriver_zip
- ninja -C out/Default third_party/electron_node:headers
- if DEFINED RAW_GOMA_AUTH (python electron\external_binaries\goma\goma_ctl.py stat)
- if "%GN_CONFIG%"=="testing" ( python %LOCAL_GOMA_DIR%\goma_ctl.py stat )
- python electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
- appveyor PushArtifact out/Default/windows_toolchain_profile.json
- appveyor PushArtifact out/Default/dist.zip

1
script/start-goma.ps1
View File

@@ -1,7 +1,6 @@
param([string]$gomaDir=$PWD)
$cmdPath = Join-Path -Path $gomaDir -ChildPath "goma_ctl.py"
Start-Process -FilePath cmd -ArgumentList "/C", "python", "$cmdPath", "ensure_start"
$env:GLOG_logtostderr = "true"
$timedOut = $false; $waitTime = 0; $waitIncrement = 5; $timeout=120;
Do { sleep $waitIncrement; $timedOut = (($waitTime+=$waitIncrement) -gt $timeout); iex "$gomaDir\gomacc.exe port 2" > $null; } Until(($LASTEXITCODE -eq 0) -or $timedOut)
if ($timedOut) {