From 0d00538da3e23d28521c71d1b1b1e1bdbf422f05 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 3 Jun 2019 11:12:50 +0200 Subject: [PATCH 1/5] Run core and package tests in parallel for macOS on Azure Pipelines --- script/test | 8 +- script/vsts/nightly-release.yml | 2 +- script/vsts/platforms/macos.yml | 108 ++++++++++++++++++++------- script/vsts/release-branch-build.yml | 2 +- 4 files changed, 88 insertions(+), 32 deletions(-) diff --git a/script/test b/script/test index adb8a5123..ab9daed6d 100755 --- a/script/test +++ b/script/test @@ -201,7 +201,13 @@ function testSuitesForPlatform (platform) { let suites = [] switch (platform) { case 'darwin': - suites = [runCoreMainProcessTests, runCoreRenderProcessTests, runBenchmarkTests].concat(packageTestSuites) + if (process.env.ATOM_RUN_CORE_TESTS === 'true') { + suites = [runCoreMainProcessTests, runCoreRenderProcessTests, runBenchmarkTests] + } else if (process.env.ATOM_RUN_PACKAGE_TESTS === 'true') { + suites = packageTestSuites + } else { + suites = [runCoreMainProcessTests, runCoreRenderProcessTests, runBenchmarkTests].concat(packageTestSuites) + } break case 'win32': suites = (process.arch === 'x64') ? [runCoreMainProcessTests, runCoreRenderProcessTests] : [runCoreMainProcessTests] diff --git a/script/vsts/nightly-release.yml b/script/vsts/nightly-release.yml index ac6537bb9..2bacc55a4 100644 --- a/script/vsts/nightly-release.yml +++ b/script/vsts/nightly-release.yml @@ -29,7 +29,7 @@ jobs: - GetReleaseVersion - Windows - Linux - - macOS + - macOS_tests variables: ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ] diff --git a/script/vsts/platforms/macos.yml b/script/vsts/platforms/macos.yml index 223d15d5f..f2f568e79 100644 --- a/script/vsts/platforms/macos.yml +++ b/script/vsts/platforms/macos.yml @@ -1,5 +1,6 @@ jobs: -- job: macOS +- job: macOS_build + displayName: macOS build dependsOn: GetReleaseVersion timeoutInMinutes: 180 @@ -61,6 +62,81 @@ jobs: ATOM_MAC_CODE_SIGNING_KEYCHAIN: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN) ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD) + - script: | + cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory) + displayName: Stage Artifacts + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/atom-mac.zip + ArtifactName: atom-mac.zip + ArtifactType: Container + displayName: Upload atom-mac.zip + condition: succeeded() + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/atom-mac-symbols.zip + ArtifactName: atom-mac-symbols.zip + ArtifactType: Container + displayName: Upload atom-mac-symbols.zip + condition: succeeded() + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.SourcesDirectory)/docs/output/atom-api.json + ArtifactName: atom-api.json + ArtifactType: Container + displayName: Upload atom-api.json + condition: succeeded() + +- job: macOS_tests + displayName: macOS test + dependsOn: macOS_build + timeoutInMinutes: 180 + pool: + vmImage: macos-10.13 + strategy: + maxParallel: 2 + matrix: + core: + RunCoreTests: true + RunPackageTests: false + packages: + RunCoreTests: false + RunPackageTests: true + + steps: + - task: NodeTool@0 + inputs: + versionSpec: 8.9.3 + displayName: Install Node.js 8.9.3 + + - task: 1ESLighthouseEng.PipelineArtifactCaching.RestoreCacheV1.RestoreCache@1 + displayName: Restore node_modules cache + inputs: + keyfile: 'package.json, **/package-lock.json, !**/node_modules/**/package-lock.json, !**/.*/**/package-lock.json' + targetfolder: '**/node_modules, !**/node_modules/**/node_modules' + vstsFeed: 'bae1bc26-220d-43c7-a955-4de039370de2' + + - task: DownloadBuildArtifacts@0 + displayName: Download atom-mac.zip + inputs: + artifactName: 'atom-mac.zip' + downloadPath: $(Build.SourcesDirectory) + + - script: unzip atom-mac.zip/atom-mac.zip -d out + displayName: Unzip atom-mac.zip + + - task: DownloadBuildArtifacts@0 + displayName: Download atom-mac-symbols.zip + inputs: + artifactName: 'atom-mac-symbols.zip' + downloadPath: $(Build.SourcesDirectory) + + - script: unzip atom-mac-symbols.zip/atom-mac-symbols.zip -d out + displayName: Unzip atom-mac-symbols.zip + - script: | osascript -e 'tell application "System Events" to keystroke "x"' # clear screen saver caffeinate -s script/test # Run with caffeinate to prevent screen saver @@ -69,6 +145,8 @@ jobs: CI_PROVIDER: VSTS ATOM_JASMINE_REPORTER: list TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit + ATOM_RUN_CORE_TESTS: $(RunCoreTests) + ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests) displayName: Run tests condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true')) @@ -87,40 +165,12 @@ jobs: testRunTitle: MacOS condition: ne(variables['Atom.SkipTests'], 'true') - - script: | - cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory) - displayName: Stage Artifacts - - script: | mkdir -p $(Build.ArtifactStagingDirectory)/crash-reports cp ${HOME}/Library/Logs/DiagnosticReports/*.crash $(Build.ArtifactStagingDirectory)/crash-reports displayName: Stage Crash Reports condition: failed() - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/atom-mac.zip - ArtifactName: atom-mac.zip - ArtifactType: Container - displayName: Upload atom-mac.zip - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.ArtifactStagingDirectory)/atom-mac-symbols.zip - ArtifactName: atom-mac-symbols.zip - ArtifactType: Container - displayName: Upload atom-mac-symbols.zip - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - - task: PublishBuildArtifacts@1 - inputs: - PathtoPublish: $(Build.SourcesDirectory)/docs/output/atom-api.json - ArtifactName: atom-api.json - ArtifactType: Container - displayName: Upload atom-api.json - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - - task: PublishBuildArtifacts@1 inputs: PathtoPublish: $(Build.ArtifactStagingDirectory)/crash-reports diff --git a/script/vsts/release-branch-build.yml b/script/vsts/release-branch-build.yml index ab600d80d..8333c3d7a 100644 --- a/script/vsts/release-branch-build.yml +++ b/script/vsts/release-branch-build.yml @@ -34,7 +34,7 @@ jobs: - GetReleaseVersion - Windows - Linux - - macOS + - macOS_tests variables: ReleaseVersion: $[ dependencies.GetReleaseVersion.outputs['Version.ReleaseVersion'] ] From 9d2494ca167362f1234f4a005bdf4d5b5f183bfc Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Mon, 3 Jun 2019 11:12:50 +0200 Subject: [PATCH 2/5] Don't run benchmark tests during script/test unless explicitly requested --- script/test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/test b/script/test index ab9daed6d..720c274f6 100755 --- a/script/test +++ b/script/test @@ -202,11 +202,11 @@ function testSuitesForPlatform (platform) { switch (platform) { case 'darwin': if (process.env.ATOM_RUN_CORE_TESTS === 'true') { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests, runBenchmarkTests] + suites = [runCoreMainProcessTests, runCoreRenderProcessTests] } else if (process.env.ATOM_RUN_PACKAGE_TESTS === 'true') { suites = packageTestSuites } else { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests, runBenchmarkTests].concat(packageTestSuites) + suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) } break case 'win32': From 791a5838601ee1003f9838a1c5d9709b7de29407 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 5 Jun 2019 10:05:19 +0200 Subject: [PATCH 3/5] Run package tests in parallel --- script/test | 6 ++++-- script/vsts/platforms/macos.yml | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/script/test b/script/test index 720c274f6..8f29a0c37 100755 --- a/script/test +++ b/script/test @@ -203,8 +203,10 @@ function testSuitesForPlatform (platform) { case 'darwin': if (process.env.ATOM_RUN_CORE_TESTS === 'true') { suites = [runCoreMainProcessTests, runCoreRenderProcessTests] - } else if (process.env.ATOM_RUN_PACKAGE_TESTS === 'true') { - suites = packageTestSuites + } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { + suites = packageTestSuites.slice(0, Math.floor(packageTestSuites.length / 2)) + } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { + suites = packageTestSuites.slice(Math.floor(packageTestSuites.length / 2)) } else { suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) } diff --git a/script/vsts/platforms/macos.yml b/script/vsts/platforms/macos.yml index f2f568e79..253b22203 100644 --- a/script/vsts/platforms/macos.yml +++ b/script/vsts/platforms/macos.yml @@ -97,14 +97,17 @@ jobs: pool: vmImage: macos-10.13 strategy: - maxParallel: 2 + maxParallel: 3 matrix: core: RunCoreTests: true RunPackageTests: false - packages: + packages-1: RunCoreTests: false - RunPackageTests: true + RunPackageTests: 1 + packages-2: + RunCoreTests: false + RunPackageTests: 2 steps: - task: NodeTool@0 From 2b0e5b0c32483914e952c94455bb15b0f9628ee5 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 5 Jun 2019 11:46:40 +0200 Subject: [PATCH 4/5] Split package tests more evenly across containers --- script/test | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/script/test b/script/test index 8f29a0c37..4383a67df 100755 --- a/script/test +++ b/script/test @@ -116,10 +116,7 @@ for (let packageName in CONFIG.appMetadata.packageDependencies) { const testSubdir = ['spec', 'test'].find(subdir => fs.existsSync(path.join(repositoryPackagePath, subdir))) if (!testSubdir) { - packageTestSuites.push(function (callback) { - console.log(`Skipping tests for ${packageName} because no test folder was found`.bold.yellow) - callback(null, {exitCode: 0, step: `package-${packageName}`}) - }) + console.log(`No test folder found for package: ${packageName}`.yellow) continue } @@ -201,12 +198,14 @@ function testSuitesForPlatform (platform) { let suites = [] switch (platform) { case 'darwin': + const PACKAGES_TO_TEST_IN_PARALLEL = 23 + if (process.env.ATOM_RUN_CORE_TESTS === 'true') { suites = [runCoreMainProcessTests, runCoreRenderProcessTests] } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { - suites = packageTestSuites.slice(0, Math.floor(packageTestSuites.length / 2)) + suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { - suites = packageTestSuites.slice(Math.floor(packageTestSuites.length / 2)) + suites = packageTestSuites.slice(PACKAGES_TO_TEST_IN_PARALLEL) } else { suites = [runCoreMainProcessTests, runCoreRenderProcessTests].concat(packageTestSuites) } From 587a4dd72d2c5a918d522bb5f3b30f413973df16 Mon Sep 17 00:00:00 2001 From: Antonio Scandurra Date: Wed, 5 Jun 2019 14:10:01 +0200 Subject: [PATCH 5/5] Run main process tests during build step For some reason, one main process test fails when run in a different container than the one in which Atom was built. These tests are pretty quick, so their impact on parallelism should be negligible. --- script/test | 6 +++-- script/vsts/platforms/macos.yml | 48 ++++++++++++++++++++++++++++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/script/test b/script/test index 4383a67df..882cc9a33 100755 --- a/script/test +++ b/script/test @@ -200,8 +200,10 @@ function testSuitesForPlatform (platform) { case 'darwin': const PACKAGES_TO_TEST_IN_PARALLEL = 23 - if (process.env.ATOM_RUN_CORE_TESTS === 'true') { - suites = [runCoreMainProcessTests, runCoreRenderProcessTests] + if (process.env.ATOM_RUN_CORE_MAIN_TESTS === 'true') { + suites = [runCoreMainProcessTests] + } else if (process.env.ATOM_RUN_CORE_RENDERER_TESTS === 'true') { + suites = [runCoreRenderProcessTests] } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '1') { suites = packageTestSuites.slice(0, PACKAGES_TO_TEST_IN_PARALLEL) } else if (process.env.ATOM_RUN_PACKAGE_TESTS === '2') { diff --git a/script/vsts/platforms/macos.yml b/script/vsts/platforms/macos.yml index 253b22203..af93ce1d5 100644 --- a/script/vsts/platforms/macos.yml +++ b/script/vsts/platforms/macos.yml @@ -62,6 +62,46 @@ jobs: ATOM_MAC_CODE_SIGNING_KEYCHAIN: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN) ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD: $(ATOM_MAC_CODE_SIGNING_KEYCHAIN_PASSWORD) + - script: | + osascript -e 'tell application "System Events" to keystroke "x"' # clear screen saver + caffeinate -s script/test # Run with caffeinate to prevent screen saver + env: + CI: true + CI_PROVIDER: VSTS + ATOM_JASMINE_REPORTER: list + TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit + ATOM_RUN_CORE_MAIN_TESTS: true + displayName: Run main process tests + condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true')) + + - script: script/postprocess-junit-results --search-folder "${TEST_JUNIT_XML_ROOT}" --test-results-files "**/*.xml" + env: + TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit + displayName: Post-process test results + condition: ne(variables['Atom.SkipTests'], 'true') + + - task: PublishTestResults@2 + inputs: + testResultsFormat: JUnit + searchFolder: $(Common.TestResultsDirectory)/junit + testResultsFiles: "**/*.xml" + mergeTestResults: true + testRunTitle: MacOS + condition: ne(variables['Atom.SkipTests'], 'true') + + - script: | + mkdir -p $(Build.ArtifactStagingDirectory)/crash-reports + cp ${HOME}/Library/Logs/DiagnosticReports/*.crash $(Build.ArtifactStagingDirectory)/crash-reports + displayName: Stage Crash Reports + condition: failed() + + - task: PublishBuildArtifacts@1 + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/crash-reports + ArtifactName: crash-reports.zip + displayName: Upload Crash Reports + condition: failed() + - script: | cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory) displayName: Stage Artifacts @@ -100,13 +140,13 @@ jobs: maxParallel: 3 matrix: core: - RunCoreTests: true + RunCoreRendererTests: true RunPackageTests: false packages-1: - RunCoreTests: false + RunCoreRendererTests: false RunPackageTests: 1 packages-2: - RunCoreTests: false + RunCoreRendererTests: false RunPackageTests: 2 steps: @@ -148,7 +188,7 @@ jobs: CI_PROVIDER: VSTS ATOM_JASMINE_REPORTER: list TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit - ATOM_RUN_CORE_TESTS: $(RunCoreTests) + ATOM_RUN_CORE_RENDERER_TESTS: $(RunCoreRendererTests) ATOM_RUN_PACKAGE_TESTS: $(RunPackageTests) displayName: Run tests condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))