Merge pull request #18802 from atom/aw/jasmine-junit-xml

Report test result metadata to Azure DevOps

script/package-lock.json

@@ -22,6 +22,11 @@
       "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-1.1.0.tgz",
       "integrity": "sha512-LAQ1d4OPfSJ/BMbI2DuizmYrrkD9JMaTdi2hQTlI53lQ4kRQPyZQRS4CYQ7O66bnBBnP/oYdRxbk++X0xuFU6A=="
     },
+    "@types/node": {
+      "version": "11.9.4",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-11.9.4.tgz",
+      "integrity": "sha512-Zl8dGvAcEmadgs1tmSPcvwzO1YRsz38bVJQvH1RvRqSR9/5n61Q1ktcDL0ht3FXWR+ZpVmXVwN1LuH4Ax23NsA=="
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -670,6 +675,11 @@
       "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-2.11.0.tgz",
       "integrity": "sha1-U0uQM8AiyVecVro7Plpcqvu2UOE="
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24="
+    },
     "boom": {
       "version": "0.4.2",
       "resolved": "https://registry.npmjs.org/boom/-/boom-0.4.2.tgz",
@@ -889,6 +899,19 @@
       "resolved": "https://registry.npmjs.org/character-reference-invalid/-/character-reference-invalid-1.1.2.tgz",
       "integrity": "sha512-7I/xceXfKyUJmSAn/jw8ve/9DyOP7XxufNYLI9Px7CmsKgEUaZLUTax6nZxGQtaoiZCjpu6cHPj20xC/vqRReQ=="
     },
+    "cheerio": {
+      "version": "1.0.0-rc.2",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.2.tgz",
+      "integrity": "sha1-S59TqBsn5NXawxwP/Qz6A8xoMNs=",
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.0",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      }
+    },
     "chownr": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.1.tgz",
@@ -1223,11 +1246,38 @@
       "resolved": "https://registry.npmjs.org/css-parse/-/css-parse-1.7.0.tgz",
       "integrity": "sha1-Mh9s9zeCpv91ERE5D8BeLGV9jJs="
     },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      },
+      "dependencies": {
+        "domutils": {
+          "version": "1.5.1",
+          "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+          "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+          "requires": {
+            "dom-serializer": "0",
+            "domelementtype": "1"
+          }
+        }
+      }
+    },
     "css-value": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/css-value/-/css-value-0.0.1.tgz",
       "integrity": "sha1-Xv1sLupeof1rasV+wEJ7GEUkJOo="
     },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg=="
+    },
     "ctype": {
       "version": "0.5.3",
       "resolved": "https://registry.npmjs.org/ctype/-/ctype-0.5.3.tgz",
@@ -7374,6 +7424,14 @@
         "set-blocking": "~2.0.0"
       }
     },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
     "nugget": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/nugget/-/nugget-2.0.1.tgz",
@@ -7602,6 +7660,14 @@
         "error-ex": "^1.2.0"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "pascalcase": {
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/pascalcase/-/pascalcase-0.1.1.tgz",

script/package.json

@@ -5,6 +5,7 @@
     "7zip-bin": "^4.0.2",
     "async": "2.0.1",
     "babel-core": "5.8.38",
+    "cheerio": "1.0.0-rc.2",
     "coffeelint": "1.15.7",
     "colors": "1.1.2",
     "donna": "1.0.16",

script/postprocess-junit-results

@@ -0,0 +1,87 @@
+#!/usr/bin/env node
+
+const yargs = require('yargs')
+
+const argv = yargs
+  .usage('Usage: $0 [options]')
+  .help('help')
+  .option('search-folder', {
+    string: true,
+    demandOption: true,
+    requiresArg: true,
+    describe: 'Directory to search for JUnit XML results'
+  })
+  .option('test-results-files', {
+    string: true,
+    demandOption: true,
+    requiresArg: true,
+    describe: 'Glob that matches JUnit XML files within searchFolder'
+  })
+  .wrap(yargs.terminalWidth())
+  .argv
+
+const fs = require('fs')
+const path = require('path')
+const glob = require('glob')
+const cheerio = require('cheerio')
+
+function discoverTestFiles() {
+  return new Promise((resolve, reject) => {
+    glob(argv.testResultsFiles, {cwd: argv.searchFolder}, (err, paths) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(paths)
+      }
+    })
+  })
+}
+
+async function postProcessJUnitXML(junitXmlPath) {
+  const fullPath = path.resolve(argv.searchFolder, junitXmlPath)
+  const friendlyName = path.basename(junitXmlPath, '.xml').replace(/-+/g, ' ')
+
+  console.log(`${fullPath}: loading`)
+
+  const original = await new Promise((resolve, reject) => {
+    fs.readFile(fullPath, {encoding: 'utf8'}, (err, content) => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve(content)
+      }
+    })
+  })
+
+  const $ = cheerio.load(original, { xmlMode: true })
+  $('testcase').attr('name', (i, oldName) => `[${friendlyName}] ${oldName}`)
+  const modified = $.xml()
+
+  await new Promise((resolve, reject) => {
+    fs.writeFile(fullPath, modified, {encoding: 'utf8'}, err => {
+      if (err) {
+        reject(err)
+      } else {
+        resolve()
+      }
+    })
+  })
+  console.log(`${fullPath}: complete`)
+}
+
+;(async function() {
+  const testResultFiles = await discoverTestFiles()
+  console.log(`Post-processing ${testResultFiles.length} JUnit XML files`)
+
+  await Promise.all(
+    testResultFiles.map(postProcessJUnitXML)
+  )
+
+  console.log(`${testResultFiles.length} JUnit XML files complete`)
+})().then(
+  () => process.exit(0),
+  err => {
+    console.error(err.stack || err)
+    process.exit(1)
+  }
+)

script/postprocess-junit-results.cmd

@@ -0,0 +1,5 @@
+@IF EXIST "%~dp0\node.exe" (
+  "%~dp0\node.exe"  "%~dp0\postprocess-junit-results" %*
+) ELSE (
+  node  "%~dp0\postprocess-junit-results" %*
+)

script/test

@@ -68,7 +68,8 @@ function prepareEnv (suiteName) {
   if (process.env.TEST_JUNIT_XML_ROOT) {
     // Tell Jasmine to output this suite's results as a JUnit XML file to a subdirectory of the root, so that a
     // CI system can interpret it.
-    const outputPath = path.join(process.env.TEST_JUNIT_XML_ROOT, suiteName, 'test-results.xml')
+    const fileName = suiteName + '.xml'
+    const outputPath = path.join(process.env.TEST_JUNIT_XML_ROOT, fileName)
     env.TEST_JUNIT_XML_PATH = outputPath
   }
 

script/vsts/platforms/linux.yml

@@ -25,9 +25,25 @@ jobs:
       CI: true
       CI_PROVIDER: VSTS
       ATOM_JASMINE_REPORTER: list
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
     displayName: Run tests
     condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))
 
+  - script: script/postprocess-junit-results --search-folder "${TEST_JUNIT_XML_ROOT}" --test-results-files "**/*.xml"
+    env:
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
+    displayName: Post-process test results
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: JUnit
+      searchFolder: $(Common.TestResultsDirectory)/junit
+      testResultsFiles: "**/*.xml"
+      mergeTestResults: true
+      testRunTitle: Linux
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
   - task: PublishBuildArtifacts@1
     inputs:
       PathtoPublish: $(Build.SourcesDirectory)/out/atom.x86_64.rpm

script/vsts/platforms/macos.yml

@@ -45,9 +45,25 @@ jobs:
       CI: true
       CI_PROVIDER: VSTS
       ATOM_JASMINE_REPORTER: list
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
     displayName: Run tests
     condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))
 
+  - script: script/postprocess-junit-results --search-folder "${TEST_JUNIT_XML_ROOT}" --test-results-files "**/*.xml"
+    env:
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)/junit
+    displayName: Post-process test results
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: JUnit
+      searchFolder: $(Common.TestResultsDirectory)/junit
+      testResultsFiles: "**/*.xml"
+      mergeTestResults: true
+      testRunTitle: MacOS
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
   - script: |
       cp $(Build.SourcesDirectory)/out/*.zip $(Build.ArtifactStagingDirectory)
     displayName: Stage Artifacts

script/vsts/platforms/windows.yml

@@ -70,10 +70,28 @@ jobs:
       CI: true
       CI_PROVIDER: VSTS
       ATOM_JASMINE_REPORTER: list
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)\junit
       BUILD_ARCH: $(buildArch)
     displayName: Run tests
     condition: and(succeeded(), ne(variables['Atom.SkipTests'], 'true'))
 
+  - script: >
+      node script\vsts\windows-run.js script\postprocess-junit-results.cmd
+      --search-folder %TEST_JUNIT_XML_ROOT% --test-results-files "**/*.xml"
+    env:
+      TEST_JUNIT_XML_ROOT: $(Common.TestResultsDirectory)\junit
+    displayName: Post-process test results
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
+  - task: PublishTestResults@2
+    inputs:
+      testResultsFormat: JUnit
+      searchFolder: $(Common.TestResultsDirectory)\junit
+      testResultsFiles: "**/*.xml"
+      mergeTestResults: true
+      testRunTitle: Windows $(buildArch)
+    condition: ne(variables['Atom.SkipTests'], 'true')
+
   - task: PublishBuildArtifacts@1
     inputs:
       PathtoPublish: $(Build.SourcesDirectory)/out/atom-x64-windows.zip

spec/jasmine-junit-reporter.js

@@ -0,0 +1,21 @@
+require('jasmine-reporters')
+
+class JasmineJUnitReporter extends jasmine.JUnitXmlReporter {
+  fullDescription (spec) {
+    let fullDescription = spec.description
+    let currentSuite = spec.suite
+    while (currentSuite) {
+      fullDescription = currentSuite.description + ' ' + fullDescription
+      currentSuite = currentSuite.parentSuite
+    }
+
+    return fullDescription
+  }
+
+  reportSpecResults (spec) {
+    spec.description = this.fullDescription(spec)
+    return super.reportSpecResults(spec)
+  }
+}
+
+module.exports = { JasmineJUnitReporter }

spec/jasmine-test-runner.coffee

@@ -10,10 +10,6 @@ module.exports = ({logFile, headless, testPaths, buildAtomEnvironment}) ->
   window[key] = value for key, value of require '../vendor/jasmine'
   require 'jasmine-tagged'
 
-  if process.env.TEST_JUNIT_XML_PATH
-    require 'jasmine-reporters'
-    jasmine.getEnv().addReporter new jasmine.JUnitXmlReporter(process.env.TEST_JUNIT_XML_PATH, true, true)
-
   # Allow document.title to be assigned in specs without screwing up spec window title
   documentTitle = null
   Object.defineProperty document, 'title',
@@ -44,6 +40,15 @@ module.exports = ({logFile, headless, testPaths, buildAtomEnvironment}) ->
   jasmineEnv.addReporter(buildReporter({logFile, headless, resolveWithExitCode}))
   TimeReporter = require './time-reporter'
   jasmineEnv.addReporter(new TimeReporter())
+
+  if process.env.TEST_JUNIT_XML_PATH
+    {JasmineJUnitReporter} = require './jasmine-junit-reporter'
+    process.stdout.write "Outputting JUnit XML to <#{process.env.TEST_JUNIT_XML_PATH}>\n"
+    outputDir = path.dirname(process.env.TEST_JUNIT_XML_PATH)
+    fileBase = path.basename(process.env.TEST_JUNIT_XML_PATH, '.xml')
+
+    jasmineEnv.addReporter new JasmineJUnitReporter(outputDir, true, false, fileBase, true)
+
   jasmineEnv.setIncludedTags([process.platform])
 
   jasmineContent = document.createElement('div')