github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

JUnit support for meteor self-test test runner. (#9014)

Browse Source
This commit is contained in:
Jesse Rosenberger
2017-08-16 18:21:07 +03:00
committed by Ben Newman
parent 34b0237364
commit 5626ad7dee
3 changed files with 189 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
.circleci/config.yml
View File

@@ -17,6 +17,10 @@ run_env_change: &run_env_change
sudo mkdir -p /tmp/core_dumps
sudo chmod a+rwx /tmp/core_dumps
# Make a place for JUnit tests to live.
sudo mkdir -p /tmp/results/junit
sudo chmod -R a+rwx /tmp/results/
# Set the pattern for core dumps, so we can find them.
echo kernel.core_pattern="/tmp/core_dumps/core.%e.%p.%h.%t" | \
sudo tee -a /etc/sysctl.conf
@@ -167,6 +171,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/0.xml \
--with-tag "custom-warehouse"
no_output_timeout: 20m
- run:
@@ -174,6 +179,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -195,6 +204,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/1.xml \
--file '^[a-b]|^c[a-n]|^co[a-l]|^compiler-plugins' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -203,6 +213,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -224,6 +238,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/2.xml \
--file "^co[n-z]|^c[p-z]|^[d-k]" \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -232,6 +247,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -253,6 +272,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/3.xml \
--file '^[l-o]' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -261,6 +281,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -282,6 +306,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/4.xml \
--file '^p' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -290,6 +315,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -311,6 +340,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/5.xml \
--file '^run' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -319,6 +349,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -340,6 +374,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/6.xml \
--file '^r(?!un)|^s' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -348,6 +383,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:
@@ -369,6 +408,7 @@ jobs:
./meteor self-test \
--exclude "${SELF_TEST_EXCLUDE}" \
--headless \
--junit /tmp/results/junit/7.xml \
--file '^[t-z]|^command-line' \
--without-tag "custom-warehouse"
no_output_timeout: 20m
@@ -377,6 +417,10 @@ jobs:
- save_cache:
key: meteor-cache
<<: *meteor_cache_dirs
- store_test_results:
path: /tmp/results
- store_artifacts:
path: /tmp/results
- store_artifacts:
path: /tmp/core_dumps
- store_artifacts:

2
tools/cli/commands.js
View File

@@ -2079,6 +2079,7 @@ main.registerCommand({
'without-tag': { type: String },
// Only run tests with this tag
'with-tag': { type: String },
junit: { type: String },
},
hidden: true,
catalogRefresh: new catalog.Refresh.Never()
@@ -2175,6 +2176,7 @@ main.registerCommand({
// other options
historyLines: options.history,
clients: clients,
junit: options.junit && files.pathResolve(options.junit),
'without-tag': options['without-tag'],
'with-tag': options['with-tag']
});

158
tools/tool-testing/selftest.js
View File

@@ -1,3 +1,4 @@
import { inspect } from 'util';
import { makeFulfillablePromise } from '../utils/fiber-helpers.js';
import { spawn, execFile } from 'child_process';
import * as files from '../fs/files.js';
@@ -1605,6 +1606,7 @@ class Test {
this.fileHash = options.fileHash;
this.tags = options.tags || [];
this.f = options.func;
this.durationMs = null;
this.cleanupHandlers = [];
}
@@ -1797,6 +1799,16 @@ function getFilteredTests(options) {
return new TestList(allTests, tagsToSkip, tagsToMatch, testState);
};
function groupTestsByFile(tests) {
const grouped = {};
tests.forEach(test => {
grouped[test.file] = grouped[test.file] || [];
grouped[test.file].push(test);
});
return grouped;
}
// A TestList is the result of getFilteredTests. It holds the original
// list of all tests, the filtered list, and stats on how many tests
// were skipped (see generateSkipReport).
@@ -1856,8 +1868,120 @@ class TestList {
// Mark a test's file as having failures. This prevents
// saveTestState from saving its hash as a potentially
// "unchanged" file to be skipped in a future run.
notifyFailed(test) {
notifyFailed(test, failureObject) {
// Mark the file that this test lives in as having failures.
this.fileInfo[test.file].hasFailures = true;
// Mark that the specific test failed.
test.failed = true;
// If there is a failure object, store that for potential output.
if (failureObject) {
test.failureObject = failureObject;
}
}
saveJUnitOutput(path) {
const grouped = groupTestsByFile(this.filteredTests);
// We'll form an collection of "testsuites"
const testSuites = [];
const attrSafe = attr => (attr || "").replace('"', "&quot;");
const durationForOutput = durationMs => durationMs / 1000;
// Each file is a testsuite.
Object.keys(grouped).forEach((file) => {
const testCases = [];
let countError = 0;
let countFailure = 0;
// Each test is a "testcase".
grouped[file].forEach((test) => {
const testCaseAttrs = [
`name="${attrSafe(test.name)}"`,
];
if (test.durationMs) {
testCaseAttrs.push(`time="${durationForOutput(test.durationMs)}"`);
}
const testCaseAttrsString = testCaseAttrs.join(' ');
if (test.failed) {
let failureElement = "";
if (test.failureObject instanceof TestFailure) {
countFailure++;
failureElement = [
`<error type="${test.failureObject.reason}">`,
'<![CDATA[',
inspect(test.failureObject.details, { depth: 4 }),
']]>',
'</error>',
].join('\n');
} else if (test.failureObject && test.failureObject.stack) {
countError++;
failureElement = [
'<failure>',
'<![CDATA[',
test.failureObject.stack,
']]>',
'</failure>',
].join('\n');
} else {
countError++;
failureElement = '<failure />';
}
testCases.push(
[
`<testcase ${testCaseAttrsString}>`,
failureElement,
'</testcase>',
].join('\n'),
);
} else {
testCases.push(`<testcase ${testCaseAttrsString}/>`);
}
});
const testSuiteAttrs = [
`name="${file}"`,
`tests="${testCases.length}"`,
`failures="${countFailure}"`,
`errors="${countError}"`,
`time="${durationForOutput(this.durationMs)}"`,
];
const testSuiteAttrsString = testSuiteAttrs.join(' ');
testSuites.push(
[
`<testsuite ${testSuiteAttrsString}>`,
testCases.join('\n'),
'</testsuite>',
].join('\n'),
);
});
const xmlHeader = '<?xml version="1.0" encoding="UTF-8"?>';
const testSuitesString = testSuites.join('\n');
files.writeFile(path,
[
xmlHeader,
`<testsuites>`,
testSuitesString,
`</testsuites>`,
].join('\n'),
'utf8',
);
}
// If this TestList was constructed with a testState,
@@ -1933,17 +2057,11 @@ export function listTests(options) {
return;
}
const testsGroupedByFile = {};
testList.filteredTests.forEach(filteredTest => {
testsGroupedByFile[filteredTest.file] =
testsGroupedByFile[filteredTest.file] || [];
const grouped = groupTestsByFile(testList.filteredTests);
testsGroupedByFile[filteredTest.file].push(filteredTest);
});
Object.keys(testsGroupedByFile).forEach((file) => {
Object.keys(grouped).forEach((file) => {
Console.rawInfo(file + ':\n');
testsGroupedByFile[file].forEach((test) => {
grouped[file].forEach((test) => {
Console.rawInfo(' - ' + test.name +
(test.tags.length ? ' [' + test.tags.join(' ') + ']'
: '') + '\n');
@@ -1971,6 +2089,8 @@ export function runTests(options) {
return 0;
}
testList.startTime = new Date;
let totalRun = 0;
const failedTests = [];
@@ -1980,6 +2100,9 @@ export function runTests(options) {
runTest(test);
});
testList.endTime = new Date;
testList.durationMs = testList.endTime - testList.startTime;
function runTest(test, tries = 3) {
let failure = null;
let startTime;
@@ -1997,6 +2120,8 @@ export function runTests(options) {
test.cleanup();
}
test.durationMs = +(new Date) - startTime;
if (failure) {
Console.error("... fail!", Console.options({ indent: 2 }));
@@ -2009,9 +2134,6 @@ export function runTests(options) {
return runTest(test, tries);
}
failedTests.push(test);
testList.notifyFailed(test);
if (failure instanceof TestFailure) {
const frames = parseStackParse(failure).outsideFiber;
const relpath = files.pathRelative(files.getCurrentToolsDir(),
@@ -2064,16 +2186,22 @@ export function runTests(options) {
} else {
Console.rawError(" => Test threw exception: " + failure.stack + "\n");
}
failedTests.push(test);
testList.notifyFailed(test, failure);
} else {
const durationMs = +(new Date) - startTime;
Console.error(
"... ok (" + durationMs + " ms)",
"... ok (" + test.durationMs + " ms)",
Console.options({ indent: 2 }));
}
}
testList.saveTestState();
if (options.junit) {
testList.saveJUnitOutput(options.junit);
}
if (totalRun > 0) {
Console.error();
}