github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Refactor selftest.runTests and create listTests

Browse Source 
Not extensively tested.  Needs comments describing options to the new functions (e.g. getFilteredTests) and updated usage for `meteor self-test ––list`.

Filtering and running tests now proceeds in stages
- Add “pseudo-tags” like non-matching and unchanged
- Remove tests whose tags are in a list of “tags to skip”
- Run or list the resulting TestList
- Optionally report skipped tests
- Optionally save the testState
This commit is contained in:
David Greenspan
2014-10-06 17:20:39 -07:00
parent ded0601d0a
commit ceda65f24b
2 changed files with 205 additions and 102 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
tools/commands.js
View File

@@ -1708,7 +1708,8 @@ main.registerCommand({
'force-online': { type: Boolean },
slow: { type: Boolean },
browserstack: { type: Boolean },
history: { type: Number }
history: { type: Number },
list: { type: Boolean }
},
hidden: true
}, function (options) {
@@ -1737,6 +1738,17 @@ main.registerCommand({
}
}
if (options.list) {
selftest.listTests({
onlyChanged: options.changed,
offline: offline,
includeSlowTests: options.slow,
testRegexp: testRegexp
});
return 0;
}
var clients = {
browserstack: options.browserstack
};

293
tools/selftest.js
View File

@@ -18,6 +18,7 @@ var webdriver = require('browserstack-webdriver');
var phantomjs = require('phantomjs');
var catalogRemote = require('./catalog-remote.js');
var Package = uniload.load({ packages: ["ejson"] });
var Console = require('./console.js').Console;
var toolPackageName = "meteor-tool";
@@ -1385,10 +1386,8 @@ var define = function (name, tagsList, f) {
tagsList = [];
}
var tags = {};
_.each(tagsList, function (tag) {
tags[tag] = true;
});
var tags = tagsList.slice();
tags.sort();
allTests.push(new Test({
name: name,
@@ -1399,94 +1398,201 @@ var define = function (name, tagsList, f) {
}));
};
///////////////////////////////////////////////////////////////////////////////
// Running tests
// Choosing tests
///////////////////////////////////////////////////////////////////////////////
var tagDescriptions = {
checkout: 'can only run from checkouts',
net: 'require an internet connection',
slow: 'take quite a long time',
// these last two are not actually test tags; they reflect the use of
// --changed and --tests
// these last two are pseudo-tags, assigned to tests when you specify
// --changed or a regex pattern
unchanged: 'unchanged since last pass',
'non-matching': "don't match specified pattern"
};
// options: onlyChanged, offline, includeSlowTests, historyLines, testRegexp
// clients:
// - browserstack (need s3cmd credentials)
var runTests = function (options) {
var failureCount = 0;
var getFilteredTests = function (options) {
options = options || {};
var tests = getAllTests();
var allTests = getAllTests();
if (! tests.length) {
process.stderr.write("No tests defined.\n");
return 0;
if (allTests.length) {
var testState = readTestState();
// Add pseudo-tags 'non-matching' and 'unchanged'
allTests = allTests.map(function (test) {
var newTags = [];
if (options.testRegexp && ! options.testRegexp.test(test.name)) {
newTags.push('non-matching');
} else if (options.onlyChanged &&
test.fileHash === testState.lastPassedHashes[test.file]) {
newTags.push('unchanged');
}
if (! newTags.length) {
return test;
}
return _.extend({}, test, { tags: test.tags.concat(newTags) });
});
}
var testStateFile = path.join(process.env.HOME, '.meteortest');
// (order of tags is significant to the "skip counts" that are displayed)
var tagsToSkip = [];
if (options.testRegexp) {
tagsToSkip.push('non-matching');
}
if (options.onlyChanged) {
tagsToSkip.push('unchanged');
}
if (! files.inCheckout()) {
tagsToSkip.push('checkout');
}
if (options.offline) {
tagsToSkip.push('net');
}
if (! options.includeSlowTests) {
tagsToSkip.push('slow');
}
return new TestList(allTests, tagsToSkip, testState);
};
var TestList = function (allTests, tagsToSkip, testState) {
tagsToSkip = (tagsToSkip || []);
testState = (testState || null);
var self = this;
self.allTests = allTests;
self.skippedTags = tagsToSkip;
self.skipCounts = {};
self.testState = testState;
_.each(tagsToSkip, function (tag) {
self.skipCounts[tag] = 0;
});
self.fileInfo = {}; // path -> {hash, hasSkips, hasFailures}
self.filteredTests = _.filter(allTests, function (test) {
if (! self.fileInfo[test.file]) {
self.fileInfo[test.file] = {
hash: test.fileHash,
hasSkips: false,
hasFailures: false
};
}
var fileInfo = self.fileInfo[test.file];
return !_.any(tagsToSkip, function (tag) {
if (_.contains(test.tags, tag)) {
self.skipCounts[tag]++;
fileInfo.hasSkips = true;
return true;
} else {
return false;
}
});
});
};
TestList.prototype.notifyFailed = function (test) {
this.fileInfo[test.file].hasFailures = true;
};
TestList.prototype.saveTestState = function () {
var self = this;
var testState = self.testState;
if (! (testState && self.filteredTests.length)) {
return;
}
_.each(self.fileInfo, function (info, f) {
if (info.hasFailures) {
delete testState.lastPassedHashes[f];
} else if (! info.hasSkips) {
testState.lastPassedHashes[f] = info.hash;
}
});
writeTestState(testState);
};
TestList.prototype.generateSkipReport = function () {
var self = this;
var result = '';
_.each(self.skippedTags, function (tag) {
var count = self.skipCounts[tag];
if (count) {
result += ("Skipped " + count + " " + tag + " test" +
(count > 1 ? "s" : "") + " (" +
tagDescriptions[tag] + ")\n");
}
});
return result;
};
var getTestStateFilePath = function () {
return path.join(process.env.HOME, '.meteortest');
};
var readTestState = function () {
var testStateFile = getTestStateFilePath();
var testState;
if (fs.existsSync(testStateFile))
testState = JSON.parse(fs.readFileSync(testStateFile, 'utf8'));
if (! testState || testState.version !== 1)
testState = { version: 1, lastPassedHashes: {} };
var currentHashes = {};
return testState;
};
// _.keys(skipCounts) is the set of tags to skip.
// skipCounts also holds counts of tests skipped for other reasons
// (like not matching the test regex) and is used for printing
// messages about how many tests were skipped.
var skipCounts = {};
if (! files.inCheckout())
skipCounts['checkout'] = 0;
var writeTestState = function (testState) {
var testStateFile = getTestStateFilePath();
fs.writeFileSync(testStateFile, JSON.stringify(testState), 'utf8');
};
if (options.offline)
skipCounts['net'] = 0;
var listTests = function (options) {
var testList = getFilteredTests(options);
if (! options.includeSlowTests)
skipCounts['slow'] = 0;
if (options.testRegexp) {
var lengthBeforeTestRegexp = tests.length;
// Filter out tests whose name doesn't match.
tests = _.filter(tests, function (test) {
return options.testRegexp.test(test.name);
});
skipCounts['non-matching'] = lengthBeforeTestRegexp - tests.length;
if (! testList.allTests.length) {
Console.stderr.write("No tests defined.\n");
return;
}
if (options.onlyChanged) {
var lengthBeforeOnlyChanged = tests.length;
// Filter out tests that haven't changed since they last passed.
tests = _.filter(tests, function (test) {
return test.fileHash !== testState.lastPassedHashes[test.file];
});
skipCounts['unchanged'] = lengthBeforeOnlyChanged - tests.length;
_.each(_.sortBy(testList.filteredTests, 'file'), function (test) {
Console.stdout.write(test.file + ': ' + test.name + ' [' +
test.tags.join(' ') + ']');
});
Console.stderr.write(testList.generateSkipReport());
};
///////////////////////////////////////////////////////////////////////////////
// Running tests
///////////////////////////////////////////////////////////////////////////////
// options: onlyChanged, offline, includeSlowTests, historyLines, testRegexp
// clients:
// - browserstack (need s3cmd credentials)
var runTests = function (options) {
var testList = getFilteredTests(options);
if (! testList.allTests.length) {
Console.stderr.write("No tests defined.\n");
return 0;
}
var failuresInFile = {};
var skipsInFile = {};
var totalRun = 0;
_.each(tests, function (test) {
currentHashes[test.file] = test.fileHash;
// Is this a test we're supposed to skip?
var shouldSkip = false;
_.each(_.keys(test.tags), function (tag) {
if (_.has(skipCounts, tag)) {
shouldSkip = true;
skipCounts[tag] ++;
}
});
if (shouldSkip) {
skipsInFile[test.file] = true;
return;
}
var failureCount = 0;
_.each(testList.filteredTests, function (test) {
totalRun++;
process.stderr.write(test.name + "... ");
Console.stderr.write(test.name + "... ");
var failure = null;
try {
@@ -1496,7 +1602,7 @@ var runTests = function (options) {
if (e instanceof TestFailure) {
failure = e;
} else {
process.stderr.write("exception\n\n");
Console.stderr.write("exception\n\n");
throw e;
}
} finally {
@@ -1505,12 +1611,14 @@ var runTests = function (options) {
}
if (failure) {
process.stderr.write("fail!\n");
Console.stderr.write("fail!\n");
failureCount++;
testList.notifyFailed(test);
var frames = parseStack.parse(failure);
var relpath = path.relative(files.getCurrentToolsDir(),
frames[0].file);
process.stderr.write(" => " + failure.reason + " at " +
Console.stderr.write(" => " + failure.reason + " at " +
relpath + ":" + frames[0].line + "\n");
if (failure.reason === 'no-match') {
}
@@ -1519,28 +1627,28 @@ var runTests = function (options) {
return status.signal || ('' + status.code) || "???";
};
process.stderr.write(" => Expected: " + s(failure.details.expected) +
Console.stderr.write(" => Expected: " + s(failure.details.expected) +
"; actual: " + s(failure.details.actual) + "\n");
}
if (failure.reason === 'expected-exception') {
}
if (failure.reason === 'not-equal') {
process.stderr.write(
" => Expected: " + JSON.stringify(failure.details.expected) +
"; actual: " + JSON.stringify(failure.details.actual) + "\n");
Console.stderr.write(
" => Expected: " + JSON.stringify(failure.details.expected) +
"; actual: " + JSON.stringify(failure.details.actual) + "\n");
}
if (failure.details.run) {
failure.details.run.outputLog.end();
var lines = failure.details.run.outputLog.get();
if (! lines.length) {
process.stderr.write(" => No output\n");
Console.stderr.write(" => No output\n");
} else {
var historyLines = options.historyLines || 100;
process.stderr.write(" => Last " + historyLines + " lines:\n");
Console.stderr.write(" => Last " + historyLines + " lines:\n");
_.each(lines.slice(-historyLines), function (line) {
process.stderr.write(" " +
Console.stderr.write(" " +
(line.channel === "stderr" ? "2| " : "1| ") +
line.text +
(line.bare ? "%" : "") + "\n");
@@ -1549,50 +1657,32 @@ var runTests = function (options) {
}
if (failure.details.messages) {
process.stderr.write(" => Errors while building:\n");
process.stderr.write(failure.details.messages.formatMessages());
Console.stderr.write(" => Errors while building:\n");
Console.stderr.write(failure.details.messages.formatMessages());
}
failuresInFile[test.file] = true;
} else {
process.stderr.write("ok\n");
Console.stderr.write("ok\n");
}
});
_.each(_.keys(currentHashes), function (f) {
if (failuresInFile[f])
delete testState.lastPassedHashes[f];
else if (! skipsInFile[f])
testState.lastPassedHashes[f] = currentHashes[f];
});
if (tests.length)
fs.writeFileSync(testStateFile, JSON.stringify(testState), 'utf8');
testList.saveTestState();
if (totalRun > 0)
process.stderr.write("\n");
Console.stderr.write("\n");
var skippedSome = false;
_.each(skipCounts, function (count, tag) {
if (count) {
skippedSome = true;
process.stderr.write("Skipped " + count + " " + tag + " test" +
(count > 1 ? "s" : "") + " (" +
tagDescriptions[tag] + ")\n");
}
});
Console.stderr.write(testList.generateSkipReport());
if (tests.length === 0) {
process.stderr.write("No tests run.\n");
if (testList.filteredTests.length === 0) {
Console.stderr.write("No tests run.\n");
return 0;
} else if (failureCount === 0) {
var disclaimers = '';
if (skippedSome)
if (testList.filteredTests.length < testList.allTests.length)
disclaimers += " other";
process.stderr.write("All" + disclaimers + " tests passed.\n");
Console.stderr.write("All" + disclaimers + " tests passed.\n");
return 0;
} else {
process.stderr.write(failureCount + " failure" +
Console.stderr.write(failureCount + " failure" +
(failureCount > 1 ? "s" : "") + ".\n");
return 1;
}
@@ -1631,6 +1721,7 @@ var runTests = function (options) {
_.extend(exports, {
runTests: runTests,
listTests: listTests,
markStack: markStack,
define: define,
Sandbox: Sandbox,