From ceda65f24be058157b31bde8a7ac8d3ea0bc71fc Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 6 Oct 2014 17:20:39 -0700 Subject: [PATCH] Refactor selftest.runTests and create listTests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Not extensively tested. Needs comments describing options to the new functions (e.g. getFilteredTests) and updated usage for `meteor self-test ––list`. Filtering and running tests now proceeds in stages - Add “pseudo-tags” like non-matching and unchanged - Remove tests whose tags are in a list of “tags to skip” - Run or list the resulting TestList - Optionally report skipped tests - Optionally save the testState --- tools/commands.js | 14 ++- tools/selftest.js | 293 ++++++++++++++++++++++++++++++---------------- 2 files changed, 205 insertions(+), 102 deletions(-) diff --git a/tools/commands.js b/tools/commands.js index 004f9c6302..d65cf193b3 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1708,7 +1708,8 @@ main.registerCommand({ 'force-online': { type: Boolean }, slow: { type: Boolean }, browserstack: { type: Boolean }, - history: { type: Number } + history: { type: Number }, + list: { type: Boolean } }, hidden: true }, function (options) { @@ -1737,6 +1738,17 @@ main.registerCommand({ } } + if (options.list) { + selftest.listTests({ + onlyChanged: options.changed, + offline: offline, + includeSlowTests: options.slow, + testRegexp: testRegexp + }); + + return 0; + } + var clients = { browserstack: options.browserstack }; diff --git a/tools/selftest.js b/tools/selftest.js index 86bc276648..7766274326 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -18,6 +18,7 @@ var webdriver = require('browserstack-webdriver'); var phantomjs = require('phantomjs'); var catalogRemote = require('./catalog-remote.js'); var Package = uniload.load({ packages: ["ejson"] }); +var Console = require('./console.js').Console; var toolPackageName = "meteor-tool"; @@ -1385,10 +1386,8 @@ var define = function (name, tagsList, f) { tagsList = []; } - var tags = {}; - _.each(tagsList, function (tag) { - tags[tag] = true; - }); + var tags = tagsList.slice(); + tags.sort(); allTests.push(new Test({ name: name, @@ -1399,94 +1398,201 @@ var define = function (name, tagsList, f) { })); }; - /////////////////////////////////////////////////////////////////////////////// -// Running tests +// Choosing tests /////////////////////////////////////////////////////////////////////////////// var tagDescriptions = { checkout: 'can only run from checkouts', net: 'require an internet connection', slow: 'take quite a long time', - // these last two are not actually test tags; they reflect the use of - // --changed and --tests + // these last two are pseudo-tags, assigned to tests when you specify + // --changed or a regex pattern unchanged: 'unchanged since last pass', 'non-matching': "don't match specified pattern" }; -// options: onlyChanged, offline, includeSlowTests, historyLines, testRegexp -// clients: -// - browserstack (need s3cmd credentials) -var runTests = function (options) { - var failureCount = 0; +var getFilteredTests = function (options) { + options = options || {}; - var tests = getAllTests(); + var allTests = getAllTests(); - if (! tests.length) { - process.stderr.write("No tests defined.\n"); - return 0; + if (allTests.length) { + var testState = readTestState(); + + // Add pseudo-tags 'non-matching' and 'unchanged' + allTests = allTests.map(function (test) { + var newTags = []; + + if (options.testRegexp && ! options.testRegexp.test(test.name)) { + newTags.push('non-matching'); + } else if (options.onlyChanged && + test.fileHash === testState.lastPassedHashes[test.file]) { + newTags.push('unchanged'); + } + + if (! newTags.length) { + return test; + } + + return _.extend({}, test, { tags: test.tags.concat(newTags) }); + }); } - var testStateFile = path.join(process.env.HOME, '.meteortest'); + // (order of tags is significant to the "skip counts" that are displayed) + var tagsToSkip = []; + if (options.testRegexp) { + tagsToSkip.push('non-matching'); + } + if (options.onlyChanged) { + tagsToSkip.push('unchanged'); + } + if (! files.inCheckout()) { + tagsToSkip.push('checkout'); + } + if (options.offline) { + tagsToSkip.push('net'); + } + if (! options.includeSlowTests) { + tagsToSkip.push('slow'); + } + + return new TestList(allTests, tagsToSkip, testState); +}; + +var TestList = function (allTests, tagsToSkip, testState) { + tagsToSkip = (tagsToSkip || []); + testState = (testState || null); + + var self = this; + self.allTests = allTests; + self.skippedTags = tagsToSkip; + self.skipCounts = {}; + self.testState = testState; + + _.each(tagsToSkip, function (tag) { + self.skipCounts[tag] = 0; + }); + + self.fileInfo = {}; // path -> {hash, hasSkips, hasFailures} + + self.filteredTests = _.filter(allTests, function (test) { + + if (! self.fileInfo[test.file]) { + self.fileInfo[test.file] = { + hash: test.fileHash, + hasSkips: false, + hasFailures: false + }; + } + var fileInfo = self.fileInfo[test.file]; + + return !_.any(tagsToSkip, function (tag) { + if (_.contains(test.tags, tag)) { + self.skipCounts[tag]++; + fileInfo.hasSkips = true; + return true; + } else { + return false; + } + }); + }); +}; + +TestList.prototype.notifyFailed = function (test) { + this.fileInfo[test.file].hasFailures = true; +}; + +TestList.prototype.saveTestState = function () { + var self = this; + var testState = self.testState; + if (! (testState && self.filteredTests.length)) { + return; + } + + _.each(self.fileInfo, function (info, f) { + if (info.hasFailures) { + delete testState.lastPassedHashes[f]; + } else if (! info.hasSkips) { + testState.lastPassedHashes[f] = info.hash; + } + }); + + writeTestState(testState); +}; + +TestList.prototype.generateSkipReport = function () { + var self = this; + var result = ''; + + _.each(self.skippedTags, function (tag) { + var count = self.skipCounts[tag]; + if (count) { + result += ("Skipped " + count + " " + tag + " test" + + (count > 1 ? "s" : "") + " (" + + tagDescriptions[tag] + ")\n"); + } + }); + + return result; +}; + +var getTestStateFilePath = function () { + return path.join(process.env.HOME, '.meteortest'); +}; + +var readTestState = function () { + var testStateFile = getTestStateFilePath(); var testState; if (fs.existsSync(testStateFile)) testState = JSON.parse(fs.readFileSync(testStateFile, 'utf8')); if (! testState || testState.version !== 1) testState = { version: 1, lastPassedHashes: {} }; - var currentHashes = {}; + return testState; +}; - // _.keys(skipCounts) is the set of tags to skip. - // skipCounts also holds counts of tests skipped for other reasons - // (like not matching the test regex) and is used for printing - // messages about how many tests were skipped. - var skipCounts = {}; - if (! files.inCheckout()) - skipCounts['checkout'] = 0; +var writeTestState = function (testState) { + var testStateFile = getTestStateFilePath(); + fs.writeFileSync(testStateFile, JSON.stringify(testState), 'utf8'); +}; - if (options.offline) - skipCounts['net'] = 0; +var listTests = function (options) { + var testList = getFilteredTests(options); - if (! options.includeSlowTests) - skipCounts['slow'] = 0; - - if (options.testRegexp) { - var lengthBeforeTestRegexp = tests.length; - // Filter out tests whose name doesn't match. - tests = _.filter(tests, function (test) { - return options.testRegexp.test(test.name); - }); - skipCounts['non-matching'] = lengthBeforeTestRegexp - tests.length; + if (! testList.allTests.length) { + Console.stderr.write("No tests defined.\n"); + return; } - if (options.onlyChanged) { - var lengthBeforeOnlyChanged = tests.length; - // Filter out tests that haven't changed since they last passed. - tests = _.filter(tests, function (test) { - return test.fileHash !== testState.lastPassedHashes[test.file]; - }); - skipCounts['unchanged'] = lengthBeforeOnlyChanged - tests.length; + _.each(_.sortBy(testList.filteredTests, 'file'), function (test) { + Console.stdout.write(test.file + ': ' + test.name + ' [' + + test.tags.join(' ') + ']'); + }); + + Console.stderr.write(testList.generateSkipReport()); +}; + +/////////////////////////////////////////////////////////////////////////////// +// Running tests +/////////////////////////////////////////////////////////////////////////////// + +// options: onlyChanged, offline, includeSlowTests, historyLines, testRegexp +// clients: +// - browserstack (need s3cmd credentials) +var runTests = function (options) { + var testList = getFilteredTests(options); + + if (! testList.allTests.length) { + Console.stderr.write("No tests defined.\n"); + return 0; } - var failuresInFile = {}; - var skipsInFile = {}; var totalRun = 0; - _.each(tests, function (test) { - currentHashes[test.file] = test.fileHash; - // Is this a test we're supposed to skip? - var shouldSkip = false; - _.each(_.keys(test.tags), function (tag) { - if (_.has(skipCounts, tag)) { - shouldSkip = true; - skipCounts[tag] ++; - } - }); - if (shouldSkip) { - skipsInFile[test.file] = true; - return; - } + var failureCount = 0; + _.each(testList.filteredTests, function (test) { totalRun++; - process.stderr.write(test.name + "... "); + Console.stderr.write(test.name + "... "); var failure = null; try { @@ -1496,7 +1602,7 @@ var runTests = function (options) { if (e instanceof TestFailure) { failure = e; } else { - process.stderr.write("exception\n\n"); + Console.stderr.write("exception\n\n"); throw e; } } finally { @@ -1505,12 +1611,14 @@ var runTests = function (options) { } if (failure) { - process.stderr.write("fail!\n"); + Console.stderr.write("fail!\n"); failureCount++; + testList.notifyFailed(test); + var frames = parseStack.parse(failure); var relpath = path.relative(files.getCurrentToolsDir(), frames[0].file); - process.stderr.write(" => " + failure.reason + " at " + + Console.stderr.write(" => " + failure.reason + " at " + relpath + ":" + frames[0].line + "\n"); if (failure.reason === 'no-match') { } @@ -1519,28 +1627,28 @@ var runTests = function (options) { return status.signal || ('' + status.code) || "???"; }; - process.stderr.write(" => Expected: " + s(failure.details.expected) + + Console.stderr.write(" => Expected: " + s(failure.details.expected) + "; actual: " + s(failure.details.actual) + "\n"); } if (failure.reason === 'expected-exception') { } if (failure.reason === 'not-equal') { - process.stderr.write( -" => Expected: " + JSON.stringify(failure.details.expected) + -"; actual: " + JSON.stringify(failure.details.actual) + "\n"); + Console.stderr.write( + " => Expected: " + JSON.stringify(failure.details.expected) + + "; actual: " + JSON.stringify(failure.details.actual) + "\n"); } if (failure.details.run) { failure.details.run.outputLog.end(); var lines = failure.details.run.outputLog.get(); if (! lines.length) { - process.stderr.write(" => No output\n"); + Console.stderr.write(" => No output\n"); } else { var historyLines = options.historyLines || 100; - process.stderr.write(" => Last " + historyLines + " lines:\n"); + Console.stderr.write(" => Last " + historyLines + " lines:\n"); _.each(lines.slice(-historyLines), function (line) { - process.stderr.write(" " + + Console.stderr.write(" " + (line.channel === "stderr" ? "2| " : "1| ") + line.text + (line.bare ? "%" : "") + "\n"); @@ -1549,50 +1657,32 @@ var runTests = function (options) { } if (failure.details.messages) { - process.stderr.write(" => Errors while building:\n"); - process.stderr.write(failure.details.messages.formatMessages()); + Console.stderr.write(" => Errors while building:\n"); + Console.stderr.write(failure.details.messages.formatMessages()); } - - failuresInFile[test.file] = true; } else { - process.stderr.write("ok\n"); + Console.stderr.write("ok\n"); } }); - _.each(_.keys(currentHashes), function (f) { - if (failuresInFile[f]) - delete testState.lastPassedHashes[f]; - else if (! skipsInFile[f]) - testState.lastPassedHashes[f] = currentHashes[f]; - }); - - if (tests.length) - fs.writeFileSync(testStateFile, JSON.stringify(testState), 'utf8'); + testList.saveTestState(); if (totalRun > 0) - process.stderr.write("\n"); + Console.stderr.write("\n"); - var skippedSome = false; - _.each(skipCounts, function (count, tag) { - if (count) { - skippedSome = true; - process.stderr.write("Skipped " + count + " " + tag + " test" + - (count > 1 ? "s" : "") + " (" + - tagDescriptions[tag] + ")\n"); - } - }); + Console.stderr.write(testList.generateSkipReport()); - if (tests.length === 0) { - process.stderr.write("No tests run.\n"); + if (testList.filteredTests.length === 0) { + Console.stderr.write("No tests run.\n"); return 0; } else if (failureCount === 0) { var disclaimers = ''; - if (skippedSome) + if (testList.filteredTests.length < testList.allTests.length) disclaimers += " other"; - process.stderr.write("All" + disclaimers + " tests passed.\n"); + Console.stderr.write("All" + disclaimers + " tests passed.\n"); return 0; } else { - process.stderr.write(failureCount + " failure" + + Console.stderr.write(failureCount + " failure" + (failureCount > 1 ? "s" : "") + ".\n"); return 1; } @@ -1631,6 +1721,7 @@ var runTests = function (options) { _.extend(exports, { runTests: runTests, + listTests: listTests, markStack: markStack, define: define, Sandbox: Sandbox,