Faster Jasmine tests, Sauce Labs settings update / trying to fix Sauce Labs issues for specific browsers

This commit is contained in:
Matthew Dean
2016-07-16 20:45:12 -07:00
parent ec30272425
commit 54affb852b
4 changed files with 270 additions and 142 deletions

View File

@@ -7,6 +7,174 @@ module.exports = function (grunt) {
var COMPRESS_FOR_TESTS = true;
// Sauce Labs browser
var browsers = [
// Desktop browsers
{
browserName: "chrome",
version: 'latest',
platform: 'Windows 7'
},
{
browserName: "firefox",
version: 'latest',
platform: 'Linux'
},
{
browserName: 'safari',
version: '9',
platform: 'OS X 10.11'
},
{
browserName: 'safari',
version: '8',
platform: 'OS X 10.10'
},
{
browserName: "internet explorer",
version: '8',
platform: 'Windows XP'
},
{
browserName: "internet explorer",
version: '9',
platform: 'Windows 7'
},
{
browserName: "internet explorer",
version: '11',
platform: 'Windows 8.1'
},
{
browserName: "edge",
version: '13',
platform: 'Windows 10'
},
// Mobile browsers
{
browserName: "ipad",
version: '8.0',
platform: 'OS X 10.9',
'device-orientation': 'portrait'
},
{
browserName: 'iphone',
version: '7.1',
platform: 'OS X 10.9'
},
{
browserName: 'iphone',
version: '9.3',
platform: 'OS X 10.10'
},
{
browerName: 'android',
version: '4.2',
platform: 'Linux'
}
];
var browsers = [
// Desktop browsers
{
browserName: "chrome",
version: 'latest',
platform: 'Windows 7'
},
{
browserName: "firefox",
version: 'latest',
platform: 'Linux'
},
{
browserName: 'safari',
version: '9',
platform: 'OS X 10.11'
},
{
browserName: 'safari',
version: '8',
platform: 'OS X 10.10'
},
{
browserName: "internet explorer",
version: '8',
platform: 'Windows XP'
},
{
browserName: "internet explorer",
version: '9',
platform: 'Windows 7'
},
{
browserName: "internet explorer",
version: '11',
platform: 'Windows 8.1'
},
{
browserName: "edge",
version: '13',
platform: 'Windows 10'
},
// Mobile browsers
{
browserName: "ipad",
version: '8.0',
platform: 'OS X 10.9',
'device-orientation': 'portrait'
},
{
browserName: 'iphone',
version: '7.1',
platform: 'OS X 10.9'
},
{
browserName: 'iphone',
version: '9.3',
platform: 'OS X 10.10'
},
{
browerName: 'android',
version: '4.2',
platform: 'Linux'
}
];
// var sauceJobs = {};
// [ "main",
// "filemanager-plugin",
// "visitor-plugin",
// "pre-processor-plugin",
// "post-processor-plugin",
// "global-vars",
// "modify-vars",
// "production",
// "rootpath-relative",
// "rootpath",
// "relative-urls",
// "browser",
// "no-js-errors",
// "legacy",
// "strict-units"
// ].map(function(testName) {
// sauceJobs[testName] = {
// options: {
// urls: ["http://localhost:8081/tmp/browser/test-runner-" + testName + ".html"],
// testname: 'Less.js test - ' + testName,
// browsers: browsers,
// public: 'public',
// recordVideo: false,
// videoUploadOnPass: false,
// recordScreenshots: process.env.TRAVIS_BRANCH !== "master",
// build: process.env.TRAVIS_BRANCH === "master" ? process.env.TRAVIS_JOB_ID : undefined,
// tags: [process.env.TRAVIS_BUILD_NUMBER, process.env.TRAVIS_PULL_REQUEST, process.env.TRAVIS_BRANCH],
// sauceConfig: {
// 'idle-timeout': 100
// },
// throttled: 5
// }
// };
// });
// Project configuration.
grunt.initConfig({
@@ -278,89 +446,56 @@ module.exports = function (grunt) {
'saucelabs-jasmine': {
all: {
options: {
urls: ["filemanager-plugin","visitor-plugin","pre-processor-plugin","post-processor-plugin","global-vars", "modify-vars", "production", "rootpath-relative",
urls: ["main", "filemanager-plugin","visitor-plugin","pre-processor-plugin","post-processor-plugin","global-vars", "modify-vars", "production", "rootpath-relative",
"rootpath", "relative-urls", "browser", "no-js-errors", "legacy", "strict-units"
].map(function(testName) {
return "http://localhost:8081/tmp/browser/test-runner-" + testName + ".html";
}),
testname: 'Sauce Unit Test for less.js',
browsers: [
// Desktop browsers
{
browserName: "chrome",
version: 'latest',
platform: 'Windows 7'
},
{
browserName: "firefox",
version: 'latest',
platform: 'Linux'
},
{
browserName: 'safari',
version: '9.1',
platform: 'OS X 10.10'
},
{
browserName: 'safari',
version: '8',
platform: 'OS X 10.10'
},
{
browserName: "internet explorer",
version: '8',
platform: 'Windows XP'
},
{
browserName: "internet explorer",
version: '9',
platform: 'Windows 7'
},
{
browserName: "internet explorer",
version: '11',
platform: 'Windows 8.1'
},
{
browserName: "edge",
version: '13',
platform: 'Windows 10'
},
// Mobile browsers
{
browserName: "ipad",
version: '8.0',
platform: 'OS X 10.9',
'device-orientation': 'portrait'
},
{
browserName: 'iphone',
version: '7.1',
platform: 'OS X 10.9'
},
{
browserName: 'iphone',
version: '9.3',
platform: 'OS X 10.10'
},
{
browerName: 'android',
version: '4.2',
platform: 'Linux'
}
],
browsers: browsers,
public: 'public',
concurrency: 3,
pollInterval: 2000,
statusCheckAttempts: 30,
recordVideo: false,
videoUploadOnPass: false,
recordScreenshots: process.env.TRAVIS_BRANCH !== "master",
build: process.env.TRAVIS_BRANCH === "master" ? process.env.TRAVIS_JOB_ID : undefined,
tags: [process.env.TRAVIS_BUILD_NUMBER, process.env.TRAVIS_PULL_REQUEST, process.env.TRAVIS_BRANCH],
'max-duration': 120,
sauceConfig: {
'idle-timeout': 100
},
throttled: 3
throttled: 5,
onTestComplete: function(result, callback) {
// Called after a unit test is done, per page, per browser
// 'result' param is the object returned by the test framework's reporter
// 'callback' is a Node.js style callback function. You must invoke it after you
// finish your work.
// Pass a non-null value as the callback's first parameter if you want to throw an
// exception. If your function is synchronous you can also throw exceptions
// directly.
// Passing true or false as the callback's second parameter passes or fails the
// test. Passing undefined does not alter the test result. Please note that this
// only affects the grunt task's result. You have to explicitly update the Sauce
// Labs job's status via its REST API, if you want so.
// This should be the encrypted value in Travis
var user = process.env.SAUCE_USERNAME;
var pass = process.env.SAUCE_ACCESS_KEY;
require('request').put({
url: ['https://saucelabs.com/rest/v1', user, 'jobs', result.job_id].join('/'),
auth: { user: user, pass: pass },
json: { passed: result.passed }
}, function (error, response, body) {
if (error) {
callback(error);
} else if (response.statusCode !== 200) {
callback(new Error('Unexpected response status'));
} else {
callback(null, result.passed);
}
});
}
}
}
},

View File

@@ -64,6 +64,8 @@
"jit-grunt": "^0.10.0",
"performance-now": "^0.2.0",
"phantomjs-prebuilt": "^2.1.7",
"promise": "^7.1.1",
"request": "^2.73.0",
"time-grunt": "^1.3.0"
},
"keywords": [

View File

@@ -3,6 +3,41 @@
jasmine.getEnv().addReporter(new jasmine.JSReporter2());
jasmine.getEnv().defaultTimeoutInterval = 3000;
// From https://github.com/axemclion/grunt-saucelabs/issues/109#issuecomment-166767282
// (function () {
// var oldJSReport = window.jasmine.getJSReport;
// window.jasmine.getJSReport = function () {
// var results = oldJSReport();
// if (results) {
// return {
// durationSec: results.durationSec,
// suites: removePassingTests(results.suites),
// passed: results.passed
// };
// } else {
// return null;
// }
// };
// function removePassingTests (suites) {
// return suites.filter(specFailed)
// .map(mapSuite);
// }
// function mapSuite (suite) {
// var result = {};
// for (var s in suite) {
// result[s] = suite[s];
// }
// result.specs = suite.specs.filter(specFailed);
// result.suites = removePassingTests(suite.suites);
// return result;
// }
// function specFailed (item) {
// return !item.passed;
// }
// })();
/* record log messages for testing */
var logMessages = [];
@@ -45,15 +80,15 @@ less.loggers = [
}
];
var testLessEqualsInDocument = function () {
testLessEqualsInDocument = function () {
testLessInDocument(testSheet);
};
var testLessErrorsInDocument = function (isConsole) {
testLessErrorsInDocument = function (isConsole) {
testLessInDocument(isConsole ? testErrorSheetConsole : testErrorSheet);
};
var testLessInDocument = function (testFunc) {
testLessInDocument = function (testFunc) {
var links = document.getElementsByTagName('link'),
typePattern = /^text\/(x-)?less$/;
@@ -65,7 +100,7 @@ var testLessInDocument = function (testFunc) {
}
};
var ieFormat = function(text) {
ieFormat = function(text) {
var styleNode = document.createElement('style');
styleNode.setAttribute('type', 'text/css');
var headNode = document.getElementsByTagName('head')[0];
@@ -84,7 +119,7 @@ var ieFormat = function(text) {
return transformedText;
};
var testSheet = function (sheet) {
testSheet = function (sheet) {
it(sheet.id + " should match the expected output", function (done) {
var lessOutputId = sheet.id.replace("original-", ""),
expectedOutputId = "expected-" + lessOutputId,
@@ -123,7 +158,7 @@ function extractId(href) {
.replace(/\./g, ':'); // Replace dots with colons(for valid id)
}
var waitFor = function (waitFunc) {
waitFor = function (waitFunc) {
return new Promise(function (resolve) {
var timeoutId = setInterval(function () {
if (waitFunc()) {
@@ -134,7 +169,7 @@ var waitFor = function (waitFunc) {
});
};
var testErrorSheet = function (sheet) {
testErrorSheet = function (sheet) {
it(sheet.id + " should match an error", function (done) {
var lessHref = sheet.href,
id = "less-error-message:" + extractId(lessHref),
@@ -183,7 +218,7 @@ var testErrorSheet = function (sheet) {
});
};
var testErrorSheetConsole = function (sheet) {
testErrorSheetConsole = function (sheet) {
it(sheet.id + " should match an error", function (done) {
var lessHref = sheet.href,
id = sheet.id.replace(/^original-less:/, "less-error-message:"),
@@ -212,7 +247,7 @@ var testErrorSheetConsole = function (sheet) {
});
};
var loadFile = function (href) {
loadFile = function (href) {
return new Promise(function (resolve, reject) {
var request = new XMLHttpRequest();
request.open('GET', href, true);

View File

@@ -3,30 +3,22 @@
<head>
<meta charset="utf-8">
<title>Jasmine Spec Runner</title>
<!-- generate script tags for tests -->
<!-- Polyfill shim for older browsers -->
<script src="https://cdn.polyfill.io/v2/polyfill.min.js"></script>
<script>
var SCRIPT_TAGS = "";
</script>
<% var generateScriptTags = function(allScripts) { allScripts.forEach(function(script){ %>
<script src="<%= script %>"></script>
<script src='<%= script %>'></script>
<% }); }; %>
<!-- generate script tags for tests -->
<% var toArray = function(scripts) {
%>[<%
scripts.forEach(function(scriptUrl, index){
%>"<%= scriptUrl %>"<%
if (index !== scripts.length -1) {
%>,<%
}
});
%>]<%
}; %>
<!-- for each test, generate CSS/LESS link tags -->
<% scripts.src.forEach(function(fullLessName) {
var pathParts = fullLessName.split('/');
var fullCssName = fullLessName.replace(/less/g, 'css');
var lessName = pathParts[pathParts.length - 1];
var name = lessName.split('.')[0]; %>
<!-- the tags to be generated -->
<link id="original-less:test-less-<%= name %>" title="test-less-<%= name %>" rel="stylesheet/less" type="text/css" href="<%= fullLessName %>">
<link id="expected-less:test-less-<%= name %>" rel="stylesheet" type="text/css" href="<%= fullCssName %>">
<% }); %>
@@ -35,60 +27,24 @@
<% css.forEach(function(style){ %>
<link rel="stylesheet" type="text/css" href="<%= style %>">
<% }) %>
<script>
function loadScript(url,callback){
var script = document.createElement('script');
if(document.documentMode === 8){
script.onreadystatechange = function(){
if (script.readyState === 'loaded'){
if (callback){callback()};
};
};
} else {
script.onload = function(){
if (callback){callback()};
};
};
script.src = url;
document.body.appendChild(script);
};
// allow sauce to query for the jasmine report
// because we have to load the page before starting the test, so the thing
// sauce queries for might not be setup yet
window.jasmine = { getJSReport: function() { } };
setTimeout(function() {
var jasmine = <% toArray([].concat(scripts.polyfills, scripts.jasmine, scripts.boot)) %>,
helpers = <% toArray(scripts.helpers) %>,
vendor = <% toArray(scripts.vendor) %>,
specs = <% toArray(scripts.specs) %>,
reporters = <% toArray([].concat(scripts.reporters)) %>,
allScripts = jasmine.concat(helpers).concat(vendor).concat(specs).concat(reporters);
function addNextScript() {
// for sauce, see above. Additional step needed between loading jasmine and loading
// the js reporter
if (!jasmine.getJSReport) {
jasmine.getJSReport = function() {};
}
if (allScripts.length) {
var scriptSrc = allScripts.shift();
loadScript(scriptSrc, addNextScript);
} else {
window.onload();
}
}
addNextScript();
},1000);
window.jasmine = { getJSReport: function() { } };
</script>
<%
var jasmine = [].concat(scripts.polyfills, scripts.jasmine, scripts.boot),
helpers = scripts.helpers,
vendor = scripts.vendor,
reporters = [].concat(scripts.reporters),
specs = scripts.specs,
allScripts = jasmine.concat(helpers).concat(vendor).concat(specs).concat(reporters);
generateScriptTags(allScripts);
%>
</head>
<body>
<!-- content -->
</body>
</html>