From 35386e49dd503ca8add096a5d2439d9fa1d1e402 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 15 May 2017 13:52:35 -0400 Subject: [PATCH 01/10] Re-run individual tests to avoid re-running the whole suite. To deal with individual flaky tests, we often just re-run the entire test suite, which feels like an enormous waste of shared computing resources. This change automatically re-runs individual failed tests as many as two more times, and considers the test successful if any of those attempts succeeds. cc @abernix @hwillson et al. --- tools/tool-testing/selftest.js | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tools/tool-testing/selftest.js b/tools/tool-testing/selftest.js index 492d7fbdce..31263f1485 100644 --- a/tools/tool-testing/selftest.js +++ b/tools/tool-testing/selftest.js @@ -1999,7 +1999,10 @@ var runTests = function (options) { _.each(testList.filteredTests, function (test) { totalRun++; Console.error(test.file + ": " + test.name + " ... "); + runTest(test); + }); + function runTest(test, tries = 3) { var failure = null; try { runningTest = test; @@ -2017,6 +2020,16 @@ var runTests = function (options) { if (failure) { Console.error("... fail!", Console.options({ indent: 2 })); + + if (--tries > 0) { + Console.error( + "... retrying (" + tries + (tries === 1 ? " try" : " tries") + " remaining) ...", + Console.options({ indent: 2 }) + ); + + return runTest(test, tries); + } + failedTests.push(test); testList.notifyFailed(test); @@ -2078,7 +2091,7 @@ var runTests = function (options) { "... ok (" + durationMs + " ms)", Console.options({ indent: 2 })); } - }); + } testList.saveTestState(); From 47496e9dba73d6a82567d7bb02464ffbbd23ed8f Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 15 May 2017 16:00:20 -0400 Subject: [PATCH 02/10] Call requestGarbageCollection in Isopack#_writeTool. This method appears to be causing large spikes in memory consumption on Circle CI during the `meteor --get-ready` preparation step, which often leads to the test process being killed. Also added a call in IsopackCache#_loadLocalPackage for good measure. We're now calling requestGarbageCollection pretty frequently when we run Node with --expose-gc, but that currently only happens during Circle CI tests, so I don't think we need to implement the improvements suggested in tools/utils/gc.js, yet. Previously: 35f488e140c5df4a2bc70de271feb38921d2a6d3, f6df21ff1ed7e54db2d1df90f6d766f971a93f58 --- tools/isobuild/isopack-cache.js | 2 ++ tools/isobuild/isopack.js | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/tools/isobuild/isopack-cache.js b/tools/isobuild/isopack-cache.js index 5da0a09ed9..4f2a5f3ed7 100644 --- a/tools/isobuild/isopack-cache.js +++ b/tools/isobuild/isopack-cache.js @@ -383,6 +383,8 @@ export class IsopackCache { }); } } + + requestGarbageCollection(); } } diff --git a/tools/isobuild/isopack.js b/tools/isobuild/isopack.js index 034aaced80..e78746124e 100644 --- a/tools/isobuild/isopack.js +++ b/tools/isobuild/isopack.js @@ -14,6 +14,7 @@ var utils = require('../utils/utils.js'); var buildPluginModule = require('./build-plugin.js'); var Console = require('../console/console.js').Console; var Profile = require('../tool-env/profile.js').Profile; +import { requestGarbageCollection } from "../utils/gc.js"; var rejectBadPath = function (p) { if (p.match(/\.\./)) { @@ -1802,6 +1803,8 @@ _.extend(Isopack.prototype, { symlink: false }); + requestGarbageCollection(); + // Build all of the isopackets now, so that no build step is required when // you're actually running meteor from a release in order to load packages. var isopacketBuildContext = isopackets.makeIsopacketBuildContext(); @@ -1812,6 +1815,8 @@ _.extend(Isopack.prototype, { // process are going to be the current tool's isopackets, not the // isopackets that we're writing out. _.each(isopackets.ISOPACKETS, function (packages, isopacketName) { + requestGarbageCollection(); + buildmessage.enterJob({ title: "compiling " + isopacketName + " packages for the tool" }, function () { @@ -1830,6 +1835,8 @@ _.extend(Isopack.prototype, { return; } + requestGarbageCollection(); + image.write( builder.enter(files.pathJoin('isopackets', isopacketName))); }); From fee86a0b17988cd796363bc36454c8179bc6f051 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Tue, 16 May 2017 14:55:20 -0400 Subject: [PATCH 03/10] Try not running a full meteor --get-ready before Circle CI tests. In the ongoing struggle with Circle CI-specific test failures, the preparatory `meteor --get-ready` has been a consistent point of failure, before any real tests have the chance to run. Using a lighter-weight command (meteor --help) that still does most of what --get-ready did seems worth a try, though it might just defer memory-intensive work until later, so we'll have to see what happens. --- circle.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/circle.yml b/circle.yml index 0241b50520..d7f983e499 100644 --- a/circle.yml +++ b/circle.yml @@ -13,9 +13,9 @@ dependencies: - ".meteor" - ".babel-cache" override: - # shouldn't take longer than 10 minutes - - ./meteor --get-ready: - timeout: 600 + # shouldn't take longer than 5 minutes + - ./meteor --help: + timeout: 300 environment: METEOR_PRETTY_OUTPUT: 0 METEOR_DISABLE_OPTIMISTIC_CACHING: 1 From 5f18c6e24ab1bf666c87cc21420777a8f9ec2c8a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 19 May 2017 17:09:03 -0700 Subject: [PATCH 04/10] tools: fix Galaxy discovery 1a036553 in 1.4.4.2 expanded on the HTTP error checking added by 30aec9f in 1.4.2. Neither of these changes were aware that discoverGalaxy invokes httpHelpers.request with json:true, resulting in a `body` that is a parsed JSON object rather than a string or Buffer. Before 1.4.4.2, this had no consequences because body.length is undefined and `undefined < 90` is false, but the change to Buffer.byteLength actually made the condition true. It's safe to not check length in the JSON case because a truncated JSON object is not legal JSON (unless the truncation just drops trailing whitespace, in white case that's OK). I check for both string and Buffer because some calls to this function pass in an encoding option. Buffer.byteLength works with both types. --- tools/utils/http-helpers.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/utils/http-helpers.js b/tools/utils/http-helpers.js index ac40002bd7..9cd2d32a23 100644 --- a/tools/utils/http-helpers.js +++ b/tools/utils/http-helpers.js @@ -308,7 +308,8 @@ _.extend(exports, { Console.debug("Doing HTTP request: ", options.method || 'GET', options.url); var request = require('request'); var req = request(options, function (error, response, body) { - if (! error && response && body) { + if (! error && response && body + && (typeof(body) == "string" || Buffer.isBuffer(body))) { const contentLength = Number(response.headers["content-length"]); const actualLength = Buffer.byteLength(body); From 2a3fd573034a98ea518684fe3435d8140db0295c Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Sun, 21 May 2017 11:02:18 -0400 Subject: [PATCH 05/10] Style tweaks and a small bug fix. These checks should still happen when body is an empty string, which (for better or worse) is falsy in JavaScript. --- tools/utils/http-helpers.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/utils/http-helpers.js b/tools/utils/http-helpers.js index 9cd2d32a23..1f608dd042 100644 --- a/tools/utils/http-helpers.js +++ b/tools/utils/http-helpers.js @@ -308,8 +308,10 @@ _.extend(exports, { Console.debug("Doing HTTP request: ", options.method || 'GET', options.url); var request = require('request'); var req = request(options, function (error, response, body) { - if (! error && response && body - && (typeof(body) == "string" || Buffer.isBuffer(body))) { + if (! error && + response && + (typeof body === "string" || + Buffer.isBuffer(body))) { const contentLength = Number(response.headers["content-length"]); const actualLength = Buffer.byteLength(body); From 48814ddbe3c9e7f0328fe4e4b58c54f2fd426017 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 22 May 2017 16:51:56 -0400 Subject: [PATCH 06/10] Update History.md to mention fix for #8709. --- History.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/History.md b/History.md index 1f9753564f..b66987aab0 100644 --- a/History.md +++ b/History.md @@ -1,5 +1,11 @@ ## v.NEXT +## v1.4.4.3, 2017-05-22 + +* A bug in checking body lengths of HTTP responses that was affecting + Galaxy deploys has been fixed. + [PR #8709](https://github.com/meteor/meteor/pull/8709). + ## v1.4.4.2, 2017-05-02 * Node has been upgraded to version 4.8.2. From 95c11c1b7897bd8f989d9e65ba944371aad13434 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 22 May 2017 16:51:33 -0400 Subject: [PATCH 07/10] Upgrade Node to version 4.8.3. https://nodejs.org/en/blog/release/v4.8.3/ --- History.md | 2 ++ scripts/build-dev-bundle-common.sh | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index b66987aab0..0eff27a71a 100644 --- a/History.md +++ b/History.md @@ -2,6 +2,8 @@ ## v1.4.4.3, 2017-05-22 +* Node has been upgraded to version 4.8.3. + * A bug in checking body lengths of HTTP responses that was affecting Galaxy deploys has been fixed. [PR #8709](https://github.com/meteor/meteor/pull/8709). diff --git a/scripts/build-dev-bundle-common.sh b/scripts/build-dev-bundle-common.sh index 5e5b8062f7..da0f8253f9 100644 --- a/scripts/build-dev-bundle-common.sh +++ b/scripts/build-dev-bundle-common.sh @@ -6,7 +6,7 @@ set -u UNAME=$(uname) ARCH=$(uname -m) MONGO_VERSION=3.2.12 -NODE_VERSION=4.8.2 +NODE_VERSION=4.8.3 NPM_VERSION=4.5.0 if [ "$UNAME" == "Linux" ] ; then From 6c998370db3e824190bf8b01ec097d43c6231d79 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 22 May 2017 16:55:13 -0400 Subject: [PATCH 08/10] Bump $BUNDLE_VERSION to 4.7.27 before rebuilding dev bundle. --- meteor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meteor b/meteor index 4068a935a6..500777dcdd 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/usr/bin/env bash -BUNDLE_VERSION=4.7.26 +BUNDLE_VERSION=4.7.27 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. From 994558ccd7982434803c55a7e578beffb64c0741 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 22 May 2017 17:00:06 -0400 Subject: [PATCH 09/10] Bump package versions for 1.4.4.3-rc.0 release. --- packages/meteor-tool/package.js | 2 +- scripts/admin/meteor-release-experimental.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/meteor-tool/package.js b/packages/meteor-tool/package.js index 0e1716adba..fbb57a3492 100644 --- a/packages/meteor-tool/package.js +++ b/packages/meteor-tool/package.js @@ -1,6 +1,6 @@ Package.describe({ summary: "The Meteor command-line tool", - version: '1.4.4_2' + version: '1.4.4-3-rc.0' }); Package.includeTool(); diff --git a/scripts/admin/meteor-release-experimental.json b/scripts/admin/meteor-release-experimental.json index a92fd7261f..9f5fdbb490 100644 --- a/scripts/admin/meteor-release-experimental.json +++ b/scripts/admin/meteor-release-experimental.json @@ -1,6 +1,6 @@ { "track": "METEOR", - "version": "1.4.4.2-rc.1", + "version": "1.4.4.3-rc.0", "recommended": false, "official": false, "description": "Meteor" From 7e52b10ba4754ecb53a2f10525c5ecf6fb998bb1 Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 22 May 2017 18:40:32 -0400 Subject: [PATCH 10/10] Bump package versions for the official 1.4.4.3 release. --- packages/meteor-tool/package.js | 2 +- scripts/admin/meteor-release-official.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/meteor-tool/package.js b/packages/meteor-tool/package.js index fbb57a3492..531b6c0664 100644 --- a/packages/meteor-tool/package.js +++ b/packages/meteor-tool/package.js @@ -1,6 +1,6 @@ Package.describe({ summary: "The Meteor command-line tool", - version: '1.4.4-3-rc.0' + version: '1.4.4_3' }); Package.includeTool(); diff --git a/scripts/admin/meteor-release-official.json b/scripts/admin/meteor-release-official.json index 1e5f97cce6..d1e963fd87 100644 --- a/scripts/admin/meteor-release-official.json +++ b/scripts/admin/meteor-release-official.json @@ -1,7 +1,8 @@ { "track": "METEOR", - "version": "1.4.4.2", + "version": "1.4.4.3", "recommended": false, "official": true, + "patchFrom": ["1.4.4.2"], "description": "The Official Meteor Distribution" }