From 3d34e5b5ef3204013d269709b673842054bfd542 Mon Sep 17 00:00:00 2001 From: karayu Date: Tue, 26 Nov 2013 15:34:03 -0800 Subject: [PATCH 001/112] added meta description --- docs/client/docs.html | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/client/docs.html b/docs/client/docs.html index f1eefb41cb..77d03ee492 100644 --- a/docs/client/docs.html +++ b/docs/client/docs.html @@ -1,6 +1,8 @@ Documentation - Meteor + From 444c56f1bdf5e5bc0ae7d2e020720f9ac6a1d422 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 26 Nov 2013 15:44:44 -0800 Subject: [PATCH 002/112] upgrade docs to follower-11. --- docs/.meteor/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index 1b52537518..2d08b78d05 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -galaxy-follower-6 +follower-11 From 9fefc349e02fd7455aa91f7f6331604ec5e7aa31 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Mon, 9 Dec 2013 13:56:04 -0800 Subject: [PATCH 003/112] Make 'satellite' a special key in logging --- packages/logging/logging.js | 6 +++++- packages/logging/logging_test.js | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/logging/logging.js b/packages/logging/logging.js index 09587906e9..1b7105c19d 100644 --- a/packages/logging/logging.js +++ b/packages/logging/logging.js @@ -51,7 +51,7 @@ var META_COLOR = 'blue'; // XXX package var RESTRICTED_KEYS = ['time', 'timeInexact', 'level', 'file', 'line', - 'program', 'originApp', 'stderr']; + 'program', 'originApp', 'satellite', 'stderr']; var FORMATTED_KEYS = RESTRICTED_KEYS.concat(['app', 'message']); @@ -202,6 +202,7 @@ Log.format = function (obj, options) { var originApp = obj.originApp; var message = obj.message || ''; var program = obj.program || ''; + var satellite = obj.satellite; var stderr = obj.stderr || ''; _.each(FORMATTED_KEYS, function(key) { @@ -239,6 +240,9 @@ Log.format = function (obj, options) { ['(', (program ? program + ':' : ''), file, ':', lineNumber, ') '].join('') : ''; + if (satellite) + sourceInfo += ['[', satellite, ']'].join(''); + var stderrIndicator = stderr ? '(STDERR) ' : ''; var metaPrefix = [ diff --git a/packages/logging/logging_test.js b/packages/logging/logging_test.js index 4e138b3ef1..d70c5584f2 100644 --- a/packages/logging/logging_test.js +++ b/packages/logging/logging_test.js @@ -92,7 +92,8 @@ Tinytest.add("logging - log", function (test) { test.throws(function () { log({level: 'not the right level'}); }); - _.each(['file', 'line', 'program', 'originApp'], function (restrictedKey) { + _.each(['file', 'line', 'program', 'originApp', 'satellite'], + function (restrictedKey) { test.throws(function () { var obj = {}; obj[restrictedKey] = 'usage of restricted key'; From db624dd162c81392a35d09c30eba64974fd1641f Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 10 Dec 2013 11:37:43 -0800 Subject: [PATCH 004/112] New dev bundle: mongo with startup speed boost. Also upgrade to 2.4.8 while we're at it. (In the meteor/mongo repository, we've created an ssl-2.4.8 branch, and also added a backported-from-Mongo-2.5 change to poll the repl set config more often on startup.) --- meteor | 2 +- scripts/generate-dev-bundle.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/meteor b/meteor index f9d4362686..54260b22d9 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/bin/bash -BUNDLE_VERSION=0.3.25 +BUNDLE_VERSION=0.3.26 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index e91efa7a65..caaa28007e 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -160,7 +160,7 @@ make install # click 'changelog' under the current version, then 'release notes' in # the upper right. cd "$DIR/build" -MONGO_VERSION="2.4.6" +MONGO_VERSION="2.4.8" # We use Meteor fork since we added some changes to the building script. # Our patches allow us to link most of the libraries statically. From 1b08ac2d9d9f1c7eab8708d4d50ee7eaf64324cb Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 10 Dec 2013 15:27:40 -0800 Subject: [PATCH 005/112] Upgrade to stock Node 0.10.22. This discards our fix for stream pausing. We'll monkey patch it in at startup instead of relying on a custom build. --- scripts/generate-dev-bundle.sh | 5 ++--- tools/meteor.js | 4 ++-- tools/server/boot.js | 4 ++-- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index caaa28007e..c3bb2522a1 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -71,13 +71,12 @@ umask 022 mkdir build cd build -# Temporarily use a fork of 0.10.21 plus a change to fix websockets. -git clone git://github.com/meteor/node.git +git clone git://github.com/joyent/node.git cd node # When upgrading node versions, also update the values of MIN_NODE_VERSION at # the top of tools/meteor.js and tools/server/boot.js, and the text in # docs/client/concepts.html and the README in tools/bundler.js. -git checkout dev-bundle-0.3.24 +git checkout v0.10.22 ./configure --prefix="$DIR" make -j4 diff --git a/tools/meteor.js b/tools/meteor.js index 929be58176..053ec38e4d 100644 --- a/tools/meteor.js +++ b/tools/meteor.js @@ -23,8 +23,8 @@ Fiber(function () { var cleanup = require('./cleanup.js'); var Future = require('fibers/future'); - // This code is duplicated in app/server/server.js. - var MIN_NODE_VERSION = 'v0.10.21'; + // This code is duplicated in tools/server/boot.js. + var MIN_NODE_VERSION = 'v0.10.22'; if (require('semver').lt(process.version, MIN_NODE_VERSION)) { process.stderr.write( 'Meteor requires Node ' + MIN_NODE_VERSION + ' or later.\n'); diff --git a/tools/server/boot.js b/tools/server/boot.js index 04cbcf1a9c..dff293fa89 100644 --- a/tools/server/boot.js +++ b/tools/server/boot.js @@ -5,8 +5,8 @@ var Future = require(path.join("fibers", "future")); var _ = require('underscore'); var sourcemap_support = require('source-map-support'); -// This code is duplicated in tools/server/server.js. -var MIN_NODE_VERSION = 'v0.10.21'; +// This code is duplicated in tools/meteor.js. +var MIN_NODE_VERSION = 'v0.10.22'; if (require('semver').lt(process.version, MIN_NODE_VERSION)) { process.stderr.write( From 24a80d49b16fe4cc5a3b68d3dbee61e4bf7edfbe Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 10 Dec 2013 15:36:46 -0800 Subject: [PATCH 006/112] Preemptively add bcrypt to the dev bundle. We think we may want it soon. --- scripts/generate-dev-bundle.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index c3bb2522a1..ec2c7af7ac 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -109,6 +109,7 @@ npm install shell-quote@0.0.1 # now at 1.3.3, which adds plenty of options to npm install eachline@2.3.3 npm install source-map@0.1.30 npm install source-map-support@0.2.3 +npm install bcrypt@0.7.7 # Using the unreleased "caronte" branch rewrite of http-proxy (which will become # 1.0.0), plus this PR: From f9b394de4f5092f092348da20c126e4e799e10d6 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 10 Dec 2013 17:47:49 -0800 Subject: [PATCH 007/112] Use smaller oplog (8MB instead of 256MB default). This uses less space on disk and starts up faster. It might mean people hit the end of the oplog if the server gets really slow. But, hey, then we can debug and test more oplog error modes. --- tools/mongo_runner.js | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 69500088d3..50238e7458 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -206,6 +206,7 @@ exports.launchMongo = function (options) { '--nohttpinterface', '--port', options.port, '--dbpath', dbPath, + '--oplogSize', '8', '--replSet', replSetName ]); From 82d5fcabf8eef684596dd34c927b65dca0615960 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 10 Dec 2013 19:02:34 -0800 Subject: [PATCH 008/112] Work around Node issue #6506. We were previously doing this with a fork of Node in the dev bundle, but that wouldn't help users in production. We now expect users to use precisely v0.10.22 in production. When new versions of Node come out, if users choose to use them before we get a chance to put out a new Meteor, we disable websockets to work around the most common effect of the issue instead of monkey-patching. --- History.md | 7 ++ packages/meteor/node-issue-6506-workaround.js | 117 ++++++++++++++++++ packages/meteor/package.js | 3 + 3 files changed, 127 insertions(+) create mode 100644 packages/meteor/node-issue-6506-workaround.js diff --git a/History.md b/History.md index 6df859ccae..99a7b04596 100644 --- a/History.md +++ b/History.md @@ -1,5 +1,11 @@ ## vNEXT +This version of Meteor contains a patch for a bug in Node 0.10 which most +commonly affects websockets. The patch is against Node v0.10.22. We strongly +recommend using this precise version of Node in production so that the patch +will be applied. If you use a newer version of Node with this version of Meteor, +Meteor will not apply the patch and will instead disable websockets. + * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded message at DDP startup. Now the hot code push only triggers when @@ -30,6 +36,7 @@ * Upgraded dependencies: * SockJS server from 0.3.7 to 0.3.8 + * Node from 0.10.21 to 0.10.22 Patches contributed by GitHub users awwx, mcbain, rzymek. diff --git a/packages/meteor/node-issue-6506-workaround.js b/packages/meteor/node-issue-6506-workaround.js new file mode 100644 index 0000000000..b4c640e70b --- /dev/null +++ b/packages/meteor/node-issue-6506-workaround.js @@ -0,0 +1,117 @@ +// Temporary workaround for https://github.com/joyent/node/issues/6506 +// Our fix involves replicating a bunch of files in order to +// +if (process.version !== 'v0.10.22') { + if (!process.env.DISABLE_WEBSOCKETS) { + console.error("This version of Meteor contains a patch for a bug in Node v0.10."); + console.error("The patch is against v0.10.22."); + console.error("You are using version " + process.version + " instead, so we cannot apply the patch."); + console.error("To mitigate the most common effect of the bug, websockets will be disabled."); + console.error("To enable websockets, use Node v0.10.22 or upgrade to a later version of Meteor (if available)."); + process.env.DISABLE_WEBSOCKETS = 't'; + } +} else { + // This code is all copied from Node's lib/_stream_writable.js, git tag + // v0.10.22, with one change (see "BUGFIX"). + var Writable = Npm.require('_stream_writable'); + var Duplex = Npm.require('_stream_duplex'); + + Writable.prototype.write = function(chunk, encoding, cb) { + var state = this._writableState; + var ret = false; + + if (typeof encoding === 'function') { + cb = encoding; + encoding = null; + } + + if (Buffer.isBuffer(chunk)) + encoding = 'buffer'; + else if (!encoding) + encoding = state.defaultEncoding; + + if (typeof cb !== 'function') + cb = function() {}; + + if (state.ended) + writeAfterEnd(this, state, cb); + else if (validChunk(this, state, chunk, cb)) + ret = writeOrBuffer(this, state, chunk, encoding, cb); + + return ret; + }; + + // Duplex doesn't directly inherit from Writable: it copies over this function + // explicitly. So we have to do it too. + Duplex.prototype.write = Writable.prototype.write; + + function writeAfterEnd(stream, state, cb) { + var er = new Error('write after end'); + // TODO: defer error events consistently everywhere, not just the cb + stream.emit('error', er); + process.nextTick(function() { + cb(er); + }); + } + + function validChunk(stream, state, chunk, cb) { + var valid = true; + if (!Buffer.isBuffer(chunk) && + 'string' !== typeof chunk && + chunk !== null && + chunk !== undefined && + !state.objectMode) { + var er = new TypeError('Invalid non-string/buffer chunk'); + stream.emit('error', er); + process.nextTick(function() { + cb(er); + }); + valid = false; + } + return valid; + } + + function writeOrBuffer(stream, state, chunk, encoding, cb) { + chunk = decodeChunk(state, chunk, encoding); + if (Buffer.isBuffer(chunk)) + encoding = 'buffer'; + var len = state.objectMode ? 1 : chunk.length; + + state.length += len; + + var ret = state.length < state.highWaterMark; + // This next line is the BUGFIX: + state.needDrain = state.needDrain || !ret; + + if (state.writing) + state.buffer.push(new WriteReq(chunk, encoding, cb)); + else + doWrite(stream, state, len, chunk, encoding, cb); + + return ret; + } + + function decodeChunk(state, chunk, encoding) { + if (!state.objectMode && + state.decodeStrings !== false && + typeof chunk === 'string') { + chunk = new Buffer(chunk, encoding); + } + return chunk; + } + + function WriteReq(chunk, encoding, cb) { + this.chunk = chunk; + this.encoding = encoding; + this.callback = cb; + } + + function doWrite(stream, state, len, chunk, encoding, cb) { + state.writelen = len; + state.writecb = cb; + state.writing = true; + state.sync = true; + stream._write(chunk, encoding, state.onwrite); + state.sync = false; + } +} diff --git a/packages/meteor/package.js b/packages/meteor/package.js index 439d80e6f2..5e16f7f9d6 100644 --- a/packages/meteor/package.js +++ b/packages/meteor/package.js @@ -15,6 +15,9 @@ Package.on_use(function (api) { api.export('Meteor'); + // Workaround for https://github.com/joyent/node/issues/6506 + api.add_files('node-issue-6506-workaround.js', 'server'); + api.add_files('client_environment.js', 'client'); api.add_files('server_environment.js', 'server'); api.add_files('helpers.js', ['client', 'server']); From b4c14af03296fadac5c947de40eb154e8e16a280 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 10 Dec 2013 19:09:41 -0800 Subject: [PATCH 009/112] Fix find_mongo_pids regexp Also add some comments to mongod options. --- tools/mongo_runner.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 50238e7458..e92e54a7fd 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -25,7 +25,7 @@ var find_mongo_pids = function (app_dir, port, callback) { _.each(stdout.split('\n'), function (ps_line) { // matches mongos we start. - var m = ps_line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db --replSet /); + var m = ps_line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db /); if (m && m.length === 4) { var found_pid = parseInt(m[1]); var found_port = parseInt(m[2]); @@ -199,13 +199,15 @@ exports.launchMongo = function (options) { var child_process = require('child_process'); var replSetName = 'meteor'; var proc = child_process.spawn(mongod_path, [ - // nb: cli-test.sh and find_mongo_pids assume that the next four arguments - // exist in this order without anything in between + // nb: cli-test.sh and find_mongo_pids make strong assumptions about the + // order of the arguments! Check them before changing any arguments. '--bind_ip', '127.0.0.1', '--smallfiles', '--nohttpinterface', '--port', options.port, '--dbpath', dbPath, + // Use an 8MB oplog rather than 256MB. Uses less space on disk and + // initializes faster. (Not recommended for production!) '--oplogSize', '8', '--replSet', replSetName ]); From 89ec9ddf6296b6a0ac04acb221f765c16002e8b6 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 10 Dec 2013 22:13:12 -0800 Subject: [PATCH 010/112] Up timeout for printing mongo message. This way it doesn't happen normally during cold start (3--4 sec). --- tools/run.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run.js b/tools/run.js index cd4da15890..8cc0b34922 100644 --- a/tools/run.js +++ b/tools/run.js @@ -677,7 +677,7 @@ exports.run = function (context, options) { mongoStartupPrintTimer = setTimeout(function () { process.stdout.write("Initializing mongo database... this may take a moment.\n"); - }, 3000); + }, 5000); updater.startUpdateChecks(context); launch(); From b611db51c2505453ba0ada39d52ab05a5795f295 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 8 Dec 2013 13:47:51 -0800 Subject: [PATCH 011/112] Implement _pollQuery. Don't call it. --- packages/mongo-livedata/doc_fetcher.js | 3 + .../mongo-livedata/oplog_observe_driver.js | 111 ++++++++++++++++-- 2 files changed, 103 insertions(+), 11 deletions(-) diff --git a/packages/mongo-livedata/doc_fetcher.js b/packages/mongo-livedata/doc_fetcher.js index 86f7e82cf7..fbc73bd3f5 100644 --- a/packages/mongo-livedata/doc_fetcher.js +++ b/packages/mongo-livedata/doc_fetcher.js @@ -15,6 +15,9 @@ _.extend(DocFetcher.prototype, { // If you make multiple calls to fetch() with the same cacheKey (a string), // DocFetcher may assume that they all return the same document. (It does // not check to see if collectionName/id match.) + // + // You may assume that callback is never called synchronously (and in fact + // OplogObserveDriver does so). fetch: function (collectionName, id, cacheKey, callback) { var self = this; diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index d42eae9451..e72895bdc7 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -2,7 +2,7 @@ var Fiber = Npm.require('fibers'); var Future = Npm.require('fibers/future'); var PHASE = { - INITIALIZING: 1, + QUERYING: 1, FETCHING: 2, STEADY: 3 }; @@ -29,7 +29,7 @@ OplogObserveDriver = function (options) { Package.facts && Package.facts.Facts.incrementServerFact( "mongo-livedata", "oplog-observers", 1); - self._phase = PHASE.INITIALIZING; + self._phase = PHASE.QUERYING; self._published = new LocalCollection._IdMap; var selector = self._cursorDescription.selector; @@ -61,8 +61,8 @@ OplogObserveDriver = function (options) { }); } else { // All other operators should be handled depending on phase - if (self._phase === PHASE.INITIALIZING) - self._handleOplogEntryInitializing(op); + if (self._phase === PHASE.QUERYING) + self._handleOplogEntryQuerying(op); else self._handleOplogEntrySteadyOrFetching(op); } @@ -125,11 +125,18 @@ _.extend(OplogObserveDriver.prototype, { self._published.remove(id); self._multiplexer.removed(id); }, - _handleDoc: function (id, newDoc) { + _handleDoc: function (id, newDoc, mustMatchNow) { var self = this; newDoc = _.clone(newDoc); + var matchesNow = newDoc && self._selectorFn(newDoc); + if (mustMatchNow && !matchesNow) { + throw Error("expected " + EJSON.stringify(newDoc) + " to match " + + EJSON.stringify(self._cursorDescription)); + } + var matchedBefore = self._published.has(id); + if (matchesNow && !matchedBefore) { self._add(newDoc); } else if (matchedBefore && !matchesNow) { @@ -168,17 +175,25 @@ _.extend(OplogObserveDriver.prototype, { if (err) { if (!anyError) anyError = err; - } else if (!self._stopped) { + } else if (!self._stopped && self._phase === PHASE.FETCHING) { + // We re-check the phase in case we've had an explicit _pollQuery + // call which pulls us out of FETCHING and back into QUERYING. self._handleDoc(id, doc); } waiting--; - if (waiting == 0) + // Because fetch() never calls its callback synchronously, this is + // safe (ie, we won't call fut.return() before the forEach is done). + if (waiting === 0) fut.return(); }); }); fut.wait(); + // XXX do this even if we've switched to PHASE.QUERYING? if (anyError) throw anyError; + // Exit now if we've had a _pollQuery call. + if (self._phase === PHASE.QUERYING) + return; self._currentlyFetching = null; } self._beSteady(); @@ -194,7 +209,7 @@ _.extend(OplogObserveDriver.prototype, { }); }); }, - _handleOplogEntryInitializing: function (op) { + _handleOplogEntryQuerying: function (op) { var self = this; self._needToFetch.set(idForOp(op), op.ts.toString()); }, @@ -251,7 +266,7 @@ _.extend(OplogObserveDriver.prototype, { if (self._stopped) throw new Error("oplog stopped surprisingly early"); - var initialCursor = new Cursor(self._mongoHandle, self._cursorDescription); + var initialCursor = self._cursorForQuery(); initialCursor.forEach(function (initialDoc) { self._add(initialDoc); }); @@ -261,13 +276,52 @@ _.extend(OplogObserveDriver.prototype, { // stop() to be called.) self._multiplexer.ready(); + self._doneQuerying(); + }, + + // In various circumstances, we may just want to stop processing the oplog and + // re-run the initial query, just as if we were a PollingObserveDriver. + _pollQuery: function () { + var self = this; + + // XXX maybe this should just return? + if (self._stopped) + throw new Error("can't re-poll a stopped query"); + + // XXX maybe this should either just return, or queue up another poll + // somehow? + if (self._phase === PHASE.QUERYING) + throw new Error("can't re-poll re-entrantly"); + + if (self._phase === PHASE.FETCHING) { + // Yay, we get to forget about all the things we thought we had to fetch. + self._needToFetch = new LocalCollection._IdMap; + self._currentlyFetching = null; + } + self._phase = PHASE.QUERYING; + + // subtle note: _published does not contain _id fields, but newResults does + var newResults = new LocalCollection._IdMap; + var cursor = self._cursorForQuery(); + cursor.forEach(function (doc) { + newResults.set(doc._id, doc); + }); + + self._publishNewResults(newResults); + + self._doneQuerying(); + }, + + _doneQuerying: function () { + var self = this; + if (self._stopped) return; self._mongoHandle._oplogHandle.waitUntilCaughtUp(); if (self._stopped) return; - if (self._phase !== PHASE.INITIALIZING) + if (self._phase !== PHASE.QUERYING) throw Error("Phase unexpectedly " + self._phase); if (self._needToFetch.empty()) { @@ -276,6 +330,42 @@ _.extend(OplogObserveDriver.prototype, { self._fetchModifiedDocuments(); } }, + + _cursorForQuery: function () { + var self = this; + // XXX this is WRONG we need to use the shared projection!! + return new Cursor(self._mongoHandle, self._cursorDescription); + }, + + + // Replace self._published with newResults (both are IdMaps), invoking observe + // callbacks on the multiplexer. + // + // XXX This is very similar to LocalCollection._diffQueryUnorderedChanges. We + // should really: (a) Unify IdMap and OrderedDict into Unordered/OrderedDict (b) + // Rewrite diff.js to use these classes instead of arrays and objects. + _publishNewResults: function (newResults) { + var self = this; + + // First remove anything that's gone. Be careful not to modify + // self._published while iterating over it. + var idsToRemove = []; + self._published.each(function (doc, id) { + if (!newResults.has(id)) + idsToRemove.push(id); + }); + _.each(idsToRemove, function (id) { + self._remove(id); + }); + + // Now do adds and changes. + newResults.each(function (doc, id) { + // "true" here means to throw if we think this doc doesn't match the + // selector. + self._handleDoc(id, doc, true); + }); + }, + // This stop function is invoked from the onStop of the ObserveMultiplexer, so // it shouldn't actually be possible to call it until the multiplexer is // ready. @@ -358,5 +448,4 @@ OplogObserveDriver.cursorSupported = function (cursorDescription) { }); }; - MongoTest.OplogObserveDriver = OplogObserveDriver; From 3dd3bd3bcabf3881b804c7bced0394200b15f204 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 10 Dec 2013 12:44:50 -0800 Subject: [PATCH 012/112] Get all relevant fields during initial query --- .../mongo-livedata/mongo_livedata_tests.js | 61 ++++++++++++++++--- .../mongo-livedata/oplog_observe_driver.js | 19 ++++-- 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index ba7e819bbb..7296864cd8 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -23,6 +23,15 @@ if (Meteor.isServer) { }); } +var runInFence = function (f) { + if (Meteor.isClient) { + f(); + } else { + var fence = new DDPServer._WriteFence; + DDPServer._CurrentWriteFence.withValue(fence, f); + fence.armAndWait(); + } +}; // Helpers for upsert tests @@ -478,16 +487,6 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, }); -var runInFence = function (f) { - if (Meteor.isClient) { - f(); - } else { - var fence = new DDPServer._WriteFence; - DDPServer._CurrentWriteFence.withValue(fence, f); - fence.armAndWait(); - } -}; - Tinytest.addAsync("mongo-livedata - scribbling, " + idGeneration, function (test, onComplete) { var run = test.runId(); var coll; @@ -1897,3 +1896,45 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - _disableOplog", functi test.isFalse(observeWithoutOplog._observeDriver._usesOplog); observeWithoutOplog.stop(); }); + +Meteor.isServer && Tinytest.add("mongo-livedata - oplog - include selector fields", function (test) { + var collName = "includeSelector" + Random.id(); + var coll = new Meteor.Collection(collName); + + var docId = coll.insert({a: 1, b: [3, 2], c: 'foo'}); + test.isTrue(docId); + + // Wait until we've processed the insert oplog entry. (If the insert shows up + // during the observeChanges, the bug in question is not consistently + // reproduced.) + MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle.waitUntilCaughtUp(); + + + var output = []; + var handle = coll.find({a: 1, b: 2}, {fields: {c: 1}}).observeChanges({ + added: function (id, fields) { + output.push(['added', id, fields]); + }, + changed: function (id, fields) { + output.push(['changed', id, fields]); + }, + removed: function (id) { + output.push(['removed', id]); + } + }); + // Initially should match the document. + test.length(output, 1); + test.equal(output.shift(), ['added', docId, {c: 'foo'}]); + + // Update in such a way that, if we only knew about the published field 'c' + // and the changed field 'b' (but not the field 'a'), we would think it didn't + // match any more. (This is a regression test for a bug that existed because + // we used to not use the shared projection in the initial query.) + runInFence(function () { + coll.update(docId, {$set: {'b.0': 2, c: 'bar'}}); + }); + test.length(output, 1); + test.equal(output.shift(), ['changed', docId, {c: 'bar'}]); + + handle.stop(); +}); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index e72895bdc7..0f05c60f52 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -39,10 +39,10 @@ OplogObserveDriver = function (options) { self._projectionFn = LocalCollection._compileProjection(projection); // Projection function, result of combining important fields for selector and // existing fields projection - var sharedProjection = LocalCollection._combineSelectorAndProjection( + self._sharedProjection = LocalCollection._combineSelectorAndProjection( selector, projection); self._sharedProjectionFn = LocalCollection._compileProjection( - sharedProjection); + self._sharedProjection); self._needToFetch = new LocalCollection._IdMap; self._currentlyFetching = null; @@ -333,8 +333,19 @@ _.extend(OplogObserveDriver.prototype, { _cursorForQuery: function () { var self = this; - // XXX this is WRONG we need to use the shared projection!! - return new Cursor(self._mongoHandle, self._cursorDescription); + + // The query we run is almost the same as the cursor we are observing, with + // a few changes. We need to read all the fields that are relevant to the + // selector, not just the fields we are going to publish (that's the + // "shared" projection). + var options = _.clone(self._cursorDescription.options); + options.fields = self._sharedProjection; + // We are NOT deep cloning fields or selector here, which should be OK. + var description = new CursorDescription( + self._cursorDescription.collectionName, + self._cursorDescription.selector, + options); + return new Cursor(self._mongoHandle, description); }, From 878cd8d4ae4ad28d7aa925242d950a73d06c2aae Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 00:34:16 -0800 Subject: [PATCH 013/112] Ensure new test passes with --disable-oplog --- packages/mongo-livedata/mongo_livedata_tests.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 7296864cd8..d270592870 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -1906,9 +1906,10 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - include selector field // Wait until we've processed the insert oplog entry. (If the insert shows up // during the observeChanges, the bug in question is not consistently - // reproduced.) - MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle.waitUntilCaughtUp(); - + // reproduced.) We don't have to do this for polling observe (eg + // --disable-oplog). + var oplog = MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle; + oplog && oplog.waitUntilCaughtUp(); var output = []; var handle = coll.find({a: 1, b: 2}, {fields: {c: 1}}).observeChanges({ From 28b8c541a9d7a9cf328021bdad9224bd85357ed7 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 00:53:55 -0800 Subject: [PATCH 014/112] oplog: Don't transform query results --- .../mongo-livedata/mongo_livedata_tests.js | 40 +++++++++++++++++++ .../mongo-livedata/oplog_observe_driver.js | 4 +- 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index d270592870..671924f816 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -1939,3 +1939,43 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - include selector field handle.stop(); }); + +Meteor.isServer && Tinytest.add("mongo-livedata - oplog - transform", function (test) { + var collName = "oplogTransform" + Random.id(); + var coll = new Meteor.Collection(collName); + + var docId = coll.insert({a: 25, x: {x: 5, y: 9}}); + test.isTrue(docId); + + // Wait until we've processed the insert oplog entry. (If the insert shows up + // during the observeChanges, the bug in question is not consistently + // reproduced.) We don't have to do this for polling observe (eg + // --disable-oplog). + var oplog = MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle; + oplog && oplog.waitUntilCaughtUp(); + + var cursor = coll.find({}, {transform: function (doc) { + return doc.x; + }}); + + var changesOutput = []; + var changesHandle = cursor.observeChanges({ + added: function (id, fields) { + changesOutput.push(['added', fields]); + } + }); + // We should get untransformed fields via observeChanges. + test.length(changesOutput, 1); + test.equal(changesOutput.shift(), ['added', {a: 25, x: {x: 5, y: 9}}]); + changesHandle.stop(); + + var transformedOutput = []; + var transformedHandle = cursor.observe({ + added: function (doc) { + transformedOutput.push(['added', doc]); + } + }); + test.length(transformedOutput, 1); + test.equal(transformedOutput.shift(), ['added', {x: 5, y: 9}]); + transformedHandle.stop(); +}); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 0f05c60f52..2222d68bfe 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -337,9 +337,11 @@ _.extend(OplogObserveDriver.prototype, { // The query we run is almost the same as the cursor we are observing, with // a few changes. We need to read all the fields that are relevant to the // selector, not just the fields we are going to publish (that's the - // "shared" projection). + // "shared" projection). And we don't want to apply any transform in the + // cursor, because observeChanges shouldn't use the transform. var options = _.clone(self._cursorDescription.options); options.fields = self._sharedProjection; + delete options.transform; // We are NOT deep cloning fields or selector here, which should be OK. var description = new CursorDescription( self._cursorDescription.collectionName, From dddc4739ccebfe692082fa026b4a0bf82e15e46c Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 03:18:43 -0800 Subject: [PATCH 015/112] History.md pass. --- .mailmap | 5 +++++ History.md | 37 +++++++++++++++++++++++++++++++++++-- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/.mailmap b/.mailmap index 6280f6b41f..588c82b80f 100644 --- a/.mailmap +++ b/.mailmap @@ -8,9 +8,13 @@ # For any emails that show up in the shortlog that aren't in one of # these lists, figure out their GitHub username and add them. +GITHUB: AlexeyMK GITHUB: ansman GITHUB: awwx GITHUB: codeinthehole +GITHUB: dandv +GITHUB: DenisGorbachev +GITHUB: FooBarWidget GITHUB: jacott GITHUB: Maxhodges GITHUB: meawoppl @@ -31,6 +35,7 @@ METEOR: estark37 METEOR: estark37 METEOR: glasser METEOR: gschmidt +METEOR: karayu METEOR: n1mmy METEOR: sixolet METEOR: Slava diff --git a/History.md b/History.md index 99a7b04596..587576268c 100644 --- a/History.md +++ b/History.md @@ -6,6 +6,8 @@ recommend using this precise version of Node in production so that the patch will be applied. If you use a newer version of Node with this version of Meteor, Meteor will not apply the patch and will instead disable websockets. +* XXX oplog tailing + * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded message at DDP startup. Now the hot code push only triggers when @@ -24,7 +26,35 @@ Meteor will not apply the patch and will instead disable websockets. * Support `EJSON.clone` for `Meteor.Error`. As a result, they are properly stringified in DDP even if thrown through a `Future`. #1482 -* Fail explicitly when publishing non-cursors. +* XXX Fix error on this.removed during session shutdown. 9e7b4bd + +* XXX fix issue with logoutOtherConnections 5afd0d5 #1540 #1553 + +* XXX sighup for clean webapp shutdown. also connection keepalive 5s change. + +* XXX Fail explicitly when publishing non-cursors. + +* XXX Introduce '--raw-logs' option to `meteor run` to disable logs parsing. ac376b6 + +* XXX fix install script to create /usr/local on mavericks b20c2c6 (not really part of the release) + +* XXX asking for password on stdout so url can be used in scripts 60f88dc + +* XXX set x-forwarded-* headers in 'meteor run' 2b5e32 + +* XXX Clean up package dirs containing only ".build" e11228a + +* XXX Properly handle projections where '_id' is the only rule. df95f1e + +* XXX Fix 0.6.6 regression in setting MAIL_URL 6582a88 + +* XXX Only count files that actually go in the cache in the cache size check. #1653. + +* XXX Fix so that it is really possible to pass null to disable transformation in validators #1659 + +* XXX Remove unused line, fix incompatibility with Phusion Passenger 8ca70e9 + +* XXX Check for matching hostname before doing end-of-oauth redirect a2b0fff * Implement `$each`, `$sort`, and `$slice` options for minimongo's `$push` modifier. #1492 @@ -37,8 +67,11 @@ Meteor will not apply the patch and will instead disable websockets. * Upgraded dependencies: * SockJS server from 0.3.7 to 0.3.8 * Node from 0.10.21 to 0.10.22 + * MongoDB from 2.4.6 to 2.4.8 + * Websocket driver from XXX to YYY -Patches contributed by GitHub users awwx, mcbain, rzymek. +Patches contributed by GitHub users AlexeyMK, awwx, dandv, +DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, sdarnell. ## v0.6.6.3 From 49b7e4529529e924d9c45a62a707fbeb6f0a6c1d Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 03:26:13 -0800 Subject: [PATCH 016/112] Package metadata updates. Make autopublish internal, as it is part of standard-app-packages and not something we want users to think about. Tweak the string for facts package. 'and custom' was a bit awkward. --- packages/autoupdate/package.js | 3 ++- packages/facts/package.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/autoupdate/package.js b/packages/autoupdate/package.js index e350789346..f56a572b1a 100644 --- a/packages/autoupdate/package.js +++ b/packages/autoupdate/package.js @@ -1,5 +1,6 @@ Package.describe({ - summary: "Update the client when new client code is available" + summary: "Update the client when new client code is available", + internal: true }); Package.on_use(function (api) { diff --git a/packages/facts/package.js b/packages/facts/package.js index f3aafae09d..1f835c15fe 100644 --- a/packages/facts/package.js +++ b/packages/facts/package.js @@ -1,5 +1,5 @@ Package.describe({ - summary: "Publish internal and custom app statistics" + summary: "Publish internal app statistics" }); Package.on_use(function (api) { From 23fb2e3b4bc64ca60b79617d2a0746290a5afa7b Mon Sep 17 00:00:00 2001 From: emgee3 Date: Wed, 11 Dec 2013 02:18:22 -0800 Subject: [PATCH 017/112] fix broken url in LICENSE.txt --- LICENSE.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE.txt b/LICENSE.txt index 67bba7d36c..ce588a1a86 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -287,7 +287,7 @@ archy: https://github.com/substack/node-archy shell-quote: https://github.com/substack/node-shell-quote deep-equal: https://github.com/substack/node-deep-equal editor: https://github.com/substack/node-editor -minimist: https://github.com/substack/node-minimist +minimist: https://github.com/substack/minimist quotemeta: https://github.com/substack/quotemeta ---------- From 28f3ec0073969296751bd3525b5b21458e458fb4 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 10:57:47 -0800 Subject: [PATCH 018/112] mailmap update --- .mailmap | 1 + 1 file changed, 1 insertion(+) diff --git a/.mailmap b/.mailmap index 588c82b80f..40bbb70898 100644 --- a/.mailmap +++ b/.mailmap @@ -14,6 +14,7 @@ GITHUB: awwx GITHUB: codeinthehole GITHUB: dandv GITHUB: DenisGorbachev +GITHUB: emgee3 GITHUB: FooBarWidget GITHUB: jacott GITHUB: Maxhodges From ff7c35c5a3781bce67c4c0a6c2f69b1cf717ad82 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 13:50:24 -0800 Subject: [PATCH 019/112] "drop collection" support in oplog tailing This version should be more correct than the previous implementation. --- packages/mongo-livedata/collection.js | 6 ++ packages/mongo-livedata/mongo_driver.js | 29 +++++-- .../mongo-livedata/mongo_livedata_tests.js | 64 ++++++++++++++ .../mongo-livedata/oplog_observe_driver.js | 86 +++++++++++-------- .../remote_collection_driver.js | 3 +- 5 files changed, 145 insertions(+), 43 deletions(-) diff --git a/packages/mongo-livedata/collection.js b/packages/mongo-livedata/collection.js index ea68480470..1a2c1315db 100644 --- a/packages/mongo-livedata/collection.js +++ b/packages/mongo-livedata/collection.js @@ -489,6 +489,12 @@ Meteor.Collection.prototype._dropIndex = function (index) { throw new Error("Can only call _dropIndex on server collections"); self._collection._dropIndex(index); }; +Meteor.Collection.prototype._dropCollection = function () { + var self = this; + if (!self._collection.dropCollection) + throw new Error("Can only call _dropCollection on server collections"); + self._collection.dropCollection(); +}; Meteor.Collection.prototype._createCappedCollection = function (byteSize) { var self = this; if (!self._collection._createCappedCollection) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index bb258e55dd..948fcc397f 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -280,7 +280,7 @@ MongoConnection.prototype._insert = function (collection_name, document, var write = self._maybeBeginWrite(); var refresh = function () { - Meteor.refresh({ collection: collection_name, id: document._id }); + Meteor.refresh({collection: collection_name, id: document._id }); }; callback = bindEnvironmentForWrite(writeCallback(write, refresh, callback)); try { @@ -341,6 +341,25 @@ MongoConnection.prototype._remove = function (collection_name, selector, } }; +MongoConnection.prototype._dropCollection = function (collectionName, cb) { + var self = this; + + var write = self._maybeBeginWrite(); + var refresh = function () { + Meteor.refresh({collection: collectionName, id: null, + dropCollection: true}); + }; + cb = bindEnvironmentForWrite(writeCallback(write, refresh, cb)); + + try { + var collection = self._getCollection(collectionName); + collection.drop(cb); + } catch (e) { + write.committed(); + throw e; + } +}; + MongoConnection.prototype._update = function (collection_name, selector, mod, options, callback) { var self = this; @@ -536,7 +555,7 @@ var simulateUpsertWithInsertedId = function (collection, selector, mod, doUpdate(); }; -_.each(["insert", "update", "remove"], function (method) { +_.each(["insert", "update", "remove", "dropCollection"], function (method) { MongoConnection.prototype[method] = function (/* arguments */) { var self = this; return Meteor._wrapAsync(self["_" + method]).apply(self, arguments); @@ -993,10 +1012,6 @@ MongoConnection.prototype._observeChanges = function ( listenAll = function (cursorDescription, listenCallback) { var listeners = []; forEachTrigger(cursorDescription, function (trigger) { - // The "drop collection" event is used by the oplog crossbar, not the - // invalidation crossbar. - if (trigger.dropCollection) - return; listeners.push(DDPServer._InvalidationCrossbar.listen( trigger, listenCallback)); }); @@ -1018,7 +1033,7 @@ forEachTrigger = function (cursorDescription, triggerCallback) { _.each(specificIds, function (id) { triggerCallback(_.extend({id: id}, key)); }); - triggerCallback(_.extend({dropCollection: true}, key)); + triggerCallback(_.extend({dropCollection: true, id: null}, key)); } else { triggerCallback(key); } diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 671924f816..750d006122 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -1979,3 +1979,67 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - transform", function ( test.equal(transformedOutput.shift(), ['added', {x: 5, y: 9}]); transformedHandle.stop(); }); + + +Meteor.isServer && Tinytest.add("mongo-livedata - oplog - drop collection", function (test) { + var collName = "dropCollection" + Random.id(); + var coll = new Meteor.Collection(collName); + + var doc1Id = coll.insert({a: 'foo', c: 1}); + var doc2Id = coll.insert({b: 'bar'}); + var doc3Id = coll.insert({a: 'foo', c: 2}); + var tmp; + + var output = []; + var handle = coll.find({a: 'foo'}).observeChanges({ + added: function (id, fields) { + output.push(['added', id, fields]); + }, + changed: function (id) { + output.push(['changed']); + }, + removed: function (id) { + output.push(['removed', id]); + } + }); + test.length(output, 2); + // make order consistent + if (output.length === 2 && output[0][1] === doc3Id) { + tmp = output[0]; + output[0] = output[1]; + output[1] = tmp; + } + test.equal(output.shift(), ['added', doc1Id, {a: 'foo', c: 1}]); + test.equal(output.shift(), ['added', doc3Id, {a: 'foo', c: 2}]); + + // Wait until we've processed the insert oplog entry, so that we are in a + // steady state (and we don't see the dropped docs because we are FETCHING). + var oplog = MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle; + oplog && oplog.waitUntilCaughtUp(); + + // Drop the collection. Should remove all docs. + runInFence(function () { + coll._dropCollection(); + }); + + test.length(output, 2); + // make order consistent + if (output.length === 2 && output[0][1] === doc3Id) { + tmp = output[0]; + output[0] = output[1]; + output[1] = tmp; + } + test.equal(output.shift(), ['removed', doc1Id]); + test.equal(output.shift(), ['removed', doc3Id]); + + // Put something back in. + var doc4Id; + runInFence(function () { + doc4Id = coll.insert({a: 'foo', c: 3}); + }); + + test.length(output, 1); + test.equal(output.shift(), ['added', doc4Id, {a: 'foo', c: 3}]); + + handle.stop(); +}); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 2222d68bfe..ec3f421379 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -14,7 +14,6 @@ var PHASE = { // it by calling the stop() method. OplogObserveDriver = function (options) { var self = this; - self._usesOplog = true; // tests look at this self._cursorDescription = options.cursorDescription; @@ -47,17 +46,18 @@ OplogObserveDriver = function (options) { self._needToFetch = new LocalCollection._IdMap; self._currentlyFetching = null; + self._requeryWhenDoneThisQuery = false; self._writesToCommitWhenWeReachSteady = []; forEachTrigger(self._cursorDescription, function (trigger) { self._stopHandles.push(self._mongoHandle._oplogHandle.onOplogEntry( trigger, function (notification) { var op = notification.op; - if (op.op === 'c') { - // XXX actually, drop collection needs to be handled by doing a - // re-query - self._published.forEach(function (fields, id) { - self._remove(id); + if (notification.dropCollection) { + // Defer because it may block on "wait for oplog to catch up", which + // isn't kosher for an oplog entry handler (will cause deadlock). + Meteor.defer(function () { + self._needToPollQuery(); }); } else { // All other operators should be handled depending on phase @@ -82,21 +82,23 @@ OplogObserveDriver = function (options) { var write = fence.beginWrite(); // This write cannot complete until we've caught up to "this point" in the // oplog, and then made it back to the steady state. - Meteor.defer(complete); - self._mongoHandle._oplogHandle.waitUntilCaughtUp(); - if (self._stopped) { - // We're stopped, so just immediately commit. - write.committed(); - } else if (self._phase === PHASE.STEADY) { - // Make sure that all of the callbacks have made it through the - // multiplexer and been delivered to ObserveHandles before committing - // writes. - self._multiplexer.onFlush(function () { + Meteor.defer(function () { + self._mongoHandle._oplogHandle.waitUntilCaughtUp(); + if (self._stopped) { + // We're stopped, so just immediately commit. write.committed(); - }); - } else { - self._writesToCommitWhenWeReachSteady.push(write); - } + } else if (self._phase === PHASE.STEADY) { + // Make sure that all of the callbacks have made it through the + // multiplexer and been delivered to ObserveHandles before committing + // writes. + self._multiplexer.onFlush(function () { + write.committed(); + }); + } else { + self._writesToCommitWhenWeReachSteady.push(write); + } + }); + complete(); } )); @@ -176,6 +178,8 @@ _.extend(OplogObserveDriver.prototype, { if (!anyError) anyError = err; } else if (!self._stopped && self._phase === PHASE.FETCHING) { + // XXX this is bad, what if we go into fetching again after + // coming back from _pollQuery? // We re-check the phase in case we've had an explicit _pollQuery // call which pulls us out of FETCHING and back into QUERYING. self._handleDoc(id, doc); @@ -284,20 +288,12 @@ _.extend(OplogObserveDriver.prototype, { _pollQuery: function () { var self = this; - // XXX maybe this should just return? if (self._stopped) - throw new Error("can't re-poll a stopped query"); + return; - // XXX maybe this should either just return, or queue up another poll - // somehow? - if (self._phase === PHASE.QUERYING) - throw new Error("can't re-poll re-entrantly"); - - if (self._phase === PHASE.FETCHING) { - // Yay, we get to forget about all the things we thought we had to fetch. - self._needToFetch = new LocalCollection._IdMap; - self._currentlyFetching = null; - } + // Yay, we get to forget about all the things we thought we had to fetch. + self._needToFetch = new LocalCollection._IdMap; + self._currentlyFetching = null; self._phase = PHASE.QUERYING; // subtle note: _published does not contain _id fields, but newResults does @@ -312,6 +308,23 @@ _.extend(OplogObserveDriver.prototype, { self._doneQuerying(); }, + _needToPollQuery: function () { + var self = this; + if (self._stopped) + return; + + // If we're not already in the middle of a query, we can query now (possibly + // pausing FETCHING). + if (self._phase !== PHASE.QUERYING) { + self._pollQuery(); + return; + } + + // We're currently in QUERYING. Set a flag to ensure that we run another + // query when we're done. + self._requeryWhenDoneThisQuery = true; + }, + _doneQuerying: function () { var self = this; @@ -324,7 +337,10 @@ _.extend(OplogObserveDriver.prototype, { if (self._phase !== PHASE.QUERYING) throw Error("Phase unexpectedly " + self._phase); - if (self._needToFetch.empty()) { + if (self._requeryWhenDoneThisQuery) { + self._requeryWhenDoneThisQuery = false; + self._pollQuery(); + } else if (self._needToFetch.empty()) { self._beSteady(); } else { self._fetchModifiedDocuments(); @@ -363,7 +379,7 @@ _.extend(OplogObserveDriver.prototype, { // First remove anything that's gone. Be careful not to modify // self._published while iterating over it. var idsToRemove = []; - self._published.each(function (doc, id) { + self._published.forEach(function (doc, id) { if (!newResults.has(id)) idsToRemove.push(id); }); @@ -372,7 +388,7 @@ _.extend(OplogObserveDriver.prototype, { }); // Now do adds and changes. - newResults.each(function (doc, id) { + newResults.forEach(function (doc, id) { // "true" here means to throw if we think this doc doesn't match the // selector. self._handleDoc(id, doc, true); diff --git a/packages/mongo-livedata/remote_collection_driver.js b/packages/mongo-livedata/remote_collection_driver.js index b56607e9f8..6cc07c7fef 100644 --- a/packages/mongo-livedata/remote_collection_driver.js +++ b/packages/mongo-livedata/remote_collection_driver.js @@ -10,7 +10,8 @@ _.extend(MongoInternals.RemoteCollectionDriver.prototype, { var ret = {}; _.each( ['find', 'findOne', 'insert', 'update', , 'upsert', - 'remove', '_ensureIndex', '_dropIndex', '_createCappedCollection'], + 'remove', '_ensureIndex', '_dropIndex', '_createCappedCollection', + 'dropCollection'], function (m) { ret[m] = _.bind(self.mongo[m], self.mongo, name); }); From 8c5ebca0d9e4feb0ac3b46edee89cce3d92f8e88 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 15:07:15 -0800 Subject: [PATCH 020/112] Ensure that we ignore old fetches when we repoll --- packages/mongo-livedata/oplog_observe_driver.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index ec3f421379..8fbc050d35 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -45,6 +45,7 @@ OplogObserveDriver = function (options) { self._needToFetch = new LocalCollection._IdMap; self._currentlyFetching = null; + self._fetchGeneration = 0; self._requeryWhenDoneThisQuery = false; self._writesToCommitWhenWeReachSteady = []; @@ -163,6 +164,7 @@ _.extend(OplogObserveDriver.prototype, { throw new Error("phase in fetchModifiedDocuments: " + self._phase); self._currentlyFetching = self._needToFetch; + var thisGeneration = ++self._fetchGeneration; self._needToFetch = new LocalCollection._IdMap; var waiting = 0; var anyError = null; @@ -177,11 +179,11 @@ _.extend(OplogObserveDriver.prototype, { if (err) { if (!anyError) anyError = err; - } else if (!self._stopped && self._phase === PHASE.FETCHING) { - // XXX this is bad, what if we go into fetching again after - // coming back from _pollQuery? - // We re-check the phase in case we've had an explicit _pollQuery - // call which pulls us out of FETCHING and back into QUERYING. + } else if (!self._stopped && self._phase === PHASE.FETCHING + && self._fetchGeneration === thisGeneration) { + // We re-check the generation in case we've had an explicit + // _pollQuery call which should effectively cancel this round of + // fetches. (_pollQuery increments the generation.) self._handleDoc(id, doc); } waiting--; @@ -294,6 +296,7 @@ _.extend(OplogObserveDriver.prototype, { // Yay, we get to forget about all the things we thought we had to fetch. self._needToFetch = new LocalCollection._IdMap; self._currentlyFetching = null; + ++self._fetchGeneration; // ignore any in-flight fetches self._phase = PHASE.QUERYING; // subtle note: _published does not contain _id fields, but newResults does From 8de5ebfc6fda4865a13b6dbc3367a761ed98baed Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 17:04:06 -0800 Subject: [PATCH 021/112] Become able to kill mongod spawned by pre-oplog branches --- tools/mongo_runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index e92e54a7fd..40d0a39bf8 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -25,7 +25,7 @@ var find_mongo_pids = function (app_dir, port, callback) { _.each(stdout.split('\n'), function (ps_line) { // matches mongos we start. - var m = ps_line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db /); + var m = ps_line.match(/^\s*(\d+).+mongod .+--port (\d+) --dbpath (.+)(?:\/|\\)\.meteor(?:\/|\\)local(?:\/|\\)db(?: |$)/); if (m && m.length === 4) { var found_pid = parseInt(m[1]); var found_port = parseInt(m[2]); From 8de0ae58dc42ffb1e2e85022913995ec3f168905 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 17:42:04 -0800 Subject: [PATCH 022/112] tinytest: test.equal should use EJSON.equals --- packages/tinytest/tinytest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tinytest/tinytest.js b/packages/tinytest/tinytest.js index 4ef0ac689f..66c6d42063 100644 --- a/packages/tinytest/tinytest.js +++ b/packages/tinytest/tinytest.js @@ -132,7 +132,7 @@ _.extend(TestCaseResults.prototype, { this.equal(actual[i], expected[i]); } } else { - matched = _.isEqual(expected, actual); + matched = EJSON.equals(expected, actual); } if (matched === !!not) { From 13c9de24b3e697688f0283cafcfc8a088e3db01a Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Wed, 11 Dec 2013 17:48:30 -0800 Subject: [PATCH 023/112] Upgrade clean-css to 2.0.2 --- History.md | 1 + packages/minifiers/.npm/package/npm-shrinkwrap.json | 2 +- packages/minifiers/minifiers.js | 8 +++++++- packages/minifiers/package.js | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/History.md b/History.md index 587576268c..e57901a09f 100644 --- a/History.md +++ b/History.md @@ -69,6 +69,7 @@ Meteor will not apply the patch and will instead disable websockets. * Node from 0.10.21 to 0.10.22 * MongoDB from 2.4.6 to 2.4.8 * Websocket driver from XXX to YYY + * clean-css from 1.1.2 to 2.0.2 Patches contributed by GitHub users AlexeyMK, awwx, dandv, DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, sdarnell. diff --git a/packages/minifiers/.npm/package/npm-shrinkwrap.json b/packages/minifiers/.npm/package/npm-shrinkwrap.json index b23b2cea61..38aac34aa9 100644 --- a/packages/minifiers/.npm/package/npm-shrinkwrap.json +++ b/packages/minifiers/.npm/package/npm-shrinkwrap.json @@ -1,7 +1,7 @@ { "dependencies": { "clean-css": { - "version": "1.1.2", + "version": "2.0.2", "dependencies": { "commander": { "version": "2.0.0" diff --git a/packages/minifiers/minifiers.js b/packages/minifiers/minifiers.js index efd8ce0a30..ad15487d26 100644 --- a/packages/minifiers/minifiers.js +++ b/packages/minifiers/minifiers.js @@ -1,2 +1,8 @@ -CleanCSSProcess = Npm.require('clean-css').process; +var CleanCss = Npm.require('clean-css'); + +CleanCSSProcess = function (source, options) { + var instance = new CleanCss(options); + return instance.minify(source); +}; + UglifyJSMinify = Npm.require('uglify-js').minify; diff --git a/packages/minifiers/package.js b/packages/minifiers/package.js index 6a3eb50d59..858ce9c0fb 100644 --- a/packages/minifiers/package.js +++ b/packages/minifiers/package.js @@ -4,7 +4,7 @@ Package.describe({ }); Npm.depends({ - "clean-css": "1.1.2", + "clean-css": "2.0.2", // Fork of 2.4.0 fixing https://github.com/mishoo/UglifyJS2/pull/308 "uglify-js": "https://github.com/meteor/UglifyJS2/tarball/bb0a762d12d2ecd058b9d7b57f16b4c289378d9c" }); From 930727d426a0ccdefbb381e57a615c31adc4922c Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 18:23:40 -0800 Subject: [PATCH 024/112] Some more real words for History.md --- History.md | 79 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/History.md b/History.md index e57901a09f..0dc2c12316 100644 --- a/History.md +++ b/History.md @@ -8,16 +8,46 @@ Meteor will not apply the patch and will instead disable websockets. * XXX oplog tailing +* Add `Meteor.onConnection` and add `this.connection` to method + invocations and publish functions. These can be used to store data + associated with individual clients between subscriptions and method + calls. See http://docs.meteor.com/#meteor_onconnection for details. + * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded message at DDP startup. Now the hot code push only triggers when client code changes; server only code changes will not cause the page to reload. -* Add `Meteor.onConnection` and add `this.connection` to method - invocations and publish functions. These can be used to store data - associated with individual clients between subscriptions and method - calls. See http://docs.meteor.com/#meteor_onconnection for details. +* New 'facts' package publishes internal statistics about Meteor. + XXX how to use + +* Add an explicit check that publish functions return a cursor, an array + of cursors, or a falsey value. This is a safety check to to prevent + users from accidentally returning Collection.findOne() or some other + value and expecting it to be published. + +* Implement `$each`, `$sort`, and `$slice` options for minimongo's `$push` + modifier. #1492 + +* Introduce '--raw-logs' option to `meteor run` to disable log + coloring and timestamps. + +* Add `WebAppInternals.setBundledJsCssPrefix()` to control where the + client loads bundled JavaScript and CSS files. This allows serving + files from a CDN to decrease page load times and reduce server load. + +* Attempt to exit cleanly on 'SIGHUP'. Stop accepting incoming + connections, kill DDP connections, and finish all outstanding requests + for static assets. + +* Fix handling of `fields` option in minimongo when only `_id` is present. #1651 + +* Fix issue where setting `process.env.MAIL_URL` in app code would not + alter where mail was sent. This was a regression from 0.6.6. #1649 + +* Prompt for passwords on stderr instead of stdout for easier automation + in shell scripts. #1600 * Bundler failures cause non-zero exit code in `meteor run`. #1515 @@ -26,49 +56,32 @@ Meteor will not apply the patch and will instead disable websockets. * Support `EJSON.clone` for `Meteor.Error`. As a result, they are properly stringified in DDP even if thrown through a `Future`. #1482 -* XXX Fix error on this.removed during session shutdown. 9e7b4bd +* Fix passing `transform: null` option to `Collection.find()` to disable + transformation in validators. #1659 -* XXX fix issue with logoutOtherConnections 5afd0d5 #1540 #1553 +* Fix livedata error on `this.removed` during session shutdown. #1540 #1553 -* XXX sighup for clean webapp shutdown. also connection keepalive 5s change. +* Fix incompatibility with Phusion Passenger by removing an unused line. #1613 -* XXX Fail explicitly when publishing non-cursors. +* Ensure install script creates /usr/local on machines where it does not + exist (eg. fresh install of OSX Mavericks). -* XXX Introduce '--raw-logs' option to `meteor run` to disable logs parsing. ac376b6 +* Set x-forwarded-* headers in `meteor run`. -* XXX fix install script to create /usr/local on mavericks b20c2c6 (not really part of the release) +* Clean up package dirs containing only ".build". -* XXX asking for password on stdout so url can be used in scripts 60f88dc +* Check for matching hostname before doing end-of-oauth redirect. -* XXX set x-forwarded-* headers in 'meteor run' 2b5e32 - -* XXX Clean up package dirs containing only ".build" e11228a - -* XXX Properly handle projections where '_id' is the only rule. df95f1e - -* XXX Fix 0.6.6 regression in setting MAIL_URL 6582a88 - -* XXX Only count files that actually go in the cache in the cache size check. #1653. - -* XXX Fix so that it is really possible to pass null to disable transformation in validators #1659 - -* XXX Remove unused line, fix incompatibility with Phusion Passenger 8ca70e9 - -* XXX Check for matching hostname before doing end-of-oauth redirect a2b0fff - -* Implement `$each`, `$sort`, and `$slice` options for minimongo's `$push` - modifier. #1492 +* Only count files that actually go in the cache towards the `appcache` + size check. #1653. * Increase the maximum size spiderable will return for a page from 200kB to 5MB. -* New 'facts' package publishes internal statistics about Meteor. - * Upgraded dependencies: - * SockJS server from 0.3.7 to 0.3.8 + * SockJS server from 0.3.7 to 0.3.8, including new faye-websocket module. * Node from 0.10.21 to 0.10.22 * MongoDB from 2.4.6 to 2.4.8 - * Websocket driver from XXX to YYY * clean-css from 1.1.2 to 2.0.2 Patches contributed by GitHub users AlexeyMK, awwx, dandv, From 506fb6a552feb86a2525ee1aa98f915f79aaff84 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 18:57:30 -0800 Subject: [PATCH 025/112] Also upgrade uglify-js. --- History.md | 1 + packages/minifiers/.npm/package/npm-shrinkwrap.json | 6 +++--- packages/minifiers/package.js | 3 +-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/History.md b/History.md index 0dc2c12316..3306e46bd7 100644 --- a/History.md +++ b/History.md @@ -83,6 +83,7 @@ Meteor will not apply the patch and will instead disable websockets. * Node from 0.10.21 to 0.10.22 * MongoDB from 2.4.6 to 2.4.8 * clean-css from 1.1.2 to 2.0.2 + * uglify-js from a fork of 2.4.0 to 2.4.7 Patches contributed by GitHub users AlexeyMK, awwx, dandv, DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, sdarnell. diff --git a/packages/minifiers/.npm/package/npm-shrinkwrap.json b/packages/minifiers/.npm/package/npm-shrinkwrap.json index 38aac34aa9..e2eb3b72aa 100644 --- a/packages/minifiers/.npm/package/npm-shrinkwrap.json +++ b/packages/minifiers/.npm/package/npm-shrinkwrap.json @@ -9,16 +9,16 @@ } }, "uglify-js": { - "from": "https://github.com/meteor/UglifyJS2/tarball/bb0a762d12d2ecd058b9d7b57f16b4c289378d9c", + "version": "2.4.7", "dependencies": { "async": { "version": "0.2.9" }, "source-map": { - "version": "0.1.30", + "version": "0.1.31", "dependencies": { "amdefine": { - "version": "0.0.8" + "version": "0.1.0" } } }, diff --git a/packages/minifiers/package.js b/packages/minifiers/package.js index 858ce9c0fb..c76efd3ee9 100644 --- a/packages/minifiers/package.js +++ b/packages/minifiers/package.js @@ -5,8 +5,7 @@ Package.describe({ Npm.depends({ "clean-css": "2.0.2", - // Fork of 2.4.0 fixing https://github.com/mishoo/UglifyJS2/pull/308 - "uglify-js": "https://github.com/meteor/UglifyJS2/tarball/bb0a762d12d2ecd058b9d7b57f16b4c289378d9c" + "uglify-js": "2.4.7" }); Package.on_use(function (api) { From 04fddf3a7738d1140ec6acba731cffe5adc7c152 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 20:38:51 -0800 Subject: [PATCH 026/112] Update tools tests for Webapp bundled asset change 6eccf8c --- tools/tests/test_bundler_options.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/tests/test_bundler_options.js b/tools/tests/test_bundler_options.js index b3affcbe0c..76a5b0a382 100644 --- a/tools/tests/test_bundler_options.js +++ b/tools/tests/test_bundler_options.js @@ -34,8 +34,8 @@ assert.doesNotThrow(function () { // verify that contents are minified var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html"), 'utf8'); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml)); - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/packages/.test(appHtml))); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml)); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages/.test(appHtml))); }); console.log("nodeModules: 'skip', no minify"); @@ -50,11 +50,11 @@ assert.doesNotThrow(function () { // verify that contents are not minified var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html"), 'utf8'); - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml))); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/meteor/.test(appHtml)); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/deps/.test(appHtml)); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml))); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/meteor/.test(appHtml)); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/deps/.test(appHtml)); // verify that tests aren't included - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/package-tests\/meteor/.test(appHtml))); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/package-tests\/meteor/.test(appHtml))); }); console.log("nodeModules: 'skip', no minify, testPackages: ['meteor']"); @@ -70,7 +70,7 @@ assert.doesNotThrow(function () { // verify that tests for the meteor package are included var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html")); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/meteor:tests\.js/.test(appHtml)); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/meteor:tests\.js/.test(appHtml)); }); console.log("nodeModules: 'copy'"); From dacc53f16e88d9177433406e9718045bb01ed443 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 23:57:09 -0800 Subject: [PATCH 027/112] First pass text for oplog tailing in History.md. --- History.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index 3306e46bd7..f13a291d3b 100644 --- a/History.md +++ b/History.md @@ -6,7 +6,17 @@ recommend using this precise version of Node in production so that the patch will be applied. If you use a newer version of Node with this version of Meteor, Meteor will not apply the patch and will instead disable websockets. -* XXX oplog tailing +* Rework how Meteor gets realtime database updates from MongoDB. Meteor + now reads the MongoDB "oplog" -- a special collection that records all + the write operations as they are applied to your database. This means + changes to the database are instantly noticed and reflected in Meteor, + whether they originated from Meteor or from an external database + client. Oplog tailing is automatically enabled in development mode + with `meteor run`, and can be enabled in production with the + `MONGO_OPLOG_URL` environment variable. Currently the only supported + selectors are equality checks; `$`-operators, `limit` and `skip` + queries fall back to the original poll-and-diff algorithm. See for details. * Add `Meteor.onConnection` and add `this.connection` to method invocations and publish functions. These can be used to store data From 738ffe50a13dc227b5cdd25d5a7b1686bb950212 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 23:37:16 -0800 Subject: [PATCH 028/112] fix missing expect() call in password-tests add a console.trace that helps debug it --- packages/accounts-password/password_tests.js | 4 ++-- packages/tinytest/tinytest.js | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/accounts-password/password_tests.js b/packages/accounts-password/password_tests.js index 1da7dff9a4..fc71e0d317 100644 --- a/packages/accounts-password/password_tests.js +++ b/packages/accounts-password/password_tests.js @@ -464,10 +464,10 @@ if (Meteor.isClient) (function () { function (test, expect) { var self = this; // Test that deleting a user logs out that user's connections. - Meteor.loginWithPassword(this.username, this.password, function (err) { + Meteor.loginWithPassword(this.username, this.password, expect(function (err) { test.isFalse(err); Meteor.call("removeUser", self.username); - }); + })); }, waitForLoggedOutStep ]); diff --git a/packages/tinytest/tinytest.js b/packages/tinytest/tinytest.js index 66c6d42063..de212efb50 100644 --- a/packages/tinytest/tinytest.js +++ b/packages/tinytest/tinytest.js @@ -309,7 +309,16 @@ _.extend(TestCase.prototype, { return true; }; - var results = new TestCaseResults(self, onEvent, + var wrappedOnEvent = function (e) { + // If this trace prints, it means you ran some test.* function after the + // test finished! Another symptom will be that the test will display as + // "waiting" even when it counts as passed or failed. + if (completed) + console.trace("event after complete!"); + return onEvent(e); + }; + + var results = new TestCaseResults(self, wrappedOnEvent, function (e) { if (markComplete()) onException(e); From c58d4d158b8439ef80535370c9980eca5fdebd12 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 18:11:20 -0800 Subject: [PATCH 029/112] Handle modifications to EJSON custom types OplogObserveDriver cannot try to directly apply these representation-level mutations; it needs to get the entire custom type. This implementation is overly conservative, but in practice Meteor-originated writes shouldn't mutate EJSON custom types anyway (they should treat them as atomic). --- .../mongo-livedata/mongo_livedata_tests.js | 100 ++++++++++++++++++ .../mongo-livedata/oplog_observe_driver.js | 21 +++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 750d006122..299d34bbd6 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -2043,3 +2043,103 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - drop collection", func handle.stop(); }); + +var TestCustomType = function (head, tail) { + // use different field names on the object than in JSON, to ensure we are + // actually treating this as an opaque object. + this.myHead = head; + this.myTail = tail; +}; +_.extend(TestCustomType.prototype, { + clone: function () { + return new TestCustomType(this.myHead, this.myTail); + }, + equals: function (other) { + return other instanceof TestCustomType + && EJSON.equals(this.myHead, other.myHead) + && EJSON.equals(this.myTail, other.myTail); + }, + typeName: function () { + return 'someCustomType'; + }, + toJSONValue: function () { + return {head: this.myHead, tail: this.myTail}; + } +}); + +EJSON.addType('someCustomType', function (json) { + return new TestCustomType(json.head, json.tail); +}); + +testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ + function (test, expect) { + var self = this; + var collectionName = "ejson" + Random.id(); + if (Meteor.isClient) { + Meteor.call('createInsecureCollection', collectionName); + Meteor.subscribe('c-' + collectionName); + } + + self.collection = new Meteor.Collection(collectionName); + + self.id = self.collection.insert( + {name: 'foo', custom: new TestCustomType('a', 'b')}, + expect(function (err, res) { + test.isFalse(err); + test.equal(self.id, res); + })); + }, + function (test, expect) { + var self = this; + self.changes = []; + self.handle = self.collection.find({}).observeChanges({ + added: function (id, fields) { + self.changes.push(['a', id, fields]); + }, + changed: function (id, fields) { + self.changes.push(['c', id, fields]); + }, + removed: function (id) { + self.changes.push(['r', id]); + } + }); + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['a', self.id, + {name: 'foo', custom: new TestCustomType('a', 'b')}]); + + // First, replace the entire custom object. + // (runInFence is useful for the server, using expect() is useful for the + // client) + runInFence(function () { + self.collection.update( + self.id, {$set: {custom: new TestCustomType('a', 'c')}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {custom: new TestCustomType('a', 'c')}]); + + // Now, sneakily replace just a piece of it. Meteor won't do this, but + // perhaps you are accessing Mongo directly. + runInFence(function () { + self.collection.update( + self.id, {$set: {'custom.EJSON$value.EJSONtail': 'd'}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {custom: new TestCustomType('a', 'd')}]); + self.handle.stop(); + } +]); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 8fbc050d35..7e3da7d1f1 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -247,18 +247,25 @@ _.extend(OplogObserveDriver.prototype, { // replacement (in which case we can just directly re-evaluate the // selector)? var isReplace = !_.has(op.o, '$set') && !_.has(op.o, '$unset'); + // If this modifier modifies something inside an EJSON custom type (ie, + // anything with EJSON$), then we can't try to use + // LocalCollection._modify, since that just mutates the EJSON encoding, + // not the actual object. + var canDirectlyModifyDoc = + !isReplace && modifierCanBeDirectlyApplied(op.o); if (isReplace) { self._handleDoc(id, _.extend({_id: id}, op.o)); - } else if (self._published.has(id)) { + } else if (self._published.has(id) && canDirectlyModifyDoc) { // Oh great, we actually know what the document is, so we can apply // this directly. var newDoc = EJSON.clone(self._published.get(id)); newDoc._id = id; LocalCollection._modify(newDoc, op.o); self._handleDoc(id, self._sharedProjectionFn(newDoc)); - } else if (LocalCollection._canSelectorBecomeTrueByModifier( - self._cursorDescription.selector, op.o)) { + } else if (!canDirectlyModifyDoc || + LocalCollection._canSelectorBecomeTrueByModifier( + self._cursorDescription.selector, op.o)) { self._needToFetch.set(id, op.ts.toString()); if (self._phase === PHASE.STEADY) self._fetchModifiedDocuments(); @@ -480,4 +487,12 @@ OplogObserveDriver.cursorSupported = function (cursorDescription) { }); }; +var modifierCanBeDirectlyApplied = function (modifier) { + return _.all(modifier, function (fields, operation) { + return _.all(fields, function (value, field) { + return !/EJSON\$/.test(field); + }); + }); +}; + MongoTest.OplogObserveDriver = OplogObserveDriver; From 7e02b83708dcca0f9662af2242573269a2c2e835 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 00:38:20 -0800 Subject: [PATCH 030/112] Test changing Date and ObjectID --- .../mongo-livedata/mongo_livedata_tests.js | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 299d34bbd6..1f163b4e4b 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -2071,7 +2071,7 @@ EJSON.addType('someCustomType', function (json) { return new TestCustomType(json.head, json.tail); }); -testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ +testAsyncMulti("mongo-livedata - oplog - update EJSON", [ function (test, expect) { var self = this; var collectionName = "ejson" + Random.id(); @@ -2081,9 +2081,12 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ } self.collection = new Meteor.Collection(collectionName); + self.date = new Date; + self.objId = new Meteor.Collection.ObjectID; self.id = self.collection.insert( - {name: 'foo', custom: new TestCustomType('a', 'b')}, + {d: self.date, oi: self.objId, + custom: new TestCustomType('a', 'b')}, expect(function (err, res) { test.isFalse(err); test.equal(self.id, res); @@ -2106,7 +2109,8 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ test.length(self.changes, 1); test.equal(self.changes.shift(), ['a', self.id, - {name: 'foo', custom: new TestCustomType('a', 'b')}]); + {d: self.date, oi: self.objId, + custom: new TestCustomType('a', 'b')}]); // First, replace the entire custom object. // (runInFence is useful for the server, using expect() is useful for the @@ -2140,6 +2144,24 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ test.length(self.changes, 1); test.equal(self.changes.shift(), ['c', self.id, {custom: new TestCustomType('a', 'd')}]); + + // Update a date and an ObjectID too. + self.date2 = new Date(self.date.valueOf() + 1000); + self.objId2 = new Meteor.Collection.ObjectID; + runInFence(function () { + self.collection.update( + self.id, {$set: {d: self.date2, oi: self.objId2}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {d: self.date2, oi: self.objId2}]); + self.handle.stop(); } ]); From 0e21a3804354ad511b60e805793282562fed5a5d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 14:39:18 -0800 Subject: [PATCH 031/112] Avoid 100% CPU in tailing during Mongo failover --- packages/mongo-livedata/mongo_driver.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 948fcc397f..9d8063fa56 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -898,7 +898,7 @@ MongoConnection.prototype.tail = function (cursorDescription, docCallback) { var stopped = false; var lastTS = undefined; - Meteor.defer(function () { + var loop = function () { while (true) { if (stopped) return; @@ -930,9 +930,16 @@ MongoConnection.prototype.tail = function (cursorDescription, docCallback) { cursorDescription.collectionName, newSelector, cursorDescription.options)); + // Mongo failover takes many seconds. Retry in a bit. (Without this + // setTimeout, we peg the CPU at 100% and never notice the actual + // failover. + Meteor.setTimeout(loop, 100); + break; } } - }); + }; + + Meteor.defer(loop); return { stop: function () { From d5d11f7897533b3d0c9144af24613a15940118b5 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 12 Dec 2013 15:17:32 -0800 Subject: [PATCH 032/112] Apply patch to node versions 0.10.22 and 0.10.23. 0.10.23 just came out and hasn't made any changes to the streams stuff we patch. --- History.md | 13 +++++++------ packages/meteor/node-issue-6506-workaround.js | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/History.md b/History.md index f13a291d3b..7538c3b436 100644 --- a/History.md +++ b/History.md @@ -1,10 +1,11 @@ -## vNEXT +## v0.6.7 -This version of Meteor contains a patch for a bug in Node 0.10 which most -commonly affects websockets. The patch is against Node v0.10.22. We strongly -recommend using this precise version of Node in production so that the patch -will be applied. If you use a newer version of Node with this version of Meteor, -Meteor will not apply the patch and will instead disable websockets. +This version of Meteor contains a patch for a bug in Node 0.10 which +most commonly affects websockets. The patch is against Node version +0.10.22 and 0.10.23. We strongly recommend using one of these precise +versions of Node in production so that the patch will be applied. If you +use a newer version of Node with this version of Meteor, Meteor will not +apply the patch and will instead disable websockets. * Rework how Meteor gets realtime database updates from MongoDB. Meteor now reads the MongoDB "oplog" -- a special collection that records all diff --git a/packages/meteor/node-issue-6506-workaround.js b/packages/meteor/node-issue-6506-workaround.js index b4c640e70b..a08b92a9d8 100644 --- a/packages/meteor/node-issue-6506-workaround.js +++ b/packages/meteor/node-issue-6506-workaround.js @@ -1,13 +1,13 @@ // Temporary workaround for https://github.com/joyent/node/issues/6506 // Our fix involves replicating a bunch of files in order to // -if (process.version !== 'v0.10.22') { +if (process.version !== 'v0.10.22' && process.version !== 'v0.10.23') { if (!process.env.DISABLE_WEBSOCKETS) { console.error("This version of Meteor contains a patch for a bug in Node v0.10."); - console.error("The patch is against v0.10.22."); + console.error("The patch is against only versions 0.10.22 and 0.10.23."); console.error("You are using version " + process.version + " instead, so we cannot apply the patch."); console.error("To mitigate the most common effect of the bug, websockets will be disabled."); - console.error("To enable websockets, use Node v0.10.22 or upgrade to a later version of Meteor (if available)."); + console.error("To enable websockets, use Node v0.10.22 or .23, or upgrade to a later version of Meteor (if available)."); process.env.DISABLE_WEBSOCKETS = 't'; } } else { From 7d490a5bec3397d13c2023007c3252c70da7918e Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 12 Dec 2013 16:02:02 -0800 Subject: [PATCH 033/112] Make facts automatically subscribe when you show the template. Also, namespace the collection and subscription name. --- History.md | 5 +++-- packages/facts/facts.js | 22 ++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/History.md b/History.md index 7538c3b436..ccaf26d906 100644 --- a/History.md +++ b/History.md @@ -30,8 +30,9 @@ apply the patch and will instead disable websockets. client code changes; server only code changes will not cause the page to reload. -* New 'facts' package publishes internal statistics about Meteor. - XXX how to use +* New 'facts' package publishes internal statistics about Meteor. To + use, simply `meteor add facts` then add `{{> serverFacts}}` somewhere + in your interface. * Add an explicit check that publish functions return a cursor, an array of cursors, or a falsey value. This is a safety check to to prevent diff --git a/packages/facts/facts.js b/packages/facts/facts.js index 65f4517530..543023b271 100644 --- a/packages/facts/facts.js +++ b/packages/facts/facts.js @@ -1,6 +1,6 @@ Facts = {}; -var serverFactsCollection = 'Facts.server'; +var serverFactsCollection = 'meteor_Facts_server'; if (Meteor.isServer) { // By default, we publish facts to no user if autopublish is off, and to all @@ -45,7 +45,7 @@ if (Meteor.isServer) { // called? Meteor.defer(function () { // XXX Also publish facts-by-package. - Meteor.publish("facts", function () { + Meteor.publish("meteor_facts", function () { var sub = this; if (!userIdFilter(this.userId)) { sub.ready(); @@ -59,13 +59,10 @@ if (Meteor.isServer) { activeSubscriptions = _.without(activeSubscriptions, sub); }); sub.ready(); - }); + }, {is_auto: true}); }); } else { Facts.server = new Meteor.Collection(serverFactsCollection); - // XXX making all clients subscribe all the time is wasteful. - // add an interface here - // Meteor.subscribe("facts"); Template.serverFacts.factsByPackage = function () { return Facts.server.find(); @@ -78,4 +75,17 @@ if (Meteor.isServer) { }); return factArray; }; + + // Subscribe when the template is first made, and unsubscribe when it + // is removed. If for some reason puts two copies of the template on + // the screen at once, we'll subscribe twice. Meh. + Template.serverFacts.created = function () { + this._stopHandle = Meteor.subscribe("meteor_facts"); + }; + Template.serverFacts.destroyed = function () { + if (this._stopHandle) { + this._stopHandle.stop(); + this._stopHandle = null; + } + }; } From 0944988adf64f43cbbd94b583319b22a198b8391 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 15:55:06 -0800 Subject: [PATCH 034/112] Add some XXX comments about when we should re-poll --- packages/mongo-livedata/oplog_observe_driver.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 7e3da7d1f1..b9c2b6533b 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -294,6 +294,15 @@ _.extend(OplogObserveDriver.prototype, { // In various circumstances, we may just want to stop processing the oplog and // re-run the initial query, just as if we were a PollingObserveDriver. + // + // XXX We should call this when we detect that we've been in FETCHING for "too + // long". + // + // XXX We should call this when we detect Mongo failover (since that might + // mean that some of the oplog entries we have processed have been rolled + // back). The Node Mongo driver is in the middle of a bunch of huge + // refactorings, including the way that it notifies you when primary + // changes. Will put off implementing this until driver 1.4 is out. _pollQuery: function () { var self = this; From 00dea841b26cede3a3e5f11a4b9a77c1923c3e52 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:10:41 -0800 Subject: [PATCH 035/112] Take out check that can fail during failover --- packages/mongo-livedata/oplog_tailing.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/mongo-livedata/oplog_tailing.js b/packages/mongo-livedata/oplog_tailing.js index 66d6874e65..1a123b0035 100644 --- a/packages/mongo-livedata/oplog_tailing.js +++ b/packages/mongo-livedata/oplog_tailing.js @@ -134,22 +134,15 @@ _.extend(OplogHandle.prototype, { return; } + + // Insert the future into our list. Almost always, this will be at the end, + // but it's conceivable that if we fail over from one primary to another, + // the oplog entries we see will go backwards. var insertAfter = self._catchingUpFutures.length; while (insertAfter - 1 > 0 && self._catchingUpFutures[insertAfter - 1].ts.greaterThan(ts)) { insertAfter--; } - - // XXX this can occur if we fail over from one primary to another. so this - // check needs to be removed before we merge oplog. that said, it has been - // helpful so far at proving that we are properly using poolSize 1. Also, we - // could keep something like it if we could actually detect failover; see - // https://github.com/mongodb/node-mongodb-native/issues/1120 - if (insertAfter !== self._catchingUpFutures.length) { - throw Error("found misordered oplog: " - + showTS(_.last(self._catchingUpFutures).ts) + " vs " - + showTS(ts)); - } var f = new Future; self._catchingUpFutures.splice(insertAfter, 0, {ts: ts, future: f}); f.wait(); From 8e31d09c667646569f3ee0d9212153afaf7570e0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:20:27 -0800 Subject: [PATCH 036/112] A few History tweaks --- History.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/History.md b/History.md index ccaf26d906..6bb2fe51e5 100644 --- a/History.md +++ b/History.md @@ -27,7 +27,7 @@ apply the patch and will instead disable websockets. * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded message at DDP startup. Now the hot code push only triggers when - client code changes; server only code changes will not cause the page + client code changes; server-only code changes will not cause the page to reload. * New 'facts' package publishes internal statistics about Meteor. To @@ -68,7 +68,7 @@ apply the patch and will instead disable websockets. * Support `EJSON.clone` for `Meteor.Error`. As a result, they are properly stringified in DDP even if thrown through a `Future`. #1482 -* Fix passing `transform: null` option to `Collection.find()` to disable +* Fix passing `transform: null` option to `collection.allow()` to disable transformation in validators. #1659 * Fix livedata error on `this.removed` during session shutdown. #1540 #1553 @@ -98,7 +98,7 @@ apply the patch and will instead disable websockets. * uglify-js from a fork of 2.4.0 to 2.4.7 Patches contributed by GitHub users AlexeyMK, awwx, dandv, -DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, sdarnell. +DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, and sdarnell. ## v0.6.6.3 From 0d5060eb6a1a9115515c98c0d03a2f2aec6bb458 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Thu, 12 Dec 2013 16:27:17 -0800 Subject: [PATCH 037/112] Before minifying CSS pull out all @import's to the top of CSS file. We need it as we concatenate all CSS files in one. @import's those were in the beginning of file foo.css are now in the middle of giant concatenated sha1.css. --- packages/minifiers/minifiers.js | 15 +++++++++++++++ packages/minifiers/minifiers_tests.js | 26 ++++++++++++++++++++++++++ packages/minifiers/package.js | 8 ++++++++ 3 files changed, 49 insertions(+) create mode 100644 packages/minifiers/minifiers_tests.js diff --git a/packages/minifiers/minifiers.js b/packages/minifiers/minifiers.js index ad15487d26..ab4ac1c2da 100644 --- a/packages/minifiers/minifiers.js +++ b/packages/minifiers/minifiers.js @@ -1,8 +1,23 @@ var CleanCss = Npm.require('clean-css'); CleanCSSProcess = function (source, options) { + options = _.extend({ processImport: false }, options); var instance = new CleanCss(options); + // after concatenation some @import's might be left in the middle of CSS file + // but they required to be in the beginning. + source = CSSPullImports(source); return instance.minify(source); }; UglifyJSMinify = Npm.require('uglify-js').minify; + +var CSSPullImports = function (source) { + var importRegExp = /^\s*@import\s*[^;]*;\s*$/gm; + var imports = source.match(importRegExp) || []; + var newSource = source.replace(importRegExp, ''); + + newSource = imports.join('') + newSource; + + return newSource; +} + diff --git a/packages/minifiers/minifiers_tests.js b/packages/minifiers/minifiers_tests.js new file mode 100644 index 0000000000..f64bf5814b --- /dev/null +++ b/packages/minifiers/minifiers_tests.js @@ -0,0 +1,26 @@ +Tinytest.add("minifiers - CSS pull the imports to the top", function (test) { + function t(source, expected, descr) { + var processed = CleanCSSProcess(source); + test.equal(processed, expected, descr); + } + + t(["@import url(//fonts.googleapis.com/css?family=Tangerine);", + ".block1{font-family:Tangerine}", + "@import url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", + ".block2{font-family:'Oleo Script Swash Caps'}"].join('\n'), + ["@import url(//fonts.googleapis.com/css?family=Tangerine);", + "@import url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", + ".block1{font-family:Tangerine}", + ".block2{font-family:'Oleo Script Swash Caps'}"].join(''), "just imports"); + t(["@import url(//fonts.googleapis.com/css?family=Tangerine),", + " url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", + ".block2{font-family:'Oleo Script Swash Caps'}"].join('\n'), + ["@import url(//fonts.googleapis.com/css?family=Tangerine),", + "url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", + ".block2{font-family:'Oleo Script Swash Caps'}"].join(''), "multiline @imports"); + t([".block2{font-family:'Oleo Script Swash Caps'}", + "div[x='@import asdf;']{font:#000}"].join('\n'), + [".block2{font-family:'Oleo Script Swash Caps'}", + "div[x='@import asdf;']{font:#000}"].join(''), "@import embeded as string"); +}); + diff --git a/packages/minifiers/package.js b/packages/minifiers/package.js index c76efd3ee9..3e050106fa 100644 --- a/packages/minifiers/package.js +++ b/packages/minifiers/package.js @@ -9,6 +9,14 @@ Npm.depends({ }); Package.on_use(function (api) { + api.use('underscore'); api.export(['CleanCSSProcess', 'UglifyJSMinify']); api.add_files('minifiers.js', 'server'); }); + +Package.on_test(function (api) { + api.use('minifiers'); + api.use('tinytest'); + api.add_files('minifiers_tests.js', 'server'); +}); + From f2e3481db3c50c7dd1384977981bbaf633bd0b47 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Thu, 12 Dec 2013 16:47:37 -0800 Subject: [PATCH 038/112] Drop the requirement for @import to be the last statement on the line + Testcase for that --- packages/minifiers/minifiers.js | 2 +- packages/minifiers/minifiers_tests.js | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/minifiers/minifiers.js b/packages/minifiers/minifiers.js index ab4ac1c2da..e96e4c309c 100644 --- a/packages/minifiers/minifiers.js +++ b/packages/minifiers/minifiers.js @@ -12,7 +12,7 @@ CleanCSSProcess = function (source, options) { UglifyJSMinify = Npm.require('uglify-js').minify; var CSSPullImports = function (source) { - var importRegExp = /^\s*@import\s*[^;]*;\s*$/gm; + var importRegExp = /^\s*@import\s*[^;]*;\s*/gm; var imports = source.match(importRegExp) || []; var newSource = source.replace(importRegExp, ''); diff --git a/packages/minifiers/minifiers_tests.js b/packages/minifiers/minifiers_tests.js index f64bf5814b..af68fcd571 100644 --- a/packages/minifiers/minifiers_tests.js +++ b/packages/minifiers/minifiers_tests.js @@ -22,5 +22,11 @@ Tinytest.add("minifiers - CSS pull the imports to the top", function (test) { "div[x='@import asdf;']{font:#000}"].join('\n'), [".block2{font-family:'Oleo Script Swash Caps'}", "div[x='@import asdf;']{font:#000}"].join(''), "@import embeded as string"); + t(["@import url(//bla.com/font.css);.block1{font-family:Tangerine}", + "@import url(//bla2.com/font2.css);.block2{font-family:Font2}"].join('\n'), + ["@import url(//bla.com/font.css);", + "@import url(//bla2.com/font2.css);", + ".block1{font-family:Tangerine}", + ".block2{font-family:Font2}"].join('')); }); From 09d129b7996fc0524745d110cf361e7c0cf8faff Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 23:57:09 -0800 Subject: [PATCH 039/112] First pass text for oplog tailing in History.md. --- History.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index 3306e46bd7..f13a291d3b 100644 --- a/History.md +++ b/History.md @@ -6,7 +6,17 @@ recommend using this precise version of Node in production so that the patch will be applied. If you use a newer version of Node with this version of Meteor, Meteor will not apply the patch and will instead disable websockets. -* XXX oplog tailing +* Rework how Meteor gets realtime database updates from MongoDB. Meteor + now reads the MongoDB "oplog" -- a special collection that records all + the write operations as they are applied to your database. This means + changes to the database are instantly noticed and reflected in Meteor, + whether they originated from Meteor or from an external database + client. Oplog tailing is automatically enabled in development mode + with `meteor run`, and can be enabled in production with the + `MONGO_OPLOG_URL` environment variable. Currently the only supported + selectors are equality checks; `$`-operators, `limit` and `skip` + queries fall back to the original poll-and-diff algorithm. See for details. * Add `Meteor.onConnection` and add `this.connection` to method invocations and publish functions. These can be used to store data From a244430f31dad5b3299e3b55e99fdc0daed016c0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 18:11:20 -0800 Subject: [PATCH 040/112] Handle modifications to EJSON custom types OplogObserveDriver cannot try to directly apply these representation-level mutations; it needs to get the entire custom type. This implementation is overly conservative, but in practice Meteor-originated writes shouldn't mutate EJSON custom types anyway (they should treat them as atomic). --- .../mongo-livedata/mongo_livedata_tests.js | 100 ++++++++++++++++++ .../mongo-livedata/oplog_observe_driver.js | 21 +++- 2 files changed, 118 insertions(+), 3 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 750d006122..299d34bbd6 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -2043,3 +2043,103 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - drop collection", func handle.stop(); }); + +var TestCustomType = function (head, tail) { + // use different field names on the object than in JSON, to ensure we are + // actually treating this as an opaque object. + this.myHead = head; + this.myTail = tail; +}; +_.extend(TestCustomType.prototype, { + clone: function () { + return new TestCustomType(this.myHead, this.myTail); + }, + equals: function (other) { + return other instanceof TestCustomType + && EJSON.equals(this.myHead, other.myHead) + && EJSON.equals(this.myTail, other.myTail); + }, + typeName: function () { + return 'someCustomType'; + }, + toJSONValue: function () { + return {head: this.myHead, tail: this.myTail}; + } +}); + +EJSON.addType('someCustomType', function (json) { + return new TestCustomType(json.head, json.tail); +}); + +testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ + function (test, expect) { + var self = this; + var collectionName = "ejson" + Random.id(); + if (Meteor.isClient) { + Meteor.call('createInsecureCollection', collectionName); + Meteor.subscribe('c-' + collectionName); + } + + self.collection = new Meteor.Collection(collectionName); + + self.id = self.collection.insert( + {name: 'foo', custom: new TestCustomType('a', 'b')}, + expect(function (err, res) { + test.isFalse(err); + test.equal(self.id, res); + })); + }, + function (test, expect) { + var self = this; + self.changes = []; + self.handle = self.collection.find({}).observeChanges({ + added: function (id, fields) { + self.changes.push(['a', id, fields]); + }, + changed: function (id, fields) { + self.changes.push(['c', id, fields]); + }, + removed: function (id) { + self.changes.push(['r', id]); + } + }); + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['a', self.id, + {name: 'foo', custom: new TestCustomType('a', 'b')}]); + + // First, replace the entire custom object. + // (runInFence is useful for the server, using expect() is useful for the + // client) + runInFence(function () { + self.collection.update( + self.id, {$set: {custom: new TestCustomType('a', 'c')}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {custom: new TestCustomType('a', 'c')}]); + + // Now, sneakily replace just a piece of it. Meteor won't do this, but + // perhaps you are accessing Mongo directly. + runInFence(function () { + self.collection.update( + self.id, {$set: {'custom.EJSON$value.EJSONtail': 'd'}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {custom: new TestCustomType('a', 'd')}]); + self.handle.stop(); + } +]); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 8fbc050d35..7e3da7d1f1 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -247,18 +247,25 @@ _.extend(OplogObserveDriver.prototype, { // replacement (in which case we can just directly re-evaluate the // selector)? var isReplace = !_.has(op.o, '$set') && !_.has(op.o, '$unset'); + // If this modifier modifies something inside an EJSON custom type (ie, + // anything with EJSON$), then we can't try to use + // LocalCollection._modify, since that just mutates the EJSON encoding, + // not the actual object. + var canDirectlyModifyDoc = + !isReplace && modifierCanBeDirectlyApplied(op.o); if (isReplace) { self._handleDoc(id, _.extend({_id: id}, op.o)); - } else if (self._published.has(id)) { + } else if (self._published.has(id) && canDirectlyModifyDoc) { // Oh great, we actually know what the document is, so we can apply // this directly. var newDoc = EJSON.clone(self._published.get(id)); newDoc._id = id; LocalCollection._modify(newDoc, op.o); self._handleDoc(id, self._sharedProjectionFn(newDoc)); - } else if (LocalCollection._canSelectorBecomeTrueByModifier( - self._cursorDescription.selector, op.o)) { + } else if (!canDirectlyModifyDoc || + LocalCollection._canSelectorBecomeTrueByModifier( + self._cursorDescription.selector, op.o)) { self._needToFetch.set(id, op.ts.toString()); if (self._phase === PHASE.STEADY) self._fetchModifiedDocuments(); @@ -480,4 +487,12 @@ OplogObserveDriver.cursorSupported = function (cursorDescription) { }); }; +var modifierCanBeDirectlyApplied = function (modifier) { + return _.all(modifier, function (fields, operation) { + return _.all(fields, function (value, field) { + return !/EJSON\$/.test(field); + }); + }); +}; + MongoTest.OplogObserveDriver = OplogObserveDriver; From 5ebf0f2ec34b4f981ff0e9e26a6a490acd856040 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 14:39:18 -0800 Subject: [PATCH 041/112] Avoid 100% CPU in tailing during Mongo failover --- packages/mongo-livedata/mongo_driver.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 948fcc397f..9d8063fa56 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -898,7 +898,7 @@ MongoConnection.prototype.tail = function (cursorDescription, docCallback) { var stopped = false; var lastTS = undefined; - Meteor.defer(function () { + var loop = function () { while (true) { if (stopped) return; @@ -930,9 +930,16 @@ MongoConnection.prototype.tail = function (cursorDescription, docCallback) { cursorDescription.collectionName, newSelector, cursorDescription.options)); + // Mongo failover takes many seconds. Retry in a bit. (Without this + // setTimeout, we peg the CPU at 100% and never notice the actual + // failover. + Meteor.setTimeout(loop, 100); + break; } } - }); + }; + + Meteor.defer(loop); return { stop: function () { From a5d5a03189300db0c50df29bed211e6a9a21e6f3 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 12 Dec 2013 15:17:32 -0800 Subject: [PATCH 042/112] Apply patch to node versions 0.10.22 and 0.10.23. 0.10.23 just came out and hasn't made any changes to the streams stuff we patch. --- History.md | 13 +++++++------ packages/meteor/node-issue-6506-workaround.js | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/History.md b/History.md index f13a291d3b..7538c3b436 100644 --- a/History.md +++ b/History.md @@ -1,10 +1,11 @@ -## vNEXT +## v0.6.7 -This version of Meteor contains a patch for a bug in Node 0.10 which most -commonly affects websockets. The patch is against Node v0.10.22. We strongly -recommend using this precise version of Node in production so that the patch -will be applied. If you use a newer version of Node with this version of Meteor, -Meteor will not apply the patch and will instead disable websockets. +This version of Meteor contains a patch for a bug in Node 0.10 which +most commonly affects websockets. The patch is against Node version +0.10.22 and 0.10.23. We strongly recommend using one of these precise +versions of Node in production so that the patch will be applied. If you +use a newer version of Node with this version of Meteor, Meteor will not +apply the patch and will instead disable websockets. * Rework how Meteor gets realtime database updates from MongoDB. Meteor now reads the MongoDB "oplog" -- a special collection that records all diff --git a/packages/meteor/node-issue-6506-workaround.js b/packages/meteor/node-issue-6506-workaround.js index b4c640e70b..a08b92a9d8 100644 --- a/packages/meteor/node-issue-6506-workaround.js +++ b/packages/meteor/node-issue-6506-workaround.js @@ -1,13 +1,13 @@ // Temporary workaround for https://github.com/joyent/node/issues/6506 // Our fix involves replicating a bunch of files in order to // -if (process.version !== 'v0.10.22') { +if (process.version !== 'v0.10.22' && process.version !== 'v0.10.23') { if (!process.env.DISABLE_WEBSOCKETS) { console.error("This version of Meteor contains a patch for a bug in Node v0.10."); - console.error("The patch is against v0.10.22."); + console.error("The patch is against only versions 0.10.22 and 0.10.23."); console.error("You are using version " + process.version + " instead, so we cannot apply the patch."); console.error("To mitigate the most common effect of the bug, websockets will be disabled."); - console.error("To enable websockets, use Node v0.10.22 or upgrade to a later version of Meteor (if available)."); + console.error("To enable websockets, use Node v0.10.22 or .23, or upgrade to a later version of Meteor (if available)."); process.env.DISABLE_WEBSOCKETS = 't'; } } else { From 71ca231c2b0c352a1e298b91218b6dea1c5b2b8b Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:10:41 -0800 Subject: [PATCH 043/112] Take out check that can fail during failover --- packages/mongo-livedata/oplog_tailing.js | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/packages/mongo-livedata/oplog_tailing.js b/packages/mongo-livedata/oplog_tailing.js index 66d6874e65..1a123b0035 100644 --- a/packages/mongo-livedata/oplog_tailing.js +++ b/packages/mongo-livedata/oplog_tailing.js @@ -134,22 +134,15 @@ _.extend(OplogHandle.prototype, { return; } + + // Insert the future into our list. Almost always, this will be at the end, + // but it's conceivable that if we fail over from one primary to another, + // the oplog entries we see will go backwards. var insertAfter = self._catchingUpFutures.length; while (insertAfter - 1 > 0 && self._catchingUpFutures[insertAfter - 1].ts.greaterThan(ts)) { insertAfter--; } - - // XXX this can occur if we fail over from one primary to another. so this - // check needs to be removed before we merge oplog. that said, it has been - // helpful so far at proving that we are properly using poolSize 1. Also, we - // could keep something like it if we could actually detect failover; see - // https://github.com/mongodb/node-mongodb-native/issues/1120 - if (insertAfter !== self._catchingUpFutures.length) { - throw Error("found misordered oplog: " - + showTS(_.last(self._catchingUpFutures).ts) + " vs " - + showTS(ts)); - } var f = new Future; self._catchingUpFutures.splice(insertAfter, 0, {ts: ts, future: f}); f.wait(); From a536b64c82070428b105ac685920ec7ca47ad9d1 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:20:27 -0800 Subject: [PATCH 044/112] A few History tweaks --- History.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/History.md b/History.md index 7538c3b436..8e5d4c96dd 100644 --- a/History.md +++ b/History.md @@ -27,7 +27,7 @@ apply the patch and will instead disable websockets. * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded message at DDP startup. Now the hot code push only triggers when - client code changes; server only code changes will not cause the page + client code changes; server-only code changes will not cause the page to reload. * New 'facts' package publishes internal statistics about Meteor. @@ -67,7 +67,7 @@ apply the patch and will instead disable websockets. * Support `EJSON.clone` for `Meteor.Error`. As a result, they are properly stringified in DDP even if thrown through a `Future`. #1482 -* Fix passing `transform: null` option to `Collection.find()` to disable +* Fix passing `transform: null` option to `collection.allow()` to disable transformation in validators. #1659 * Fix livedata error on `this.removed` during session shutdown. #1540 #1553 @@ -97,7 +97,7 @@ apply the patch and will instead disable websockets. * uglify-js from a fork of 2.4.0 to 2.4.7 Patches contributed by GitHub users AlexeyMK, awwx, dandv, -DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, sdarnell. +DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, and sdarnell. ## v0.6.6.3 From 9d5fb4d1e10d0510a7555f2780d72b3d428a3848 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 13 Dec 2013 02:19:13 -0800 Subject: [PATCH 045/112] upgrade examples and docs. --- docs/.meteor/release | 2 +- examples/leaderboard/.meteor/release | 2 +- examples/parties/.meteor/release | 2 +- examples/todos/.meteor/release | 2 +- examples/wordplay/.meteor/release | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index 1b52537518..bb2638e6b6 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -galaxy-follower-6 +0.6.7-rc1 diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release index b65d3d9aeb..bb2638e6b6 100644 --- a/examples/leaderboard/.meteor/release +++ b/examples/leaderboard/.meteor/release @@ -1 +1 @@ -0.6.6.2 +0.6.7-rc1 diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release index b65d3d9aeb..bb2638e6b6 100644 --- a/examples/parties/.meteor/release +++ b/examples/parties/.meteor/release @@ -1 +1 @@ -0.6.6.2 +0.6.7-rc1 diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release index b65d3d9aeb..bb2638e6b6 100644 --- a/examples/todos/.meteor/release +++ b/examples/todos/.meteor/release @@ -1 +1 @@ -0.6.6.2 +0.6.7-rc1 diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release index b65d3d9aeb..bb2638e6b6 100644 --- a/examples/wordplay/.meteor/release +++ b/examples/wordplay/.meteor/release @@ -1 +1 @@ -0.6.6.2 +0.6.7-rc1 From de6fb4276da05da6cb3b5920001a790d4c2b33d4 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 13 Dec 2013 02:19:34 -0800 Subject: [PATCH 046/112] update displayed release version number for docs. --- docs/lib/release-override.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js index 7ee3b0eafc..8b66d7eea5 100644 --- a/docs/lib/release-override.js +++ b/docs/lib/release-override.js @@ -1,5 +1,5 @@ // While galaxy apps are on their own special meteor releases, override // Meteor.release here. if (Meteor.isClient) { - Meteor.release = Meteor.release ? "0.6.6.3" : undefined; + Meteor.release = Meteor.release ? "0.6.7" : undefined; } From 6a9a797e83135026a11eed4cb80b2b47ea32d0a8 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:43:12 -0800 Subject: [PATCH 047/112] Make observe driver facts names consistent --- packages/mongo-livedata/oplog_observe_driver.js | 4 ++-- packages/mongo-livedata/polling_observe_driver.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index b9c2b6533b..dbd44c310b 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -26,7 +26,7 @@ OplogObserveDriver = function (options) { self._stopHandles = []; Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "oplog-observers", 1); + "mongo-livedata", "observe-drivers-oplog", 1); self._phase = PHASE.QUERYING; @@ -444,7 +444,7 @@ _.extend(OplogObserveDriver.prototype, { self._listenersHandle = null; Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "oplog-observers", -1); + "mongo-livedata", "observe-drivers-oplog", -1); } }); diff --git a/packages/mongo-livedata/polling_observe_driver.js b/packages/mongo-livedata/polling_observe_driver.js index 938798e519..82020f2147 100644 --- a/packages/mongo-livedata/polling_observe_driver.js +++ b/packages/mongo-livedata/polling_observe_driver.js @@ -72,7 +72,7 @@ PollingObserveDriver = function (options) { self._unthrottledEnsurePollIsScheduled(); Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "mongo-pollsters", 1); + "mongo-livedata", "observe-drivers-polling", 1); }; _.extend(PollingObserveDriver.prototype, { @@ -174,6 +174,6 @@ _.extend(PollingObserveDriver.prototype, { self._stopped = true; _.each(self._stopCallbacks, function (c) { c(); }); Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "mongo-pollsters", -1); + "mongo-livedata", "observe-drivers-polling", -1); } }); From b7188769f25f1e313c950d249a8c046d6b77ce4c Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 9 Dec 2013 12:16:06 -0500 Subject: [PATCH 048/112] remove scale --- packages/ctl/ctl.js | 48 --------------------------------------------- 1 file changed, 48 deletions(-) diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index cc2833bd11..930a0fbe5e 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -96,54 +96,6 @@ Ctl.Commands.push({ func: stopFun }); -Ctl.Commands.push({ - name: "scale", - help: "Scale jobs", - func: function (argv) { - if (argv.help || argv._.length === 0 || _.contains(argv._, 'ctl')) { - process.stderr.write( -"Usage: ctl scale program1=n [...] \n" + - "\n" + -"Scales some programs. Runs or kills jobs until there are n non-done jobs\n" + -"in that state.\n" -); - process.exit(1); - } - - var scales = _.map(argv._, function (arg) { - var m = arg.match(/^(.+)=(\d+)$/); - if (!m) { - console.log("Bad scaling argument; should be program=number."); - process.exit(1); - } - return {program: m[1], scale: parseInt(m[2])}; - }); - - _.each(scales, function (s) { - var jobs = Ctl.getJobsByApp( - Ctl.myAppName(), {program: s.program, done: false}); - jobs.forEach(function (job) { - --s.scale; - // Is this an extraneous job, more than the number that we need? Kill - // it! - if (s.scale < 0) { - Ctl.kill(s.program, job._id); - } - }); - // Now start any jobs that are necessary. - if (s.scale <= 0) - return; - console.log("Starting %d jobs for %s", s.scale, s.program); - _.times(s.scale, function () { - // XXX args? env? - Ctl.prettyCall(Ctl.findGalaxy(), 'run', [Ctl.myAppName(), s.program, { - exitPolicy: 'restart' - }]); - }); - }); - } -}); - main = function (argv) { return Ctl.main(argv); }; From a951ab2fee5efb92d4866bad83a2ea91bb668882 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 9 Dec 2013 14:19:04 -0500 Subject: [PATCH 049/112] New beginUpdate flow for ctl, run new and old job simultaneously --- packages/ctl/ctl.js | 70 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 930a0fbe5e..d51db44835 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -1,3 +1,5 @@ +var Future = Npm.require("fibers/future"); + Ctl.Commands.push({ name: "help", func: function (argv) { @@ -89,11 +91,77 @@ Ctl.Commands.push({ func: stopFun }); +var waitForDone = function (jobCollection, jobId) { + var fut = new Future(); + var found = false; + try { + var observation = jobCollection.find(jobId).observe({ + added: function (doc) { + found = true; + if (doc.done) + fut['return'](); + }, + changed: function (doc) { + if (doc.done) + fut['return'](); + }, + removed: function (doc) { + fut['return'](); + } + }); + // if the document doesn't exist at all, it's certainly done. + if (!found) + fut['return'](); + fut.wait(); + } finally { + observation.stop(); + } +}; + Ctl.Commands.push({ name: "beginUpdate", help: "Stop this app to begin an update", - func: stopFun + func: function (argv) { + Ctl.subscribeToAppJobs(Ctl.myAppName()); + var jobs = Ctl.jobsCollection(); + var thisJob = jobs.findOne(Ctl.myJobId()); + // Look at all the server jobs that are on the old star. + var oldJobSelector = { + app: Ctl.myAppName(), + star: {$ne: thisJob.star}, + program: "server", + done: false + }; + var oldServers = jobs.find(oldJobSelector).fetch(); + // Start a new job for each of them. + _.each(oldServers, function (oldServer) { + Ctl.startServerlikeProgram("server", oldServer.tags, oldServer.env.ADMIN_APP); + }); + // Wait for them all to come up and bind to the proxy. + Meteor._sleepForMs(5000); // XXX: Eventually make sure they're proxy-bound. + // (eventually) tell the proxy to switch over to using the new star + // One by one, kill all the old star's server jobs. + var jobToKill = jobs.findOne(oldJobSelector); + while (jobToKill) { + Ctl.kill("server", jobToKill._id); + // Wait for it to go down + waitForDone(jobs, jobToKill._id); + // Spend some time in between to allow any reconnect storm to die down. + Meteor._sleepForMs(1000); + jobToKill = jobs.findOne(oldJobSelector); + } + // Now kill all non-server jobs. They're less important. + jobs.find({ + app: Ctl.myAppName(), + star: {$ne: thisJob.star}, + program: {$ne: "server"}, + done: false + }).forEach(function (job) { + Ctl.kill(job.program, job._id); + }); + // fin + } }); main = function (argv) { From 19964b351c05a60c189a5687c158505fe372d103 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 9 Dec 2013 16:02:28 -0500 Subject: [PATCH 050/112] ctl-helper to return job id on creation of serverlike programs --- packages/ctl-helper/ctl-helper.js | 7 +++++-- packages/ctl/ctl.js | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index 86e15404ee..d48e93a2b1 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -33,10 +33,11 @@ _.extend(Ctl, { var numServers = Ctl.getJobsByApp( Ctl.myAppName(), {program: program, done: false}).count(); if (numServers === 0) { - Ctl.startServerlikeProgram(program, tags, admin); + return Ctl.startServerlikeProgram(program, tags, admin); } else { console.log(program, "already running."); } + return null; }, startServerlikeProgram: function (program, tags, admin) { @@ -47,6 +48,7 @@ _.extend(Ctl, { var proxyConfig; var bindPathPrefix = ""; + var jobId = null; if (admin) { bindPathPrefix = "/" + encodeURIComponent(Ctl.myAppName()).replace(/\./g, '_'); } @@ -60,7 +62,7 @@ _.extend(Ctl, { }); // XXX args? env? - Ctl.prettyCall(Ctl.findGalaxy(), 'run', [Ctl.myAppName(), program, { + jobId = Ctl.prettyCall(Ctl.findGalaxy(), 'run', [Ctl.myAppName(), program, { exitPolicy: 'restart', env: { ROOT_URL: "https://" + appConfig.sitename + bindPathPrefix, @@ -80,6 +82,7 @@ _.extend(Ctl, { tags: tags }]); console.log("Started", program); + return jobId; }, findCommand: function (name) { diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index d51db44835..3dc1bdc7d1 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -151,7 +151,7 @@ Ctl.Commands.push({ Meteor._sleepForMs(1000); jobToKill = jobs.findOne(oldJobSelector); } - // Now kill all non-server jobs. They're less important. + // Now kill all old non-server jobs. They're less important. jobs.find({ app: Ctl.myAppName(), star: {$ne: thisJob.star}, From 7da8004ddfb1d4d049ae6d7770694208ca759ec1 Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 11 Dec 2013 14:50:43 -0800 Subject: [PATCH 051/112] version as tag --- packages/application-configuration/config.js | 20 +++++++++++++------- packages/webapp/webapp_server.js | 7 ++++++- tools/deploy-galaxy.js | 1 + 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index bec41318e4..7de777e90c 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -69,30 +69,36 @@ try { AppConfig.getAppConfig = function () { if (!subFuture.isResolved() && staticAppConfig) { + console.log("STATIC APP CONFIG"); return staticAppConfig; } subFuture.wait(); var myApp = OneAppApps.findOne(process.env.GALAXY_APP); - if (myApp) - return myApp.config; - throw new Error("there is no app config for this app"); + if (!myApp) { + throw new Error("there is no app config for this app"); + } + var config = myApp.config; + config.version = myApp.currentStar; + return config; }; AppConfig.configurePackage = function (packageName, configure) { var appConfig = AppConfig.getAppConfig(); // Will either be based in the env var, // or wait for galaxy to connect. var lastConfig = - (appConfig && appConfig.packages && appConfig.packages[packageName]) || {}; + (appConfig && appConfig.packages && + appConfig.packages[packageName]) || {}; + // Always call the configure callback "soon" even if the initial configuration // is empty (synchronously, though deferred would be OK). // XXX make sure that all callers of configurePackage deal well with multiple // callback invocations! eg, email does not - configure(lastConfig); + configure(_.extend({version: appConfig.version}, lastConfig)); var configureIfDifferent = function (app) { - if (!EJSON.equals(app.config && app.config.packages && app.config.packages[packageName], + if (!EJSON.equals(app.config && app.config.packages && app.config.packages[packageName] && app.config.version, lastConfig)) { lastConfig = app.config.packages[packageName]; - configure(lastConfig); + configure(_.extend({version: appConfig.version}, lastConfig)); } }; var subHandle; diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 66ae581ca4..83ec4d5996 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -565,8 +565,8 @@ var runWebAppServer = function () { } else { proxyConf = configuration.proxy; } + proxyConf.version = configuration.version; Log("Attempting to bind to proxy at " + proxyService.providers.proxy); - console.log(proxyConf); WebAppInternals.bindToProxy(_.extend({ proxyEndpoint: proxyService.providers.proxy }, proxyConf), proxyServiceName); @@ -661,10 +661,13 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { }; }; + var version = (proxyConfig.version) ? proxyConfig.version : ""; + proxy.call('bindDdp', { pid: pid, bindTo: ddpBindTo, proxyTo: { + tags: [version], host: host, port: port, pathPrefix: bindPathPrefix + '/websocket' @@ -678,6 +681,7 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { pathPrefix: bindPathPrefix }, proxyTo: { + tags: [version], host: host, port: port, pathPrefix: bindPathPrefix @@ -693,6 +697,7 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { ssl: true }, proxyTo: { + tags: [version], host: host, port: port, pathPrefix: bindPathPrefix diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js index ddff62077c..5d2f70fbdf 100644 --- a/tools/deploy-galaxy.js +++ b/tools/deploy-galaxy.js @@ -205,6 +205,7 @@ exports.deploy = function (options) { // fails (we already know the app exists.) var info = prettyCall(galaxy, 'beginUploadStar', [options.app, bundleResult.starManifest]); + console.log("StarId: ", info.id); // Upload // XXX copied from galaxy/tool/galaxy.js var fileSize = fs.statSync(starball).size; From 9373bce6381fac314d445cc5189fea967e9c3a00 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Wed, 11 Dec 2013 15:04:28 -0800 Subject: [PATCH 052/112] Call updateTags on proxy in ctl --- packages/ctl/ctl.js | 27 +++++++++++++++++++++++++++ packages/ctl/package.js | 2 +- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 3dc1bdc7d1..d989ef0167 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -140,6 +140,32 @@ Ctl.Commands.push({ }); // Wait for them all to come up and bind to the proxy. Meteor._sleepForMs(5000); // XXX: Eventually make sure they're proxy-bound. + var proxy; + var proxyTagSwitchFuture = new Future; + AppConfig.configureService('proxy', function (proxyService) { + try { + proxy = Follower.connect(proxyService.providers.proxy, { + group: "proxy" + }); + proxy.call('updateTags', Ctl.myAppName(), ['', thisJob.star]); + proxy.disconnect(); + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['return'](); + } catch (e) { + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['throw'](e); + } + }); + + var proxyTimeout = Meteor.setTimeout(function () { + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['throw'](new Error("timed out looking for a proxy " + + "or trying to change tags on it " + + proxy.status().status)); + }, 10*1000); + proxyTagSwitchFuture.wait(); + Meteor.clearTimeout(proxyTimeout); + // (eventually) tell the proxy to switch over to using the new star // One by one, kill all the old star's server jobs. var jobToKill = jobs.findOne(oldJobSelector); @@ -161,6 +187,7 @@ Ctl.Commands.push({ Ctl.kill(job.program, job._id); }); // fin + process.exit(0); } }); diff --git a/packages/ctl/package.js b/packages/ctl/package.js index 200c4a1408..314d6e2a63 100644 --- a/packages/ctl/package.js +++ b/packages/ctl/package.js @@ -4,7 +4,7 @@ Package.describe({ }); Package.on_use(function (api) { - api.use(['underscore', 'livedata', 'mongo-livedata', 'ctl-helper'], 'server'); + api.use(['underscore', 'livedata', 'mongo-livedata', 'ctl-helper', 'application-configuration', 'follower-livedata'], 'server'); api.export('main', 'server'); api.add_files('ctl.js', 'server'); }); From 57c4c075d3e6f7944cb27800a896ab814cffde77 Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 11 Dec 2013 15:06:19 -0800 Subject: [PATCH 053/112] removing error statement --- packages/application-configuration/config.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index 7de777e90c..0a51ca133b 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -69,7 +69,6 @@ try { AppConfig.getAppConfig = function () { if (!subFuture.isResolved() && staticAppConfig) { - console.log("STATIC APP CONFIG"); return staticAppConfig; } subFuture.wait(); From 276034617afea0ca7d65a31113a871e7b29b59f3 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 12 Dec 2013 16:51:32 -0800 Subject: [PATCH 054/112] First version of a reload safetybelt against javascript or css not being loaded properly --- packages/webapp/css_detect.css | 3 +++ packages/webapp/package.js | 1 + packages/webapp/webapp_server.js | 34 ++++++++++++++++++++++++++++---- tools/bundler.js | 1 + 4 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 packages/webapp/css_detect.css diff --git a/packages/webapp/css_detect.css b/packages/webapp/css_detect.css new file mode 100644 index 0000000000..fa54606703 --- /dev/null +++ b/packages/webapp/css_detect.css @@ -0,0 +1,3 @@ +._meteor_detect_css { + width: 0px; +} \ No newline at end of file diff --git a/packages/webapp/package.js b/packages/webapp/package.js index bdd4c6d862..a824d65fc9 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -19,4 +19,5 @@ Package.on_use(function (api) { // loaded after webapp. api.export(['WebApp', 'main', 'WebAppInternals'], 'server'); api.add_files('webapp_server.js', 'server'); + api.add_files('css_detect.css', 'client'); }); diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 83ec4d5996..ae40585385 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -20,6 +20,16 @@ WebAppInternals = {}; var bundledJsCssPrefix; +var RELOAD_SAFETYBELT = "\n" + + "if (typeof Meteor === 'undefined' || \n" + + " ! _.find(document.styleSheets, function (sheet) { \n" + + " return _.find(sheet.cssRules, function (rule) { \n" + + " return rule.selectorText === '._meteor_detect_css'; \n" + + " }); \n" + + " })) \n" + + " document.location.reload(); \n"; + + var makeAppNamePathPrefix = function (appName) { return encodeURIComponent(appName).replace(/\./g, '_'); }; @@ -290,6 +300,7 @@ var runWebAppServer = function () { } }); + // Serve static files from the manifest. // This is inspired by the 'static' middleware. app.use(function (req, res, next) { @@ -306,12 +317,20 @@ var runWebAppServer = function () { return; } + var serveStaticJs = function (s) { + res.writeHead(200, { 'Content-type': 'application/javascript' }); + res.write(s); + res.end(); + }; + if (pathname === "/meteor_runtime_config.js" && ! WebAppInternals.inlineScriptsAllowed()) { - res.writeHead(200, { 'Content-type': 'application/javascript' }); - res.write("__meteor_runtime_config__ = " + - JSON.stringify(__meteor_runtime_config__) + ";"); - res.end(); + serveStaticJs("__meteor_runtime_config__ = " + + JSON.stringify(__meteor_runtime_config__) + ";"); + return; + } else if (pathname === "/meteor_reload_safetybelt.js" && + ! WebAppInternals.inlineScriptsAllowed()) { + serveStaticJs(RELOAD_SAFETYBELT); return; } @@ -519,11 +538,18 @@ var runWebAppServer = function () { /##RUNTIME_CONFIG##/, ""); + boilerplateHtml = boilerplateHtml.replace( + /##RELOAD_SAFETYBELT##/, + ""); } else { boilerplateHtml = boilerplateHtml.replace( /##RUNTIME_CONFIG##/, "" ); + boilerplateHtml = boilerplateHtml.replace( + /##RELOAD_SAFETYBELT##/, + ""); + } boilerplateHtml = boilerplateHtml.replace( /##ROOT_URL_PATH_PREFIX##/g, diff --git a/tools/bundler.js b/tools/bundler.js index 07e21fc625..5ea02d7110 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -792,6 +792,7 @@ _.extend(ClientTarget.prototype, { html.push(_.escape(js.url)); html.push('">\n'); }); + html.push('\n\n##RELOAD_SAFETYBELT##'); html.push('\n\n'); html.push(self.head.join('\n')); // unescaped! html.push('\n' + From 4ab6f8ee88adaf948f8824f0f0447628d22c1008 Mon Sep 17 00:00:00 2001 From: ekatek Date: Thu, 12 Dec 2013 17:12:05 -0800 Subject: [PATCH 055/112] tags --- packages/application-configuration/config.js | 27 ++++++++++++++++++-- packages/webapp/webapp_server.js | 8 ++++-- tools/deploy-galaxy.js | 1 - 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index 0a51ca133b..4af8d2635b 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -13,8 +13,12 @@ AppConfig.findGalaxy = _.once(function () { var ultra = AppConfig.findGalaxy(); var subFuture = new Future(); -if (ultra) +var subFutureJobs = new Future(); +if (ultra) { ultra.subscribe("oneApp", process.env.GALAXY_APP, subFuture.resolver()); + ultra.subscribe("oneJob", process.env.GALAXY_APP, subFutureJobs.resolver()); +} + var OneAppApps; var Services; var collectionFuture = new Future(); @@ -24,6 +28,9 @@ Meteor.startup(function () { OneAppApps = new Meteor.Collection("apps", { connection: ultra }); + OneAppJobs = new Meteor.Collection("jobs", { + connection: ultra + }); Services = new Meteor.Collection('services', { connection: ultra }); @@ -39,6 +46,12 @@ AppConfig._getAppCollection = function () { return OneAppApps; }; +AppConfig._getJobsCollection = function () { + collectionFuture.wait(); + return OneAppJobs; +}; + + var staticAppConfig; try { @@ -81,6 +94,17 @@ AppConfig.getAppConfig = function () { return config; }; +AppConfig.getStarForThisJob = function () { + if (ultra) { + subFutureJobs.wait(); + var job = OneAppJobs.findOne(process.env.GALAXY_JOB); + if (job) { + return job.star; + } + } + return ""; +}; + AppConfig.configurePackage = function (packageName, configure) { var appConfig = AppConfig.getAppConfig(); // Will either be based in the env var, // or wait for galaxy to connect. @@ -124,7 +148,6 @@ AppConfig.configurePackage = function (packageName, configure) { }; }; - AppConfig.configureService = function (serviceName, configure) { if (ultra) { // there's a Meteor.startup() that produces the various collections, make diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index ae40585385..7058eb2270 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -687,8 +687,12 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { }; }; - var version = (proxyConfig.version) ? proxyConfig.version : ""; - + var version = ""; + var adminPrograms = ["panel", "ultraworld", "proxy"]; + if (!_.contains(adminPrograms, bindPathPrefix)) { + var AppConfig = Package["application-configuration"].AppConfig; + version = AppConfig.getStarForThisJob(); + } proxy.call('bindDdp', { pid: pid, bindTo: ddpBindTo, diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js index 5d2f70fbdf..ddff62077c 100644 --- a/tools/deploy-galaxy.js +++ b/tools/deploy-galaxy.js @@ -205,7 +205,6 @@ exports.deploy = function (options) { // fails (we already know the app exists.) var info = prettyCall(galaxy, 'beginUploadStar', [options.app, bundleResult.starManifest]); - console.log("StarId: ", info.id); // Upload // XXX copied from galaxy/tool/galaxy.js var fileSize = fs.statSync(starball).size; From 5750dcd7c6b30304215e1d2a70516fcebe516231 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 12 Dec 2013 17:18:07 -0800 Subject: [PATCH 056/112] Move the check for css to be loaded out of a string into code --- packages/webapp/package.js | 2 ++ packages/webapp/webapp_client.js | 7 +++++++ packages/webapp/webapp_server.js | 6 +----- 3 files changed, 10 insertions(+), 5 deletions(-) create mode 100644 packages/webapp/webapp_client.js diff --git a/packages/webapp/package.js b/packages/webapp/package.js index a824d65fc9..d32a012233 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -9,6 +9,7 @@ Npm.depends({connect: "2.9.0", Package.on_use(function (api) { api.use(['logging', 'underscore', 'routepolicy'], 'server'); + api.use(['underscore'], 'client'); api.use(['application-configuration', 'follower-livedata'], { unordered: true }); @@ -19,5 +20,6 @@ Package.on_use(function (api) { // loaded after webapp. api.export(['WebApp', 'main', 'WebAppInternals'], 'server'); api.add_files('webapp_server.js', 'server'); + api.add_files('webapp_client.js', 'client'); api.add_files('css_detect.css', 'client'); }); diff --git a/packages/webapp/webapp_client.js b/packages/webapp/webapp_client.js new file mode 100644 index 0000000000..6851791794 --- /dev/null +++ b/packages/webapp/webapp_client.js @@ -0,0 +1,7 @@ +Meteor._isCssLoaded = function () { + return _.find(document.styleSheets, function (sheet) { + return _.find(sheet.cssRules, function (rule) { + return rule.selectorText === '._meteor_detect_css'; + }); + }); +}; diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 7058eb2270..4b3c460200 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -22,11 +22,7 @@ var bundledJsCssPrefix; var RELOAD_SAFETYBELT = "\n" + "if (typeof Meteor === 'undefined' || \n" + - " ! _.find(document.styleSheets, function (sheet) { \n" + - " return _.find(sheet.cssRules, function (rule) { \n" + - " return rule.selectorText === '._meteor_detect_css'; \n" + - " }); \n" + - " })) \n" + + " ! Meteor._isCssLoaded()) \n" + " document.location.reload(); \n"; From 8e9ad07a09361cab19fe4292e6f14abd5b904469 Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 13 Dec 2013 10:46:20 -0800 Subject: [PATCH 057/112] use process.env.admin_app instead of a hardcoded array --- packages/webapp/webapp_server.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 4b3c460200..fcb7549f17 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -684,8 +684,7 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { }; var version = ""; - var adminPrograms = ["panel", "ultraworld", "proxy"]; - if (!_.contains(adminPrograms, bindPathPrefix)) { + if (!process.env.ADMIN_APP) { var AppConfig = Package["application-configuration"].AppConfig; version = AppConfig.getStarForThisJob(); } From d62969fbce2bd1bf1836c09b45b890508c61d748 Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 13 Dec 2013 11:53:02 -0800 Subject: [PATCH 058/112] Cleaning up previous propagation of version --- packages/application-configuration/config.js | 8 ++++---- packages/webapp/webapp_server.js | 1 - 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index 4af8d2635b..1080bf833f 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -90,7 +90,6 @@ AppConfig.getAppConfig = function () { throw new Error("there is no app config for this app"); } var config = myApp.config; - config.version = myApp.currentStar; return config; }; @@ -116,10 +115,11 @@ AppConfig.configurePackage = function (packageName, configure) { // is empty (synchronously, though deferred would be OK). // XXX make sure that all callers of configurePackage deal well with multiple // callback invocations! eg, email does not - configure(_.extend({version: appConfig.version}, lastConfig)); + configure(lastConfig); var configureIfDifferent = function (app) { - if (!EJSON.equals(app.config && app.config.packages && app.config.packages[packageName] && app.config.version, - lastConfig)) { + if (!EJSON.equals( + app.config && app.config.packages && app.config.packages[packageName], + lastConfig)) { lastConfig = app.config.packages[packageName]; configure(_.extend({version: appConfig.version}, lastConfig)); } diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index fcb7549f17..c257a026fd 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -587,7 +587,6 @@ var runWebAppServer = function () { } else { proxyConf = configuration.proxy; } - proxyConf.version = configuration.version; Log("Attempting to bind to proxy at " + proxyService.providers.proxy); WebAppInternals.bindToProxy(_.extend({ proxyEndpoint: proxyService.providers.proxy From 73f6d3c81c80221523c1919d66b6459b8cf96e5f Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 13 Dec 2013 13:21:37 -0800 Subject: [PATCH 059/112] Move code to adjust proxy binding tags to ctl-helper --- packages/application-configuration/config.js | 21 +++++++------- packages/ctl-helper/ctl-helper.js | 28 ++++++++++++++++++ packages/ctl-helper/package.js | 2 +- packages/ctl/ctl.js | 30 ++++---------------- packages/webapp/webapp_server.js | 2 +- 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index 1080bf833f..ee16561883 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -16,19 +16,20 @@ var subFuture = new Future(); var subFutureJobs = new Future(); if (ultra) { ultra.subscribe("oneApp", process.env.GALAXY_APP, subFuture.resolver()); - ultra.subscribe("oneJob", process.env.GALAXY_APP, subFutureJobs.resolver()); + ultra.subscribe("oneJob", process.env.GALAXY_JOB, subFutureJobs.resolver()); } -var OneAppApps; +var Apps; +var Jobs; var Services; var collectionFuture = new Future(); Meteor.startup(function () { if (ultra) { - OneAppApps = new Meteor.Collection("apps", { + Apps = new Meteor.Collection("apps", { connection: ultra }); - OneAppJobs = new Meteor.Collection("jobs", { + Jobs = new Meteor.Collection("jobs", { connection: ultra }); Services = new Meteor.Collection('services', { @@ -43,12 +44,12 @@ Meteor.startup(function () { // places. AppConfig._getAppCollection = function () { collectionFuture.wait(); - return OneAppApps; + return Apps; }; AppConfig._getJobsCollection = function () { collectionFuture.wait(); - return OneAppJobs; + return Jobs; }; @@ -85,7 +86,7 @@ AppConfig.getAppConfig = function () { return staticAppConfig; } subFuture.wait(); - var myApp = OneAppApps.findOne(process.env.GALAXY_APP); + var myApp = Apps.findOne(process.env.GALAXY_APP); if (!myApp) { throw new Error("there is no app config for this app"); } @@ -96,12 +97,12 @@ AppConfig.getAppConfig = function () { AppConfig.getStarForThisJob = function () { if (ultra) { subFutureJobs.wait(); - var job = OneAppJobs.findOne(process.env.GALAXY_JOB); + var job = Jobs.findOne(process.env.GALAXY_JOB); if (job) { return job.star; } } - return ""; + return null; }; AppConfig.configurePackage = function (packageName, configure) { @@ -133,7 +134,7 @@ AppConfig.configurePackage = function (packageName, configure) { // there's a Meteor.startup() that produces the various collections, make // sure it runs first before we continue. collectionFuture.wait(); - subHandle = OneAppApps.find(process.env.GALAXY_APP).observe({ + subHandle = Apps.find(process.env.GALAXY_APP).observe({ added: configureIfDifferent, changed: configureIfDifferent }); diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index d48e93a2b1..901bf9712a 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -133,6 +133,34 @@ _.extend(Ctl, { } }, + updateProxyActiveTags: function (tags) { + var proxy; + var proxyTagSwitchFuture = new Future; + AppConfig.configureService('proxy', function (proxyService) { + try { + proxy = Follower.connect(proxyService.providers.proxy, { + group: "proxy" + }); + proxy.call('updateTags', Ctl.myAppName(), tags); + proxy.disconnect(); + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['return'](); + } catch (e) { + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['throw'](e); + } + }); + + var proxyTimeout = Meteor.setTimeout(function () { + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['throw'](new Error("timed out looking for a proxy " + + "or trying to change tags on it " + + proxy.status().status)); + }, 10*1000); + proxyTagSwitchFuture.wait(); + Meteor.clearTimeout(proxyTimeout); + }, + jobsCollection: _.once(function () { return new Meteor.Collection("jobs", {manager: Ctl.findGalaxy()}); }), diff --git a/packages/ctl-helper/package.js b/packages/ctl-helper/package.js index 01a93e9fd2..2360d8dcfb 100644 --- a/packages/ctl-helper/package.js +++ b/packages/ctl-helper/package.js @@ -6,7 +6,7 @@ Package.describe({ Npm.depends({optimist: '0.6.0'}); Package.on_use(function (api) { - api.use(['underscore', 'livedata', 'mongo-livedata', 'follower-livedata'], 'server'); + api.use(['underscore', 'livedata', 'mongo-livedata', 'follower-livedata', 'application-configuration'], 'server'); api.export('Ctl', 'server'); api.add_files('ctl-helper.js', 'server'); }); diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index d989ef0167..76abbffa8f 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -37,6 +37,10 @@ var startFun = function (argv) { ); process.exit(1); } + Ctl.subscribeToAppJobs(Ctl.myAppName()); + var jobs = Ctl.jobsCollection(); + var thisJob = jobs.findOne(Ctl.myJobId()); + Ctl.updateProxyActiveTags(['', thisJob.star]); if (Ctl.hasProgram("console")) { console.log("starting console for app", Ctl.myAppName()); Ctl.startServerlikeProgramIfNotPresent("console", ["admin"], true); @@ -140,31 +144,7 @@ Ctl.Commands.push({ }); // Wait for them all to come up and bind to the proxy. Meteor._sleepForMs(5000); // XXX: Eventually make sure they're proxy-bound. - var proxy; - var proxyTagSwitchFuture = new Future; - AppConfig.configureService('proxy', function (proxyService) { - try { - proxy = Follower.connect(proxyService.providers.proxy, { - group: "proxy" - }); - proxy.call('updateTags', Ctl.myAppName(), ['', thisJob.star]); - proxy.disconnect(); - if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['return'](); - } catch (e) { - if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['throw'](e); - } - }); - - var proxyTimeout = Meteor.setTimeout(function () { - if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['throw'](new Error("timed out looking for a proxy " + - "or trying to change tags on it " + - proxy.status().status)); - }, 10*1000); - proxyTagSwitchFuture.wait(); - Meteor.clearTimeout(proxyTimeout); + Ctl.updateProxyActiveTags(['', thisJob.star]); // (eventually) tell the proxy to switch over to using the new star // One by one, kill all the old star's server jobs. diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index c257a026fd..ff8b972c01 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -685,7 +685,7 @@ WebAppInternals.bindToProxy = function (proxyConfig, proxyServiceName) { var version = ""; if (!process.env.ADMIN_APP) { var AppConfig = Package["application-configuration"].AppConfig; - version = AppConfig.getStarForThisJob(); + version = AppConfig.getStarForThisJob() || ""; } proxy.call('bindDdp', { pid: pid, From 7449577ef411ac04ca719a4e46200cff1abb5b36 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 13 Dec 2013 14:40:10 -0800 Subject: [PATCH 060/112] Did my own phabricator comments; mostly small touches --- packages/application-configuration/config.js | 2 +- packages/ctl-helper/ctl-helper.js | 7 ++++--- packages/ctl/ctl.js | 4 +++- packages/webapp/webapp_server.js | 5 +++++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index ee16561883..7268dc2ae8 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -122,7 +122,7 @@ AppConfig.configurePackage = function (packageName, configure) { app.config && app.config.packages && app.config.packages[packageName], lastConfig)) { lastConfig = app.config.packages[packageName]; - configure(_.extend({version: appConfig.version}, lastConfig)); + configure(lastConfig); } }; var subHandle; diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index 901bf9712a..885c050da9 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -153,9 +153,10 @@ _.extend(Ctl, { var proxyTimeout = Meteor.setTimeout(function () { if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['throw'](new Error("timed out looking for a proxy " + - "or trying to change tags on it " + - proxy.status().status)); + proxyTagSwitchFuture['throw']( + new Error("timed out looking for a proxy " + + "or trying to change tags on it " + + proxy.status().status)); }, 10*1000); proxyTagSwitchFuture.wait(); Meteor.clearTimeout(proxyTimeout); diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 76abbffa8f..185c5c4f67 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -140,7 +140,9 @@ Ctl.Commands.push({ var oldServers = jobs.find(oldJobSelector).fetch(); // Start a new job for each of them. _.each(oldServers, function (oldServer) { - Ctl.startServerlikeProgram("server", oldServer.tags, oldServer.env.ADMIN_APP); + Ctl.startServerlikeProgram("server", + oldServer.tags, + oldServer.env.ADMIN_APP); }); // Wait for them all to come up and bind to the proxy. Meteor._sleepForMs(5000); // XXX: Eventually make sure they're proxy-bound. diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index ff8b972c01..5ca19625d9 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -20,6 +20,11 @@ WebAppInternals = {}; var bundledJsCssPrefix; +// The reload safetybelt is some js that will be loaded after everything else in +// the HTML. In some multi-server deployments, when you update, you have a +// chance of hitting an old server for the HTML and the new server for the JS or +// CSS. This prevents you from displaying the page in that case, and instead +// reloads it, presumably all on the new version now. var RELOAD_SAFETYBELT = "\n" + "if (typeof Meteor === 'undefined' || \n" + " ! Meteor._isCssLoaded()) \n" + From 001d2df9682ef4579e0b486ec77af402a5b21a62 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 13 Dec 2013 03:46:11 -0800 Subject: [PATCH 061/112] Retry subscription on error. --- packages/autoupdate/autoupdate_client.js | 61 ++++++++++++++++++------ packages/livedata/livedata_common.js | 5 ++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/packages/autoupdate/autoupdate_client.js b/packages/autoupdate/autoupdate_client.js index 34e90184f9..dfd4063ddd 100644 --- a/packages/autoupdate/autoupdate_client.js +++ b/packages/autoupdate/autoupdate_client.js @@ -43,19 +43,50 @@ Autoupdate.newClientAvailable = function () { }; -Meteor.subscribe("meteor_autoupdate_clientVersions", { - onError: function (error) { - Meteor._debug("autoupdate subscription failed:", error); - }, - onReady: function () { - if (Package.reload) { - Deps.autorun(function (computation) { - if (ClientVersions.findOne({current: true}) && - (! ClientVersions.findOne({_id: autoupdateVersion}))) { - computation.stop(); - Package.reload.Reload._reload(); - } - }); - } - } + +// XXX Livedata exporting this via DDP is a hack. See +// packages/livedata/livedata_common.js +var retry = new DDP._Retry({ + // Unlike the stream reconnect use of Retry, which we want to be instant + // in normal operation, this is a wacky failure. We don't want to retry + // right away, we can start slowly. + // + // A better way than timeconstants here might be to use the knowledge + // of when we reconnect to help trigger these retries. Typically, the + // server fixing code will result in a restart and reconnect, but + // potentially the subscription could have a transient error. + minCount: 0, // don't do any immediate retries + baseTimeout: 30*1000 // start with 30s }); +var failures = 0; + +Autoupdate._retrySubscription = function () { + Meteor.subscribe("meteor_autoupdate_clientVersions", { + onError: function (error) { + Meteor._debug("autoupdate subscription failed:", error); + failures++; + retry.retryLater(failures, function () { + // Just retry making the subscription, don't reload the whole + // page. While reloading would catch more cases (for example, + // the server went back a version and is now doing old-style hot + // code push), it would also be more prone to reload loops, + // which look really bad to the user. Just retrying the + // subscription over DDP means it is at least possible to fix by + // updating the server. + Autoupdate._retrySubscription(); + }); + }, + onReady: function () { + if (Package.reload) { + Deps.autorun(function (computation) { + if (ClientVersions.findOne({current: true}) && + (! ClientVersions.findOne({_id: autoupdateVersion}))) { + computation.stop(); + Package.reload.Reload._reload(); + } + }); + } + } + }); +}; +Autoupdate._retrySubscription(); diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index 9c3d31bc78..f0b6159ec6 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -115,3 +115,8 @@ stringifyDDP = function (msg) { // state in the DDP session. Meteor.setTimeout and friends clear // it. We can probably find a better way to factor this. DDP._CurrentInvocation = new Meteor.EnvironmentVariable; + + +// This is private and a hack. It is used by autoupdate_client. We +// should refactor. Maybe a separate 'exponential-backoff' package? +DDP._Retry = Retry; From cb4a278d4bffbc1fd2f61b40de29cdc5509d90e6 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Fri, 13 Dec 2013 17:53:52 -0800 Subject: [PATCH 062/112] Revert "Before minifying CSS pull out all @import's to the top of CSS file." This reverts commit 0d5060eb6a1a9115515c98c0d03a2f2aec6bb458. This reverts commit f2e3481db3c50c7dd1384977981bbaf633bd0b47. --- packages/minifiers/minifiers.js | 15 ------------- packages/minifiers/minifiers_tests.js | 32 --------------------------- packages/minifiers/package.js | 8 ------- 3 files changed, 55 deletions(-) delete mode 100644 packages/minifiers/minifiers_tests.js diff --git a/packages/minifiers/minifiers.js b/packages/minifiers/minifiers.js index e96e4c309c..ad15487d26 100644 --- a/packages/minifiers/minifiers.js +++ b/packages/minifiers/minifiers.js @@ -1,23 +1,8 @@ var CleanCss = Npm.require('clean-css'); CleanCSSProcess = function (source, options) { - options = _.extend({ processImport: false }, options); var instance = new CleanCss(options); - // after concatenation some @import's might be left in the middle of CSS file - // but they required to be in the beginning. - source = CSSPullImports(source); return instance.minify(source); }; UglifyJSMinify = Npm.require('uglify-js').minify; - -var CSSPullImports = function (source) { - var importRegExp = /^\s*@import\s*[^;]*;\s*/gm; - var imports = source.match(importRegExp) || []; - var newSource = source.replace(importRegExp, ''); - - newSource = imports.join('') + newSource; - - return newSource; -} - diff --git a/packages/minifiers/minifiers_tests.js b/packages/minifiers/minifiers_tests.js deleted file mode 100644 index af68fcd571..0000000000 --- a/packages/minifiers/minifiers_tests.js +++ /dev/null @@ -1,32 +0,0 @@ -Tinytest.add("minifiers - CSS pull the imports to the top", function (test) { - function t(source, expected, descr) { - var processed = CleanCSSProcess(source); - test.equal(processed, expected, descr); - } - - t(["@import url(//fonts.googleapis.com/css?family=Tangerine);", - ".block1{font-family:Tangerine}", - "@import url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", - ".block2{font-family:'Oleo Script Swash Caps'}"].join('\n'), - ["@import url(//fonts.googleapis.com/css?family=Tangerine);", - "@import url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", - ".block1{font-family:Tangerine}", - ".block2{font-family:'Oleo Script Swash Caps'}"].join(''), "just imports"); - t(["@import url(//fonts.googleapis.com/css?family=Tangerine),", - " url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", - ".block2{font-family:'Oleo Script Swash Caps'}"].join('\n'), - ["@import url(//fonts.googleapis.com/css?family=Tangerine),", - "url('//fonts.googleapis.com/css?family=Oleo Script Swash Caps');", - ".block2{font-family:'Oleo Script Swash Caps'}"].join(''), "multiline @imports"); - t([".block2{font-family:'Oleo Script Swash Caps'}", - "div[x='@import asdf;']{font:#000}"].join('\n'), - [".block2{font-family:'Oleo Script Swash Caps'}", - "div[x='@import asdf;']{font:#000}"].join(''), "@import embeded as string"); - t(["@import url(//bla.com/font.css);.block1{font-family:Tangerine}", - "@import url(//bla2.com/font2.css);.block2{font-family:Font2}"].join('\n'), - ["@import url(//bla.com/font.css);", - "@import url(//bla2.com/font2.css);", - ".block1{font-family:Tangerine}", - ".block2{font-family:Font2}"].join('')); -}); - diff --git a/packages/minifiers/package.js b/packages/minifiers/package.js index 3e050106fa..c76efd3ee9 100644 --- a/packages/minifiers/package.js +++ b/packages/minifiers/package.js @@ -9,14 +9,6 @@ Npm.depends({ }); Package.on_use(function (api) { - api.use('underscore'); api.export(['CleanCSSProcess', 'UglifyJSMinify']); api.add_files('minifiers.js', 'server'); }); - -Package.on_test(function (api) { - api.use('minifiers'); - api.use('tinytest'); - api.add_files('minifiers_tests.js', 'server'); -}); - From fd215633cc32498b7b189716c1fedac514d40ea4 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 16:43:12 -0800 Subject: [PATCH 063/112] Make observe driver facts names consistent --- packages/mongo-livedata/oplog_observe_driver.js | 4 ++-- packages/mongo-livedata/polling_observe_driver.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 7e3da7d1f1..ac2094ba75 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -26,7 +26,7 @@ OplogObserveDriver = function (options) { self._stopHandles = []; Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "oplog-observers", 1); + "mongo-livedata", "observe-drivers-oplog", 1); self._phase = PHASE.QUERYING; @@ -435,7 +435,7 @@ _.extend(OplogObserveDriver.prototype, { self._listenersHandle = null; Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "oplog-observers", -1); + "mongo-livedata", "observe-drivers-oplog", -1); } }); diff --git a/packages/mongo-livedata/polling_observe_driver.js b/packages/mongo-livedata/polling_observe_driver.js index 938798e519..82020f2147 100644 --- a/packages/mongo-livedata/polling_observe_driver.js +++ b/packages/mongo-livedata/polling_observe_driver.js @@ -72,7 +72,7 @@ PollingObserveDriver = function (options) { self._unthrottledEnsurePollIsScheduled(); Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "mongo-pollsters", 1); + "mongo-livedata", "observe-drivers-polling", 1); }; _.extend(PollingObserveDriver.prototype, { @@ -174,6 +174,6 @@ _.extend(PollingObserveDriver.prototype, { self._stopped = true; _.each(self._stopCallbacks, function (c) { c(); }); Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "mongo-pollsters", -1); + "mongo-livedata", "observe-drivers-polling", -1); } }); From 8b37c0084d18f47c50169464807b277e7b48b5f3 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 12 Dec 2013 16:02:02 -0800 Subject: [PATCH 064/112] Make facts automatically subscribe when you show the template. Also, namespace the collection and subscription name. --- History.md | 5 +++-- packages/facts/facts.js | 22 ++++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/History.md b/History.md index 8e5d4c96dd..6bb2fe51e5 100644 --- a/History.md +++ b/History.md @@ -30,8 +30,9 @@ apply the patch and will instead disable websockets. client code changes; server-only code changes will not cause the page to reload. -* New 'facts' package publishes internal statistics about Meteor. - XXX how to use +* New 'facts' package publishes internal statistics about Meteor. To + use, simply `meteor add facts` then add `{{> serverFacts}}` somewhere + in your interface. * Add an explicit check that publish functions return a cursor, an array of cursors, or a falsey value. This is a safety check to to prevent diff --git a/packages/facts/facts.js b/packages/facts/facts.js index 65f4517530..543023b271 100644 --- a/packages/facts/facts.js +++ b/packages/facts/facts.js @@ -1,6 +1,6 @@ Facts = {}; -var serverFactsCollection = 'Facts.server'; +var serverFactsCollection = 'meteor_Facts_server'; if (Meteor.isServer) { // By default, we publish facts to no user if autopublish is off, and to all @@ -45,7 +45,7 @@ if (Meteor.isServer) { // called? Meteor.defer(function () { // XXX Also publish facts-by-package. - Meteor.publish("facts", function () { + Meteor.publish("meteor_facts", function () { var sub = this; if (!userIdFilter(this.userId)) { sub.ready(); @@ -59,13 +59,10 @@ if (Meteor.isServer) { activeSubscriptions = _.without(activeSubscriptions, sub); }); sub.ready(); - }); + }, {is_auto: true}); }); } else { Facts.server = new Meteor.Collection(serverFactsCollection); - // XXX making all clients subscribe all the time is wasteful. - // add an interface here - // Meteor.subscribe("facts"); Template.serverFacts.factsByPackage = function () { return Facts.server.find(); @@ -78,4 +75,17 @@ if (Meteor.isServer) { }); return factArray; }; + + // Subscribe when the template is first made, and unsubscribe when it + // is removed. If for some reason puts two copies of the template on + // the screen at once, we'll subscribe twice. Meh. + Template.serverFacts.created = function () { + this._stopHandle = Meteor.subscribe("meteor_facts"); + }; + Template.serverFacts.destroyed = function () { + if (this._stopHandle) { + this._stopHandle.stop(); + this._stopHandle = null; + } + }; } From 20e8e4241c68d526257515f9f83412d615df86d6 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 13 Dec 2013 03:46:11 -0800 Subject: [PATCH 065/112] Retry subscription on error. --- packages/autoupdate/autoupdate_client.js | 61 ++++++++++++++++++------ packages/livedata/livedata_common.js | 5 ++ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/packages/autoupdate/autoupdate_client.js b/packages/autoupdate/autoupdate_client.js index 34e90184f9..dfd4063ddd 100644 --- a/packages/autoupdate/autoupdate_client.js +++ b/packages/autoupdate/autoupdate_client.js @@ -43,19 +43,50 @@ Autoupdate.newClientAvailable = function () { }; -Meteor.subscribe("meteor_autoupdate_clientVersions", { - onError: function (error) { - Meteor._debug("autoupdate subscription failed:", error); - }, - onReady: function () { - if (Package.reload) { - Deps.autorun(function (computation) { - if (ClientVersions.findOne({current: true}) && - (! ClientVersions.findOne({_id: autoupdateVersion}))) { - computation.stop(); - Package.reload.Reload._reload(); - } - }); - } - } + +// XXX Livedata exporting this via DDP is a hack. See +// packages/livedata/livedata_common.js +var retry = new DDP._Retry({ + // Unlike the stream reconnect use of Retry, which we want to be instant + // in normal operation, this is a wacky failure. We don't want to retry + // right away, we can start slowly. + // + // A better way than timeconstants here might be to use the knowledge + // of when we reconnect to help trigger these retries. Typically, the + // server fixing code will result in a restart and reconnect, but + // potentially the subscription could have a transient error. + minCount: 0, // don't do any immediate retries + baseTimeout: 30*1000 // start with 30s }); +var failures = 0; + +Autoupdate._retrySubscription = function () { + Meteor.subscribe("meteor_autoupdate_clientVersions", { + onError: function (error) { + Meteor._debug("autoupdate subscription failed:", error); + failures++; + retry.retryLater(failures, function () { + // Just retry making the subscription, don't reload the whole + // page. While reloading would catch more cases (for example, + // the server went back a version and is now doing old-style hot + // code push), it would also be more prone to reload loops, + // which look really bad to the user. Just retrying the + // subscription over DDP means it is at least possible to fix by + // updating the server. + Autoupdate._retrySubscription(); + }); + }, + onReady: function () { + if (Package.reload) { + Deps.autorun(function (computation) { + if (ClientVersions.findOne({current: true}) && + (! ClientVersions.findOne({_id: autoupdateVersion}))) { + computation.stop(); + Package.reload.Reload._reload(); + } + }); + } + } + }); +}; +Autoupdate._retrySubscription(); diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index 9c3d31bc78..f0b6159ec6 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -115,3 +115,8 @@ stringifyDDP = function (msg) { // state in the DDP session. Meteor.setTimeout and friends clear // it. We can probably find a better way to factor this. DDP._CurrentInvocation = new Meteor.EnvironmentVariable; + + +// This is private and a hack. It is used by autoupdate_client. We +// should refactor. Maybe a separate 'exponential-backoff' package? +DDP._Retry = Retry; From 2f745bb52f010c487027406a621e76705878b525 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 00:38:20 -0800 Subject: [PATCH 066/112] Test changing Date and ObjectID --- .../mongo-livedata/mongo_livedata_tests.js | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 299d34bbd6..1f163b4e4b 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -2071,7 +2071,7 @@ EJSON.addType('someCustomType', function (json) { return new TestCustomType(json.head, json.tail); }); -testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ +testAsyncMulti("mongo-livedata - oplog - update EJSON", [ function (test, expect) { var self = this; var collectionName = "ejson" + Random.id(); @@ -2081,9 +2081,12 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ } self.collection = new Meteor.Collection(collectionName); + self.date = new Date; + self.objId = new Meteor.Collection.ObjectID; self.id = self.collection.insert( - {name: 'foo', custom: new TestCustomType('a', 'b')}, + {d: self.date, oi: self.objId, + custom: new TestCustomType('a', 'b')}, expect(function (err, res) { test.isFalse(err); test.equal(self.id, res); @@ -2106,7 +2109,8 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ test.length(self.changes, 1); test.equal(self.changes.shift(), ['a', self.id, - {name: 'foo', custom: new TestCustomType('a', 'b')}]); + {d: self.date, oi: self.objId, + custom: new TestCustomType('a', 'b')}]); // First, replace the entire custom object. // (runInFence is useful for the server, using expect() is useful for the @@ -2140,6 +2144,24 @@ testAsyncMulti("mongo-livedata - oplog - update inside EJSON", [ test.length(self.changes, 1); test.equal(self.changes.shift(), ['c', self.id, {custom: new TestCustomType('a', 'd')}]); + + // Update a date and an ObjectID too. + self.date2 = new Date(self.date.valueOf() + 1000); + self.objId2 = new Meteor.Collection.ObjectID; + runInFence(function () { + self.collection.update( + self.id, {$set: {d: self.date2, oi: self.objId2}}, + expect(function (err) { + test.isFalse(err); + })); + }); + }, + function (test, expect) { + var self = this; + test.length(self.changes, 1); + test.equal(self.changes.shift(), + ['c', self.id, {d: self.date2, oi: self.objId2}]); + self.handle.stop(); } ]); From 2991ac931221dc43b11444ebea189e854944628d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 11 Dec 2013 23:37:16 -0800 Subject: [PATCH 067/112] fix missing expect() call in password-tests add a console.trace that helps debug it --- packages/accounts-password/password_tests.js | 4 ++-- packages/tinytest/tinytest.js | 11 ++++++++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/accounts-password/password_tests.js b/packages/accounts-password/password_tests.js index 1da7dff9a4..fc71e0d317 100644 --- a/packages/accounts-password/password_tests.js +++ b/packages/accounts-password/password_tests.js @@ -464,10 +464,10 @@ if (Meteor.isClient) (function () { function (test, expect) { var self = this; // Test that deleting a user logs out that user's connections. - Meteor.loginWithPassword(this.username, this.password, function (err) { + Meteor.loginWithPassword(this.username, this.password, expect(function (err) { test.isFalse(err); Meteor.call("removeUser", self.username); - }); + })); }, waitForLoggedOutStep ]); diff --git a/packages/tinytest/tinytest.js b/packages/tinytest/tinytest.js index 66c6d42063..de212efb50 100644 --- a/packages/tinytest/tinytest.js +++ b/packages/tinytest/tinytest.js @@ -309,7 +309,16 @@ _.extend(TestCase.prototype, { return true; }; - var results = new TestCaseResults(self, onEvent, + var wrappedOnEvent = function (e) { + // If this trace prints, it means you ran some test.* function after the + // test finished! Another symptom will be that the test will display as + // "waiting" even when it counts as passed or failed. + if (completed) + console.trace("event after complete!"); + return onEvent(e); + }; + + var results = new TestCaseResults(self, wrappedOnEvent, function (e) { if (markComplete()) onException(e); From 9dfa0ee95f98b792c3d8895ec8634aea6535a7d6 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Wed, 11 Dec 2013 20:38:51 -0800 Subject: [PATCH 068/112] Update tools tests for Webapp bundled asset change 6eccf8c --- tools/tests/test_bundler_options.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tools/tests/test_bundler_options.js b/tools/tests/test_bundler_options.js index b3affcbe0c..76a5b0a382 100644 --- a/tools/tests/test_bundler_options.js +++ b/tools/tests/test_bundler_options.js @@ -34,8 +34,8 @@ assert.doesNotThrow(function () { // verify that contents are minified var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html"), 'utf8'); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml)); - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/packages/.test(appHtml))); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml)); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages/.test(appHtml))); }); console.log("nodeModules: 'skip', no minify"); @@ -50,11 +50,11 @@ assert.doesNotThrow(function () { // verify that contents are not minified var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html"), 'utf8'); - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml))); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/meteor/.test(appHtml)); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/deps/.test(appHtml)); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/[0-9a-f]{40,40}.js\"/.test(appHtml))); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/meteor/.test(appHtml)); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/deps/.test(appHtml)); // verify that tests aren't included - assert(!(/src=\"##ROOT_URL_PATH_PREFIX##\/package-tests\/meteor/.test(appHtml))); + assert(!(/src=\"##BUNDLED_JS_CSS_PREFIX##\/package-tests\/meteor/.test(appHtml))); }); console.log("nodeModules: 'skip', no minify, testPackages: ['meteor']"); @@ -70,7 +70,7 @@ assert.doesNotThrow(function () { // verify that tests for the meteor package are included var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", "client", "app.html")); - assert(/src=\"##ROOT_URL_PATH_PREFIX##\/packages\/meteor:tests\.js/.test(appHtml)); + assert(/src=\"##BUNDLED_JS_CSS_PREFIX##\/packages\/meteor:tests\.js/.test(appHtml)); }); console.log("nodeModules: 'copy'"); From a6c8b5eef3f67b712fc010fb6b4def497b028573 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Sun, 15 Dec 2013 19:40:57 -0800 Subject: [PATCH 069/112] Give up on documenting facts current api. --- History.md | 4 +--- packages/facts/package.js | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/History.md b/History.md index 6bb2fe51e5..925f6a5e94 100644 --- a/History.md +++ b/History.md @@ -30,9 +30,7 @@ apply the patch and will instead disable websockets. client code changes; server-only code changes will not cause the page to reload. -* New 'facts' package publishes internal statistics about Meteor. To - use, simply `meteor add facts` then add `{{> serverFacts}}` somewhere - in your interface. +* New 'facts' package publishes internal statistics about Meteor. * Add an explicit check that publish functions return a cursor, an array of cursors, or a falsey value. This is a safety check to to prevent diff --git a/packages/facts/package.js b/packages/facts/package.js index 1f835c15fe..f36b50d054 100644 --- a/packages/facts/package.js +++ b/packages/facts/package.js @@ -1,5 +1,6 @@ Package.describe({ - summary: "Publish internal app statistics" + summary: "Publish internal app statistics", + internal: true }); Package.on_use(function (api) { From 37c63ffae72c250b22a9142f74a6bac7836403a6 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Mon, 16 Dec 2013 12:06:52 -0800 Subject: [PATCH 070/112] Package instead of Meteor to measure js loading; allow use of webapp a la carte Also change the timing to allow more time for new servers to come up. --- packages/ctl/ctl.js | 4 ++-- packages/webapp/package.js | 1 + packages/webapp/webapp_client.js | 13 ++++++++----- packages/webapp/webapp_server.js | 6 ++++-- 4 files changed, 15 insertions(+), 9 deletions(-) diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 185c5c4f67..5c4ce425a2 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -145,7 +145,7 @@ Ctl.Commands.push({ oldServer.env.ADMIN_APP); }); // Wait for them all to come up and bind to the proxy. - Meteor._sleepForMs(5000); // XXX: Eventually make sure they're proxy-bound. + Meteor._sleepForMs(10000); // XXX: Eventually make sure they're proxy-bound. Ctl.updateProxyActiveTags(['', thisJob.star]); // (eventually) tell the proxy to switch over to using the new star @@ -156,7 +156,7 @@ Ctl.Commands.push({ // Wait for it to go down waitForDone(jobs, jobToKill._id); // Spend some time in between to allow any reconnect storm to die down. - Meteor._sleepForMs(1000); + Meteor._sleepForMs(5000); jobToKill = jobs.findOne(oldJobSelector); } // Now kill all old non-server jobs. They're less important. diff --git a/packages/webapp/package.js b/packages/webapp/package.js index d32a012233..a8f9d66e66 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -19,6 +19,7 @@ Package.on_use(function (api) { // way on browser-policy here, but we use it when it is loaded, and it can be // loaded after webapp. api.export(['WebApp', 'main', 'WebAppInternals'], 'server'); + api.export(['WebApp'], 'client'); api.add_files('webapp_server.js', 'server'); api.add_files('webapp_client.js', 'client'); api.add_files('css_detect.css', 'client'); diff --git a/packages/webapp/webapp_client.js b/packages/webapp/webapp_client.js index 6851791794..15ae6311af 100644 --- a/packages/webapp/webapp_client.js +++ b/packages/webapp/webapp_client.js @@ -1,7 +1,10 @@ -Meteor._isCssLoaded = function () { - return _.find(document.styleSheets, function (sheet) { - return _.find(sheet.cssRules, function (rule) { - return rule.selectorText === '._meteor_detect_css'; +WebApp = { + + _isCssLoaded: function () { + return _.find(document.styleSheets, function (sheet) { + return _.find(sheet.cssRules, function (rule) { + return rule.selectorText === '._meteor_detect_css'; + }); }); - }); + } }; diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 5ca19625d9..71e984635c 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -26,8 +26,10 @@ var bundledJsCssPrefix; // CSS. This prevents you from displaying the page in that case, and instead // reloads it, presumably all on the new version now. var RELOAD_SAFETYBELT = "\n" + - "if (typeof Meteor === 'undefined' || \n" + - " ! Meteor._isCssLoaded()) \n" + + "if (typeof Package === 'undefined' || \n" + + " ! Package.webapp || \n" + + " ! Package.webapp.WebApp || \n" + + " ! Package.webapp.WebApp._isCssLoaded()) \n" + " document.location.reload(); \n"; From e6c87236a6286664e43b75235be419ef0229ebcc Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 16 Dec 2013 15:14:43 -0800 Subject: [PATCH 071/112] Fix "drop collection" test failure There was a race condition in the manipulation of the write fence by the drop collection code. Specifically, when seeing a "drop collection" oplog entry, OplogObserveDriver took no immediate action and instead deferred some "should I re-poll now or later?" logic. This allowed the write fence's oplogHandle.waitUntilCaughtUp code to think that everything was "caught up" even though the effects of the "drop collection" entry had not yet affected the cursor's state (by transitioning it away from the STEADY state). The fix is to ensure that the state-change aspects of processing the entry occur immediately; the actual re-query is still deferred in order to not block the oplog tailer from continuing. Reported by: @awwx (The previous failure can made consistently reproducible by replacing the `Meteor.defer` that is removed by this commit with a `Meteor.setTimeout(,200)`.) --- .../mongo-livedata/oplog_observe_driver.js | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index dbd44c310b..e4bbaccb73 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -55,11 +55,10 @@ OplogObserveDriver = function (options) { trigger, function (notification) { var op = notification.op; if (notification.dropCollection) { - // Defer because it may block on "wait for oplog to catch up", which - // isn't kosher for an oplog entry handler (will cause deadlock). - Meteor.defer(function () { - self._needToPollQuery(); - }); + // Note: this call is not allowed to block on anything (especially on + // waiting for oplog entries to catch up) because that will block + // onOplogEntry! + self._needToPollQuery(); } else { // All other operators should be handled depending on phase if (self._phase === PHASE.QUERYING) @@ -295,6 +294,9 @@ _.extend(OplogObserveDriver.prototype, { // In various circumstances, we may just want to stop processing the oplog and // re-run the initial query, just as if we were a PollingObserveDriver. // + // This function may not block, because it is called from an oplog entry + // handler. + // // XXX We should call this when we detect that we've been in FETCHING for "too // long". // @@ -315,18 +317,27 @@ _.extend(OplogObserveDriver.prototype, { ++self._fetchGeneration; // ignore any in-flight fetches self._phase = PHASE.QUERYING; - // subtle note: _published does not contain _id fields, but newResults does - var newResults = new LocalCollection._IdMap; - var cursor = self._cursorForQuery(); - cursor.forEach(function (doc) { - newResults.set(doc._id, doc); + // Defer so that we don't block. + Meteor.defer(function () { + // subtle note: _published does not contain _id fields, but newResults + // does + var newResults = new LocalCollection._IdMap; + var cursor = self._cursorForQuery(); + cursor.forEach(function (doc) { + newResults.set(doc._id, doc); + }); + + self._publishNewResults(newResults); + + self._doneQuerying(); }); - - self._publishNewResults(newResults); - - self._doneQuerying(); }, + // Transitions to QUERYING and runs another query, or (if already in QUERYING) + // ensures that we will query again later. + // + // This function may not block, because it is called from an oplog entry + // handler. _needToPollQuery: function () { var self = this; if (self._stopped) From cce9d36dfaa3b50a894db4ed267a8ae2e5ba87a0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 12 Dec 2013 15:55:06 -0800 Subject: [PATCH 072/112] Add some XXX comments about when we should re-poll --- packages/mongo-livedata/oplog_observe_driver.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index ac2094ba75..dbd44c310b 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -294,6 +294,15 @@ _.extend(OplogObserveDriver.prototype, { // In various circumstances, we may just want to stop processing the oplog and // re-run the initial query, just as if we were a PollingObserveDriver. + // + // XXX We should call this when we detect that we've been in FETCHING for "too + // long". + // + // XXX We should call this when we detect Mongo failover (since that might + // mean that some of the oplog entries we have processed have been rolled + // back). The Node Mongo driver is in the middle of a bunch of huge + // refactorings, including the way that it notifies you when primary + // changes. Will put off implementing this until driver 1.4 is out. _pollQuery: function () { var self = this; From e042e5a85a0fb38b51637aee2715564272ec1fdc Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 16 Dec 2013 15:14:43 -0800 Subject: [PATCH 073/112] Fix "drop collection" test failure There was a race condition in the manipulation of the write fence by the drop collection code. Specifically, when seeing a "drop collection" oplog entry, OplogObserveDriver took no immediate action and instead deferred some "should I re-poll now or later?" logic. This allowed the write fence's oplogHandle.waitUntilCaughtUp code to think that everything was "caught up" even though the effects of the "drop collection" entry had not yet affected the cursor's state (by transitioning it away from the STEADY state). The fix is to ensure that the state-change aspects of processing the entry occur immediately; the actual re-query is still deferred in order to not block the oplog tailer from continuing. Reported by: @awwx (The previous failure can made consistently reproducible by replacing the `Meteor.defer` that is removed by this commit with a `Meteor.setTimeout(,200)`.) --- .../mongo-livedata/oplog_observe_driver.js | 39 ++++++++++++------- 1 file changed, 25 insertions(+), 14 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index dbd44c310b..e4bbaccb73 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -55,11 +55,10 @@ OplogObserveDriver = function (options) { trigger, function (notification) { var op = notification.op; if (notification.dropCollection) { - // Defer because it may block on "wait for oplog to catch up", which - // isn't kosher for an oplog entry handler (will cause deadlock). - Meteor.defer(function () { - self._needToPollQuery(); - }); + // Note: this call is not allowed to block on anything (especially on + // waiting for oplog entries to catch up) because that will block + // onOplogEntry! + self._needToPollQuery(); } else { // All other operators should be handled depending on phase if (self._phase === PHASE.QUERYING) @@ -295,6 +294,9 @@ _.extend(OplogObserveDriver.prototype, { // In various circumstances, we may just want to stop processing the oplog and // re-run the initial query, just as if we were a PollingObserveDriver. // + // This function may not block, because it is called from an oplog entry + // handler. + // // XXX We should call this when we detect that we've been in FETCHING for "too // long". // @@ -315,18 +317,27 @@ _.extend(OplogObserveDriver.prototype, { ++self._fetchGeneration; // ignore any in-flight fetches self._phase = PHASE.QUERYING; - // subtle note: _published does not contain _id fields, but newResults does - var newResults = new LocalCollection._IdMap; - var cursor = self._cursorForQuery(); - cursor.forEach(function (doc) { - newResults.set(doc._id, doc); + // Defer so that we don't block. + Meteor.defer(function () { + // subtle note: _published does not contain _id fields, but newResults + // does + var newResults = new LocalCollection._IdMap; + var cursor = self._cursorForQuery(); + cursor.forEach(function (doc) { + newResults.set(doc._id, doc); + }); + + self._publishNewResults(newResults); + + self._doneQuerying(); }); - - self._publishNewResults(newResults); - - self._doneQuerying(); }, + // Transitions to QUERYING and runs another query, or (if already in QUERYING) + // ensures that we will query again later. + // + // This function may not block, because it is called from an oplog entry + // handler. _needToPollQuery: function () { var self = this; if (self._stopped) From 2859e30e60dd740d77e005d1df4b3815bef1231f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 16 Dec 2013 20:25:48 -0800 Subject: [PATCH 074/112] A few more History.md things --- History.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/History.md b/History.md index 925f6a5e94..37670e5d5f 100644 --- a/History.md +++ b/History.md @@ -22,7 +22,7 @@ apply the patch and will instead disable websockets. * Add `Meteor.onConnection` and add `this.connection` to method invocations and publish functions. These can be used to store data associated with individual clients between subscriptions and method - calls. See http://docs.meteor.com/#meteor_onconnection for details. + calls. See http://docs.meteor.com/#meteor_onconnection for details. #1611 * Rework hot code push. The new `autoupdate` package drives automatic reloads on update using standard DDP messages instead of a hardcoded @@ -51,13 +51,16 @@ apply the patch and will instead disable websockets. connections, kill DDP connections, and finish all outstanding requests for static assets. +* In the HTTP server, only keep sockets with no active HTTP requests alive for 5 + seconds. + * Fix handling of `fields` option in minimongo when only `_id` is present. #1651 * Fix issue where setting `process.env.MAIL_URL` in app code would not - alter where mail was sent. This was a regression from 0.6.6. #1649 + alter where mail was sent. This was a regression in 0.6.6 from 0.6.5. #1649 -* Prompt for passwords on stderr instead of stdout for easier automation - in shell scripts. #1600 +* Use stderr instead of stdout (for easier automation in shell scripts) when + prompting for passwords and when downloading the dev bundle. #1600 * Bundler failures cause non-zero exit code in `meteor run`. #1515 @@ -94,9 +97,10 @@ apply the patch and will instead disable websockets. * MongoDB from 2.4.6 to 2.4.8 * clean-css from 1.1.2 to 2.0.2 * uglify-js from a fork of 2.4.0 to 2.4.7 + * handlebars npm module no longer available outside of handlebars package -Patches contributed by GitHub users AlexeyMK, awwx, dandv, -DenisGorbachev, FooBarWidget, mitar, mcbain, rzymek, and sdarnell. +Patches contributed by GitHub users AlexeyMK, awwx, dandv, DenisGorbachev, +emgee3, FooBarWidget, mitar, mcbain, rzymek, and sdarnell. ## v0.6.6.3 From 46fb7b6d214c089984a26be6c8558be2e0b0e4da Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 16 Dec 2013 20:40:34 -0800 Subject: [PATCH 075/112] LICENSE update (script follows!) find . -name 'node_modules' | grep -v .meteor | xargs ls | cat | grep -ve '^\.' | sort | uniq > bar for i in `cat bar` ; do echo -n "$i " grep -e "^$i:" LICENSE.txt || echo "NOT FOUND" done --- LICENSE.txt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index ce588a1a86..420c8c773e 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -217,6 +217,7 @@ Copyright (c) 2011 Esa-Matti Suuronen esa-matti@suuronen.org ---------- node-gyp: https://github.com/TooTallNate/node-gyp keypress: https://github.com/TooTallNate/keypress +bindings: https://github.com/TooTallNate/node-bindings ---------- Copyright (c) 2012 Nathan Rajlich @@ -542,6 +543,14 @@ geojson-utils: https://github.com/maxogden/geojson-js-utils Copyright (c) 2010 Max Ogden + +---------- +bcrypt: https://github.com/ncb000gt/node.bcrypt.js +---------- + +Copyright (c) 2010 Nicholas Campbell + + ============== Apache License ============== @@ -641,11 +650,10 @@ BSD Licenses ============ ---------- -uglify-js: https://github.com/mishoo/UglifyJS +uglify-js: https://github.com/mishoo/UglifyJS2 ---------- -Copyright 2010 (c) Mihai Bazon -Based on parse-js (http://marijn.haverbeke.nl/parse-js/). +Copyright 2012-2013 (c) Mihai Bazon Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -1263,6 +1271,7 @@ IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------- npm-user-validate: https://github.com/robertkowalski/npm-user-validate +github-url-from-username-repo: https://github.com/robertkowalski/github-url-from-username-repo ---------- Copyright (c) Robert Kowalski @@ -1738,7 +1747,9 @@ The externally maintained libraries used by libuv are: ---------- nodejs: http://nodejs.org/ -readable-stream: https://github.com/isaacs/readable-stream/ +readable-stream: https://github.com/isaacs/readable-stream +npm: https://github.com/isaacs/npm +init-package-json: https://github.com/isaacs/init-package-json ---------- From a0734bfe58970547781090ead70ff9bd914130df Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 17 Dec 2013 01:42:49 -0800 Subject: [PATCH 076/112] strawman banner.txt and notices. --- scripts/admin/banner.txt | 7 ++++--- scripts/admin/notices.json | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index 99a12572dc..78e71c8c36 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,5 +1,6 @@ -=> Meteor 0.6.6.3: Fix CPU runaway while watching files in large - projects and occasional server crash on session disconnect. +=> Meteor 0.6.7: featuring a new live database driver that uses Mongo's + database replication log to watch queries more efficiently without + requerying the database on each update. This release is being downloaded in the background. Update your - project to Meteor 0.6.6.3 by running 'meteor update'. + project to Meteor 0.6.7 by running 'meteor update'. diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json index b15c708877..6e7d3e323d 100644 --- a/scripts/admin/notices.json +++ b/scripts/admin/notices.json @@ -72,6 +72,9 @@ { "release": "0.6.6.3" }, + { + "release": "0.6.7" + }, { "release": "NEXT" } From 9cb6df57fb9537ce2cbee868efbb684772f56b98 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 17 Dec 2013 01:51:49 -0800 Subject: [PATCH 077/112] Link to wiki page in History. --- History.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/History.md b/History.md index 37670e5d5f..da4ec32d9f 100644 --- a/History.md +++ b/History.md @@ -16,8 +16,9 @@ apply the patch and will instead disable websockets. with `meteor run`, and can be enabled in production with the `MONGO_OPLOG_URL` environment variable. Currently the only supported selectors are equality checks; `$`-operators, `limit` and `skip` - queries fall back to the original poll-and-diff algorithm. See for details. + queries fall back to the original poll-and-diff algorithm. See + https://github.com/meteor/meteor/wiki/Oplog-Observe-Driver + for details. * Add `Meteor.onConnection` and add `this.connection` to method invocations and publish functions. These can be used to store data From 4771b7173446eec9c2fbb94ef8b5d5ca6437c2c3 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 17 Dec 2013 10:48:54 -0800 Subject: [PATCH 078/112] Rename 0.6.7 to 0.7.0 --- docs/lib/release-override.js | 2 +- packages/livedata/stream_server.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js index 8b66d7eea5..0f61b680aa 100644 --- a/docs/lib/release-override.js +++ b/docs/lib/release-override.js @@ -1,5 +1,5 @@ // While galaxy apps are on their own special meteor releases, override // Meteor.release here. if (Meteor.isClient) { - Meteor.release = Meteor.release ? "0.6.7" : undefined; + Meteor.release = Meteor.release ? "0.7.0" : undefined; } diff --git a/packages/livedata/stream_server.js b/packages/livedata/stream_server.js index 436d891dd3..8367eb2708 100644 --- a/packages/livedata/stream_server.js +++ b/packages/livedata/stream_server.js @@ -74,7 +74,7 @@ StreamServer = function () { // XXX COMPAT WITH 0.6.6. Send the old style welcome message, which // will force old clients to reload. Remove this once we're not - // concerned about people upgrading from a pre-0.6.7 release. Also, + // concerned about people upgrading from a pre-0.7.0 release. Also, // remove the clause in the client that ignores the welcome message // (livedata_connection.js) socket.send(JSON.stringify({server_id: "0"})); From 8bd400e56065b0f2d61e18dd220becb6a2bf7ac8 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Tue, 17 Dec 2013 10:49:35 -0800 Subject: [PATCH 079/112] lets do 0.7.0 instead --- History.md | 2 +- scripts/admin/banner.txt | 7 +++---- scripts/admin/notices.json | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/History.md b/History.md index da4ec32d9f..e4ff101642 100644 --- a/History.md +++ b/History.md @@ -1,4 +1,4 @@ -## v0.6.7 +## v0.7.0 This version of Meteor contains a patch for a bug in Node 0.10 which most commonly affects websockets. The patch is against Node version diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index 78e71c8c36..def13df77b 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,6 +1,5 @@ -=> Meteor 0.6.7: featuring a new live database driver that uses Mongo's - database replication log to watch queries more efficiently without - requerying the database on each update. +=> Meteor 0.7.0: use MongoDB's operations log to observe equality + queries instead of polling the database server on each update. This release is being downloaded in the background. Update your - project to Meteor 0.6.7 by running 'meteor update'. + project to Meteor 0.7.0 by running 'meteor update'. diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json index 6e7d3e323d..ab41d828c8 100644 --- a/scripts/admin/notices.json +++ b/scripts/admin/notices.json @@ -73,7 +73,7 @@ "release": "0.6.6.3" }, { - "release": "0.6.7" + "release": "0.7.0" }, { "release": "NEXT" From dbdab6a15657b15b8bd4d4af76d16c746c96051c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 17 Dec 2013 11:00:07 -0800 Subject: [PATCH 080/112] Formatting improvements to History --- History.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/History.md b/History.md index e4ff101642..98e61ba25f 100644 --- a/History.md +++ b/History.md @@ -31,7 +31,7 @@ apply the patch and will instead disable websockets. client code changes; server-only code changes will not cause the page to reload. -* New 'facts' package publishes internal statistics about Meteor. +* New `facts` package publishes internal statistics about Meteor. * Add an explicit check that publish functions return a cursor, an array of cursors, or a falsey value. This is a safety check to to prevent @@ -41,14 +41,14 @@ apply the patch and will instead disable websockets. * Implement `$each`, `$sort`, and `$slice` options for minimongo's `$push` modifier. #1492 -* Introduce '--raw-logs' option to `meteor run` to disable log +* Introduce `--raw-logs` option to `meteor run` to disable log coloring and timestamps. * Add `WebAppInternals.setBundledJsCssPrefix()` to control where the client loads bundled JavaScript and CSS files. This allows serving files from a CDN to decrease page load times and reduce server load. -* Attempt to exit cleanly on 'SIGHUP'. Stop accepting incoming +* Attempt to exit cleanly on `SIGHUP`. Stop accepting incoming connections, kill DDP connections, and finish all outstanding requests for static assets. From b6c7d424bc73456b64180d730272a9ed18562359 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 17 Dec 2013 11:08:19 -0800 Subject: [PATCH 081/112] Update examples and docs to 0.7.0 --- docs/.meteor/release | 2 +- examples/leaderboard/.meteor/release | 2 +- examples/parties/.meteor/release | 2 +- examples/todos/.meteor/release | 2 +- examples/wordplay/.meteor/release | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index bb2638e6b6..faef31a435 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.6.7-rc1 +0.7.0 diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release index bb2638e6b6..faef31a435 100644 --- a/examples/leaderboard/.meteor/release +++ b/examples/leaderboard/.meteor/release @@ -1 +1 @@ -0.6.7-rc1 +0.7.0 diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release index bb2638e6b6..faef31a435 100644 --- a/examples/parties/.meteor/release +++ b/examples/parties/.meteor/release @@ -1 +1 @@ -0.6.7-rc1 +0.7.0 diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release index bb2638e6b6..faef31a435 100644 --- a/examples/todos/.meteor/release +++ b/examples/todos/.meteor/release @@ -1 +1 @@ -0.6.7-rc1 +0.7.0 diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release index bb2638e6b6..faef31a435 100644 --- a/examples/wordplay/.meteor/release +++ b/examples/wordplay/.meteor/release @@ -1 +1 @@ -0.6.7-rc1 +0.7.0 From f0f7ae2d16260e671252a29f34736aa2c7f0bfc5 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 17 Dec 2013 10:49:27 -0800 Subject: [PATCH 082/112] Do not loop forever on ie8 --- packages/webapp/webapp_client.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/webapp/webapp_client.js b/packages/webapp/webapp_client.js index 15ae6311af..09a0eb44e2 100644 --- a/packages/webapp/webapp_client.js +++ b/packages/webapp/webapp_client.js @@ -2,6 +2,8 @@ WebApp = { _isCssLoaded: function () { return _.find(document.styleSheets, function (sheet) { + if (sheet.cssText && !sheet.cssRules) // IE8 + return sheet.cssText.match(/_meteor_detect_css/); return _.find(sheet.cssRules, function (rule) { return rule.selectorText === '._meteor_detect_css'; }); From fa25bcafb48fc168c516f916924a8bb37a84ee90 Mon Sep 17 00:00:00 2001 From: icellan Date: Tue, 17 Dec 2013 23:05:44 +0100 Subject: [PATCH 083/112] Removed extra comma at end of method. Fixes IE7 --- packages/livedata/livedata_common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index f0b6159ec6..1420da5797 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -45,7 +45,7 @@ _.extend(MethodInvocation.prototype, { throw new Error("Can't call setUserId in a method after calling unblock"); self.userId = userId; self._setUserId(userId); - }, + } }); parseDDP = function (stringMessage) { From ebb729f0f69867747aaa9db5dcbce84741a1999d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 17 Dec 2013 18:35:31 -0800 Subject: [PATCH 084/112] Fix crash with an empty programs/foo dir We should never use the existence of a directory in our source tree to make a decision, because git doesn't track directory existence, and it's easy to end up with extraneous directories (containing gitignored files, eg). So we should ignore programs/foo directories in apps if they don't contain package.js. --- tools/bundler.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/bundler.js b/tools/bundler.js index 5ea02d7110..e7acccdf43 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1675,6 +1675,16 @@ exports.bundle = function (appDir, outputPath, options) { // Recover by ignoring this program return; } + // Programs must (for now) contain a `package.js` file. If not, then + // perhaps the directory we are seeing is left over from another git + // branch or something and we should ignore it. We don't actually parse + // the package.js file here, though (but we do restart if it is later + // added or changed). + if (watch.readAndWatchFile( + watchSet, path.join(programsDir, item, 'package.js')) === null) { + return; + } + targets[item] = true; // will be overwritten with actual target later // Read attributes.json, if it exists From 0f4a21f89fcc1fd4b9391e0b8bd380c3923ad80c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 17 Dec 2013 18:55:28 -0800 Subject: [PATCH 085/112] meteor_npm: don't lose interesting resolved If an indirect dependency resolves to something other than a semver (or a GitHub tarball), it will be stored in the 'resolved'. Our shrinkwrap minifier (which helps to reduce spurious shrinkwrap file changes) needs to recognize that. Also, consistently use the "version" field in the minified shrinkwrap file (which a comment already claimed we could do). Fixes #1684. --- .../handlebars/.npm/package/npm-shrinkwrap.json | 2 +- .../js-analyze/.npm/package/npm-shrinkwrap.json | 2 +- .../.npm/package/npm-shrinkwrap.json | 2 +- tools/meteor_npm.js | 15 ++++++++++----- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/handlebars/.npm/package/npm-shrinkwrap.json b/packages/handlebars/.npm/package/npm-shrinkwrap.json index 6f6d7d42d0..563d2c159b 100644 --- a/packages/handlebars/.npm/package/npm-shrinkwrap.json +++ b/packages/handlebars/.npm/package/npm-shrinkwrap.json @@ -1,7 +1,7 @@ { "dependencies": { "handlebars": { - "from": "https://github.com/meteor/handlebars.js/tarball/543ec6689cf663cfda2d8f26c3c27de40aad7bd5", + "version": "https://github.com/meteor/handlebars.js/tarball/543ec6689cf663cfda2d8f26c3c27de40aad7bd5", "dependencies": { "optimist": { "version": "0.3.7", diff --git a/packages/js-analyze/.npm/package/npm-shrinkwrap.json b/packages/js-analyze/.npm/package/npm-shrinkwrap.json index 29842c42b1..a084b12ea5 100644 --- a/packages/js-analyze/.npm/package/npm-shrinkwrap.json +++ b/packages/js-analyze/.npm/package/npm-shrinkwrap.json @@ -1,7 +1,7 @@ { "dependencies": { "esprima": { - "from": "https://github.com/ariya/esprima/tarball/2a41dbf0ddadade0b09a9a7cc9a0c8df9c434018" + "version": "https://github.com/ariya/esprima/tarball/2a41dbf0ddadade0b09a9a7cc9a0c8df9c434018" }, "escope": { "version": "1.0.0", diff --git a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json index e6d581a1fd..6b4a386b67 100644 --- a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json +++ b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json @@ -1,7 +1,7 @@ { "dependencies": { "mongodb": { - "from": "https://github.com/meteor/node-mongodb-native/tarball/779bbac916a751f305d84c727a6cc7dfddab7924", + "version": "https://github.com/meteor/node-mongodb-native/tarball/779bbac916a751f305d84c727a6cc7dfddab7924", "dependencies": { "bson": { "version": "0.2.2" diff --git a/tools/meteor_npm.js b/tools/meteor_npm.js index b959b5ec4c..d9298976ab 100644 --- a/tools/meteor_npm.js +++ b/tools/meteor_npm.js @@ -543,11 +543,16 @@ _.extend(exports, { var topLevel = self._shrinkwrappedDependenciesTree(dir); var minimizeModule = function (module) { - var minimized = {}; - if (self._isGitHubTarball(module.from)) - minimized.from = module.from; - else - minimized.version = module.version; + var version; + if (module.resolved && + !module.resolved.match(/^https:\/\/registry.npmjs.org\//)) { + version = module.resolved; + } else if (self._isGitHubTarball(module.from)) { + version = module.from; + } else { + version = module.version; + } + var minimized = {version: version}; if (module.dependencies) { minimized.dependencies = {}; From 2b36c8be335c20f00054be74d4fae756dac74074 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Thu, 19 Dec 2013 13:32:19 -0800 Subject: [PATCH 086/112] Count new servers already started in ctl before starting lots more --- packages/ctl/ctl.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/packages/ctl/ctl.js b/packages/ctl/ctl.js index 5c4ce425a2..a0c844dc8b 100644 --- a/packages/ctl/ctl.js +++ b/packages/ctl/ctl.js @@ -139,6 +139,15 @@ Ctl.Commands.push({ }; var oldServers = jobs.find(oldJobSelector).fetch(); // Start a new job for each of them. + var newServersAlreadyPresent = jobs.find({ + app: Ctl.myAppName(), + star: thisJob.star, + program: "server", + done: false + }).count(); + // discount any new servers we've already started. + oldServers.splice(0, newServersAlreadyPresent); + console.log("starting " + oldServers.length + " new servers to match old"); _.each(oldServers, function (oldServer) { Ctl.startServerlikeProgram("server", oldServer.tags, From 39e3f1ee1af9d7de2a9cf6a779a53f80d6430c4c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 15:07:39 -0800 Subject: [PATCH 087/112] In "dev mode" email, be more explicit. Make it clear that nothing is being sent and point users to MAIL_URL. Fixes #1196. --- packages/email/email.js | 2 ++ packages/email/email_tests.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/email/email.js b/packages/email/email.js index 8fce809930..4ff6155be8 100644 --- a/packages/email/email.js +++ b/packages/email/email.js @@ -78,6 +78,8 @@ var devModeSend = function (mc) { // This approach does not prevent other writers to stdout from interleaving. stream.write("====== BEGIN MAIL #" + devmode_mail_id + " ======\n"); + stream.write("(Mail not sent; to enable sending, set the MAIL_URL " + + "environment variable.)\n"); mc.streamMessage(); mc.pipe(stream, {end: false}); var future = new Future; diff --git a/packages/email/email_tests.js b/packages/email/email_tests.js index 1c367b2178..dbdddafeea 100644 --- a/packages/email/email_tests.js +++ b/packages/email/email_tests.js @@ -20,6 +20,8 @@ Tinytest.add("email - dev mode smoke test", function (test) { // XXX brittle if mailcomposer changes header order, etc test.equal(stream.getContentsAsString("utf8"), "====== BEGIN MAIL #0 ======\n" + + "(Mail not sent; to enable sending, set the MAIL_URL " + + "environment variable.)\n" + "MIME-Version: 1.0\r\n" + "X-Meteor-Test: a custom header\r\n" + "From: foo@example.com\r\n" + From eeae8ad11ee0b6585262fec853c6db11464007c4 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 19 Dec 2013 15:14:16 -0800 Subject: [PATCH 088/112] Reword docs around file load order to be more correct. Fixes #1675. --- docs/client/concepts.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 57dec594cd..88564871a4 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -154,8 +154,7 @@ other packages. However sometimes load order dependencies in your application are unavoidable. The JavaScript and CSS files in an application are loaded according to these rules: -* Files in the `lib` directory at the root of your application are - loaded first. +* Files in directories named `lib` are loaded first. * Files that match `main.*` are loaded after everything else. From 0db5b9393a121dc1d142cd85f8e99c6350ed9740 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 19 Dec 2013 15:46:44 -0800 Subject: [PATCH 089/112] update readme to note build requirements. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f128f12057..1cd7553747 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,8 @@ can run Meteor directly from a git checkout. If you're the sort of person who likes to build everything from scratch, you can build all the Meteor dependencies (node.js, npm, mongodb, etc) -with the provided script. If you do not run this script, Meteor will +with the provided script. This requires git, a C and C++ compiler, +autotools, and scons. If you do not run this script, Meteor will automatically download pre-compiled binaries when you first run it. # OPTIONAL From a21f0d2e582a9cdbde61b67345dd36cd4b1527a7 Mon Sep 17 00:00:00 2001 From: Mitar Date: Sat, 7 Dec 2013 23:46:13 -0800 Subject: [PATCH 090/112] Pass "bare" option when using CoffeeScript as well. --- packages/coffeescript/plugin/compile-coffeescript.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/coffeescript/plugin/compile-coffeescript.js b/packages/coffeescript/plugin/compile-coffeescript.js index b0c94ce870..bb96c681be 100644 --- a/packages/coffeescript/plugin/compile-coffeescript.js +++ b/packages/coffeescript/plugin/compile-coffeescript.js @@ -149,7 +149,10 @@ var handler = function (compileStep, isLiterate) { path: outputFile, sourcePath: compileStep.inputPath, data: sourceWithMap.source, - sourceMap: sourceWithMap.sourceMap + sourceMap: sourceWithMap.sourceMap, + // XXX eventually get rid of backward-compatibility "raw" name + // XXX COMPAT WITH 0.6.4 + bare: compileStep.fileOptions.bare || compileStep.fileOptions.raw }); }; From 6844cb2615b0020c5bcb445bc50a1c04e9ae9460 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:00:38 -0800 Subject: [PATCH 091/112] Add test for #1668 --- packages/coffeescript/bare_test_setup.coffee | 4 ++++ packages/coffeescript/bare_tests.js | 3 +++ packages/coffeescript/package.js | 2 ++ 3 files changed, 9 insertions(+) create mode 100644 packages/coffeescript/bare_test_setup.coffee create mode 100644 packages/coffeescript/bare_tests.js diff --git a/packages/coffeescript/bare_test_setup.coffee b/packages/coffeescript/bare_test_setup.coffee new file mode 100644 index 0000000000..4e6a84eb02 --- /dev/null +++ b/packages/coffeescript/bare_test_setup.coffee @@ -0,0 +1,4 @@ +# Normally, variables should be file-local, but this file is loaded with {bare: +# true}, so it should be readable by bare_tests.js + +VariableSetByCoffeeBareTestSetup = 5678 diff --git a/packages/coffeescript/bare_tests.js b/packages/coffeescript/bare_tests.js new file mode 100644 index 0000000000..9a5e21f4fa --- /dev/null +++ b/packages/coffeescript/bare_tests.js @@ -0,0 +1,3 @@ +Tinytest.add("coffeescript - bare", function (test) { + test.equal(VariableSetByCoffeeBareTestSetup, 5678); +}); diff --git a/packages/coffeescript/package.js b/packages/coffeescript/package.js index e31e47b9db..2091a9daae 100644 --- a/packages/coffeescript/package.js +++ b/packages/coffeescript/package.js @@ -14,6 +14,8 @@ Package._transitional_registerBuildPlugin({ Package.on_test(function (api) { api.use(['coffeescript', 'tinytest']); api.use(['coffeescript-test-helper'], ['client', 'server']); + api.add_files('bare_test_setup.coffee', ['client'], {bare: true}); + api.add_files('bare_tests.js', ['client']); api.add_files([ 'coffeescript_test_setup.js', 'tests/coffeescript_tests.coffee', From f65b1cd2ae88fde77be6ec64d363971d38acf0c0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:00:58 -0800 Subject: [PATCH 092/112] Remove backwards-compatibility code There's no need to be backwards-compatible here with something that has never worked (or at least hasn't worked since 0.6.5). --- packages/coffeescript/plugin/compile-coffeescript.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/coffeescript/plugin/compile-coffeescript.js b/packages/coffeescript/plugin/compile-coffeescript.js index bb96c681be..dd712b634f 100644 --- a/packages/coffeescript/plugin/compile-coffeescript.js +++ b/packages/coffeescript/plugin/compile-coffeescript.js @@ -150,9 +150,7 @@ var handler = function (compileStep, isLiterate) { sourcePath: compileStep.inputPath, data: sourceWithMap.source, sourceMap: sourceWithMap.sourceMap, - // XXX eventually get rid of backward-compatibility "raw" name - // XXX COMPAT WITH 0.6.4 - bare: compileStep.fileOptions.bare || compileStep.fileOptions.raw + bare: compileStep.fileOptions.bare }); }; From 3f12f77a3f6a0bc1309ea30124a36567ddd42bde Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:13:28 -0800 Subject: [PATCH 093/112] Test that localStorage works before uses it Fixes accounts-password login with private-browsing Safari, and probably some IE configurations too. Fixes #1291 (I hope, not tested on IE). Similar to #1688. This still means that multi-tab won't work with private browsing, unfortunately. --- packages/localstorage/localstorage.js | 47 +++++++++++++++++++-------- packages/localstorage/package.js | 1 + 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/localstorage/localstorage.js b/packages/localstorage/localstorage.js index 4dd1e49ba2..ccaefcca08 100644 --- a/packages/localstorage/localstorage.js +++ b/packages/localstorage/localstorage.js @@ -1,20 +1,37 @@ -// This is not an ideal name, but we can change it later. +// Meteor._localStorage is not an ideal name, but we can change it later. if (window.localStorage) { - Meteor._localStorage = { - getItem: function (key) { - return window.localStorage.getItem(key); - }, - setItem: function (key, value) { - window.localStorage.setItem(key, value); - }, - removeItem: function (key) { - window.localStorage.removeItem(key); - } - }; + // Let's test to make sure that localStorage actually works. For example, in + // Safari with private browsing on, window.localStorage exists but actually + // trying to use it throws. + + var key = '_localstorage_test_' + Random.id(); + var retrieved; + try { + window.localStorage.setItem(key, key); + retrieved = window.localStorage.getItem(key); + window.localStorage.removeItem(key); + } catch (e) { + // ... ignore + } + if (key === retrieved) { + Meteor._localStorage = { + getItem: function (key) { + return window.localStorage.getItem(key); + }, + setItem: function (key, value) { + window.localStorage.setItem(key, value); + }, + removeItem: function (key) { + window.localStorage.removeItem(key); + } + }; + } } + // XXX eliminate dependency on jQuery, detect browsers ourselves -else if ($.browser.msie) { // If we are on IE, which support userData +// Else, if we are on IE, which support userData +if (!Meteor._localStorage && $.browser.msie) { var userdata = document.createElement('span'); // could be anything userdata.style.behavior = 'url("#default#userData")'; userdata.id = 'localstorage-helper'; @@ -40,7 +57,9 @@ else if ($.browser.msie) { // If we are on IE, which support userData return userdata.getAttribute(key); } }; -} else { +} + +if (!Meteor._localStorage) { Meteor._debug( "You are running a browser with no localStorage or userData " + "support. Logging in from one tab will not cause another " diff --git a/packages/localstorage/package.js b/packages/localstorage/package.js index c6570323cb..434872dc84 100644 --- a/packages/localstorage/package.js +++ b/packages/localstorage/package.js @@ -5,6 +5,7 @@ Package.describe({ Package.on_use(function (api) { api.use('jquery', 'client'); // XXX only used for browser detection. remove. + api.use('random', 'client'); api.add_files('localstorage.js', 'client'); }); From 7424bb63cd741fb6a75cbb0885a394bc5194fad5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:25:02 -0800 Subject: [PATCH 094/112] mongo_runner: Don't send rs.initiate too early Fixes #1696. Thanks to @Maxpain177 for reporting and providing access to a machine where this was easily reproducible. --- tools/mongo_runner.js | 67 ++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 40d0a39bf8..09a3efbacf 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -234,6 +234,8 @@ exports.launchMongo = function (options) { proc.stdout.setEncoding('utf8'); var listening = false; var replSetReady = false; + var replSetReadyToBeInitiated = false; + var alreadyInitiatedReplSet = false; var maybeCallOnListen = function () { if (listening && replSetReady) { if (createReplSet) @@ -241,29 +243,54 @@ exports.launchMongo = function (options) { onListen(); } }; + + var maybeInitiateReplset = function () { + // We need to want to create a replset, be confident that the server is + // listening, be confident that the server's replset implementation is + // ready to be initiated, and have not already done it. + if (!(createReplSet && listening && replSetReadyToBeInitiated + && !alreadyInitiatedReplSet)) { + return; + } + + alreadyInitiatedReplSet = true; + + // Connect to it and start a replset. + var db = new mongoNpmModule.Db( + 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), + {safe: true}); + db.open(function(err, db) { + if (err) + throw err; + db.admin().command({ + replSetInitiate: { + _id: replSetName, + members: [{_id : 0, host: '127.0.0.1:' + options.port}] + } + }, function (err, result) { + if (err) + throw err; + // why this isn't in the error is unclear. + if (result && result.documents && result.documents[0] + && result.documents[0].errmsg) { + throw result.document[0].errmsg; + } + db.close(true); + }); + }); + }; + proc.stdout.on('data', function (data) { + // note: don't use "else ifs" in this, because 'data' can have multiple + // lines + if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) { + replSetReadyToBeInitiated = true; + maybeInitiateReplset(); + } + if (/ \[initandlisten\] waiting for connections on port/.test(data)) { - if (createReplSet) { - // Connect to it and start a replset. - var db = new mongoNpmModule.Db( - 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), - {safe: true}); - db.open(function(err, db) { - if (err) - throw err; - db.admin().command({ - replSetInitiate: { - _id: replSetName, - members: [{_id : 0, host: '127.0.0.1:' + options.port}] - } - }, function (err, result) { - if (err) - throw err; - db.close(true); - }); - }); - } listening = true; + maybeInitiateReplset(); maybeCallOnListen(); } From 4294d40220d506a55a4fc404ffa1be732b33064f Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Thu, 19 Dec 2013 18:14:22 -0800 Subject: [PATCH 095/112] update docs app to release sso-1 --- docs/.meteor/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index faef31a435..400877328a 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.7.0 +sso-1 From c0626667ec9439f571347ac2287ae70dccf8d474 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:19:15 -0800 Subject: [PATCH 096/112] Don't leave invalid METEOR-PORT files around Could cause mongo startup to hang. Reproduction: $ meteor => Meteor server running on: http://localhost:3000/ # ... wait for server to start, ctrl-c. # leaves '3002' in .meteor/local/db/METEOR-PORT $ meteor -p 5000 # ctrl-c in about a second: once we've wiped the old local db # but before we've configured the new one. # before this commit, '3002' is still in the METEOR-PORT file. $ meteor # before this commit, hangs with: Initializing mongo database... this may take a moment. --- tools/mongo_runner.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 09a3efbacf..b7d0824c99 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -161,9 +161,11 @@ exports.launchMongo = function (options) { } var portFile = path.join(dbPath, 'METEOR-PORT'); + var portFileExists = false; var createReplSet = true; try { createReplSet = +(fs.readFileSync(portFile)) !== options.port; + portFileExists = true; } catch (e) { if (!e || e.code !== 'ENOENT') throw e; @@ -176,6 +178,11 @@ exports.launchMongo = function (options) { // replSet configuration. It's also a little slow to initiate a new replSet, // thus the attempt to not do it unless the port changes.) if (createReplSet) { + // Delete the port file, so we don't mistakenly believe that the DB is + // still configured. + if (portFileExists) + fs.unlinkSync(portFile); + try { var dbFiles = fs.readdirSync(dbPath); } catch (e) { From d84fe821196185e87fa0f87d3d5d73078d705c70 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:54:27 -0800 Subject: [PATCH 097/112] Add some comments to selector compiler --- packages/minimongo/selector.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e077b9363c..44ad5d5a62 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -4,12 +4,25 @@ var isArray = function (x) { return _.isArray(x) && !EJSON.isBinary(x); }; +// If x is an array, true if f(e) is true for some e in x +// (but never try f(x) directly) +// Otherwise, true if f(x) is true. +// +// Use this in cases where f(Array) should never be true... +// for example, equality comparisons to non-arrays, +// ordering comparisons (which should always be false if either side +// is an array), regexps (need string), mod (needs number)... +// XXX ensure comparisons are always false if LHS is an array +// XXX ensure comparisons among different types are false var _anyIfArray = function (x, f) { if (isArray(x)) return _.any(x, f); return f(x); }; +// True if f(x) is true, or x is an array and f(e) is true for some e in x. +// +// Use this for most operators where an array could satisfy the predicate. var _anyIfArrayPlus = function (x, f) { if (f(x)) return true; From 515e6f28158ded457891df3441420d5ab480da5c Mon Sep 17 00:00:00 2001 From: icellan Date: Tue, 17 Dec 2013 23:05:44 +0100 Subject: [PATCH 098/112] Removed extra comma at end of method. Fixes IE7 --- packages/livedata/livedata_common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index f0b6159ec6..1420da5797 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -45,7 +45,7 @@ _.extend(MethodInvocation.prototype, { throw new Error("Can't call setUserId in a method after calling unblock"); self.userId = userId; self._setUserId(userId); - }, + } }); parseDDP = function (stringMessage) { From 995d7282786c899765f90f227f9cc01f0e485f3a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:25:02 -0800 Subject: [PATCH 099/112] mongo_runner: Don't send rs.initiate too early Fixes #1696. Thanks to @Maxpain177 for reporting and providing access to a machine where this was easily reproducible. --- tools/mongo_runner.js | 67 ++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 40d0a39bf8..09a3efbacf 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -234,6 +234,8 @@ exports.launchMongo = function (options) { proc.stdout.setEncoding('utf8'); var listening = false; var replSetReady = false; + var replSetReadyToBeInitiated = false; + var alreadyInitiatedReplSet = false; var maybeCallOnListen = function () { if (listening && replSetReady) { if (createReplSet) @@ -241,29 +243,54 @@ exports.launchMongo = function (options) { onListen(); } }; + + var maybeInitiateReplset = function () { + // We need to want to create a replset, be confident that the server is + // listening, be confident that the server's replset implementation is + // ready to be initiated, and have not already done it. + if (!(createReplSet && listening && replSetReadyToBeInitiated + && !alreadyInitiatedReplSet)) { + return; + } + + alreadyInitiatedReplSet = true; + + // Connect to it and start a replset. + var db = new mongoNpmModule.Db( + 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), + {safe: true}); + db.open(function(err, db) { + if (err) + throw err; + db.admin().command({ + replSetInitiate: { + _id: replSetName, + members: [{_id : 0, host: '127.0.0.1:' + options.port}] + } + }, function (err, result) { + if (err) + throw err; + // why this isn't in the error is unclear. + if (result && result.documents && result.documents[0] + && result.documents[0].errmsg) { + throw result.document[0].errmsg; + } + db.close(true); + }); + }); + }; + proc.stdout.on('data', function (data) { + // note: don't use "else ifs" in this, because 'data' can have multiple + // lines + if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) { + replSetReadyToBeInitiated = true; + maybeInitiateReplset(); + } + if (/ \[initandlisten\] waiting for connections on port/.test(data)) { - if (createReplSet) { - // Connect to it and start a replset. - var db = new mongoNpmModule.Db( - 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), - {safe: true}); - db.open(function(err, db) { - if (err) - throw err; - db.admin().command({ - replSetInitiate: { - _id: replSetName, - members: [{_id : 0, host: '127.0.0.1:' + options.port}] - } - }, function (err, result) { - if (err) - throw err; - db.close(true); - }); - }); - } listening = true; + maybeInitiateReplset(); maybeCallOnListen(); } From 96c907654418b2b9fb12e26575fc4621ba8c5751 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:19:15 -0800 Subject: [PATCH 100/112] Don't leave invalid METEOR-PORT files around Could cause mongo startup to hang. Reproduction: $ meteor => Meteor server running on: http://localhost:3000/ # ... wait for server to start, ctrl-c. # leaves '3002' in .meteor/local/db/METEOR-PORT $ meteor -p 5000 # ctrl-c in about a second: once we've wiped the old local db # but before we've configured the new one. # before this commit, '3002' is still in the METEOR-PORT file. $ meteor # before this commit, hangs with: Initializing mongo database... this may take a moment. --- tools/mongo_runner.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 09a3efbacf..b7d0824c99 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -161,9 +161,11 @@ exports.launchMongo = function (options) { } var portFile = path.join(dbPath, 'METEOR-PORT'); + var portFileExists = false; var createReplSet = true; try { createReplSet = +(fs.readFileSync(portFile)) !== options.port; + portFileExists = true; } catch (e) { if (!e || e.code !== 'ENOENT') throw e; @@ -176,6 +178,11 @@ exports.launchMongo = function (options) { // replSet configuration. It's also a little slow to initiate a new replSet, // thus the attempt to not do it unless the port changes.) if (createReplSet) { + // Delete the port file, so we don't mistakenly believe that the DB is + // still configured. + if (portFileExists) + fs.unlinkSync(portFile); + try { var dbFiles = fs.readdirSync(dbPath); } catch (e) { From 508ad66513cc09ff1b44cf3323d1150c3e62321f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:35:29 -0800 Subject: [PATCH 101/112] Support websockets on Node 0.10.24 too --- docs/client/concepts.html | 8 ++++---- packages/meteor/node-issue-6506-workaround.js | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 57dec594cd..d1e4d30170 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -826,10 +826,10 @@ To get started, run This command will generate a fully-contained Node.js application in the form of a tarball. To run this application, you need to provide Node.js 0.10 and a MongoDB server. (The current release of Meteor has been tested with Node -0.10.21.) You can then run the application by invoking node, specifying the HTTP -port for the application to listen on, and the MongoDB endpoint. If you don't -already have a MongoDB server, we can recommend our friends at -[MongoHQ](http://mongohq.com). +0.10.22, and is recommended for use with 0.10.22 through 0.10.24 only.) You can +then run the application by invoking node, specifying the HTTP port for the +application to listen on, and the MongoDB endpoint. If you don't already have a +MongoDB server, we can recommend our friends at [MongoHQ](http://mongohq.com). $ PORT=3000 MONGO_URL=mongodb://localhost:27017/myapp node bundle/main.js diff --git a/packages/meteor/node-issue-6506-workaround.js b/packages/meteor/node-issue-6506-workaround.js index a08b92a9d8..bd41b7c384 100644 --- a/packages/meteor/node-issue-6506-workaround.js +++ b/packages/meteor/node-issue-6506-workaround.js @@ -1,13 +1,16 @@ // Temporary workaround for https://github.com/joyent/node/issues/6506 -// Our fix involves replicating a bunch of files in order to -// -if (process.version !== 'v0.10.22' && process.version !== 'v0.10.23') { +// Our fix involves replicating a bunch of functions in order to change +// a single line. + +var PATCH_VERSIONS = ['v0.10.22', 'v0.10.23', 'v0.10.24']; + +if (!_.contains(PATCH_VERSIONS, process.version)) { if (!process.env.DISABLE_WEBSOCKETS) { console.error("This version of Meteor contains a patch for a bug in Node v0.10."); - console.error("The patch is against only versions 0.10.22 and 0.10.23."); + console.error("The patch is against only versions 0.10.22 through 0.10.24."); console.error("You are using version " + process.version + " instead, so we cannot apply the patch."); console.error("To mitigate the most common effect of the bug, websockets will be disabled."); - console.error("To enable websockets, use Node v0.10.22 or .23, or upgrade to a later version of Meteor (if available)."); + console.error("To enable websockets, use Node v0.10.22 through v0.10.24, or upgrade to a later version of Meteor (if available)."); process.env.DISABLE_WEBSOCKETS = 't'; } } else { From c5eefd9504ef9bb415d8f7324e988753b9af5b00 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:45:27 -0800 Subject: [PATCH 102/112] History update for 0.7.0.1 --- History.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/History.md b/History.md index 98e61ba25f..43f557d9cb 100644 --- a/History.md +++ b/History.md @@ -1,3 +1,13 @@ +## v0.7.0.1 + +* Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the + message "Initializing mongo database... this may take a moment.". + +* Apply the Node patch to 0.10.24 as well (see the 0.7.0 section for details). + +* Fix gratuitous IE7 incompatibility. + + ## v0.7.0 This version of Meteor contains a patch for a bug in Node 0.10 which From d98b7ed423731874b255ec1c2df682a995181bbb Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:47:13 -0800 Subject: [PATCH 103/112] Don't call onListen more than once eg, maybe the replset loses and regains its PRIMARY. --- tools/mongo_runner.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index b7d0824c99..d726ad7a16 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -243,10 +243,12 @@ exports.launchMongo = function (options) { var replSetReady = false; var replSetReadyToBeInitiated = false; var alreadyInitiatedReplSet = false; + var alreadyCalledOnListen = false; var maybeCallOnListen = function () { - if (listening && replSetReady) { + if (listening && replSetReady && !alreadyCalledOnListen) { if (createReplSet) fs.writeFileSync(portFile, options.port); + alreadyCalledOnListen = true; onListen(); } }; From 51ad916cafdc78212731345e25d85b61dc66f22d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:53:08 -0800 Subject: [PATCH 104/112] Banner and notice update for 0.7.0.1. --- scripts/admin/banner.txt | 5 ++--- scripts/admin/notices.json | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index def13df77b..1abd8fb7da 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,5 +1,4 @@ -=> Meteor 0.7.0: use MongoDB's operations log to observe equality - queries instead of polling the database server on each update. +=> Meteor 0.7.0.1: Fix hang while setting up local MongoDB. This release is being downloaded in the background. Update your - project to Meteor 0.7.0 by running 'meteor update'. + project to Meteor 0.7.0.1 by running 'meteor update'. diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json index ab41d828c8..7a9375f70a 100644 --- a/scripts/admin/notices.json +++ b/scripts/admin/notices.json @@ -75,6 +75,9 @@ { "release": "0.7.0" }, + { + "release": "0.7.0.1" + }, { "release": "NEXT" } From 42069551405ed7fe9759a8c4a293751f904a75db Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:58:30 -0800 Subject: [PATCH 105/112] Better banner text --- scripts/admin/banner.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index 1abd8fb7da..a52ccd54d2 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,4 +1,4 @@ -=> Meteor 0.7.0.1: Fix hang while setting up local MongoDB. +=> Meteor 0.7.0.1: Fix failure to initialize local MongoDB server. This release is being downloaded in the background. Update your project to Meteor 0.7.0.1 by running 'meteor update'. From 5cae2c426a3b599dcd492e2d4514ef22c16b8d84 Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Thu, 19 Dec 2013 18:14:22 -0800 Subject: [PATCH 106/112] update docs app to release sso-1 --- docs/.meteor/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index faef31a435..400877328a 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.7.0 +sso-1 From 6fc8332c99872b09d8ef416e0b80436bfe3ef662 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:23:11 -0800 Subject: [PATCH 107/112] Update docs and examples to 0.7.0.1. --- docs/lib/release-override.js | 2 +- examples/leaderboard/.meteor/release | 2 +- examples/parties/.meteor/release | 2 +- examples/todos/.meteor/release | 2 +- examples/wordplay/.meteor/release | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js index 0f61b680aa..60276835d0 100644 --- a/docs/lib/release-override.js +++ b/docs/lib/release-override.js @@ -1,5 +1,5 @@ // While galaxy apps are on their own special meteor releases, override // Meteor.release here. if (Meteor.isClient) { - Meteor.release = Meteor.release ? "0.7.0" : undefined; + Meteor.release = Meteor.release ? "0.7.0.1" : undefined; } diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/leaderboard/.meteor/release +++ b/examples/leaderboard/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/parties/.meteor/release +++ b/examples/parties/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/todos/.meteor/release +++ b/examples/todos/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/wordplay/.meteor/release +++ b/examples/wordplay/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 From caa528733d469d53604abf76a14f0f98368f98cf Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:33:09 -0800 Subject: [PATCH 108/112] Add issue numbers to History --- History.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/History.md b/History.md index 43f557d9cb..d09f53d286 100644 --- a/History.md +++ b/History.md @@ -1,11 +1,11 @@ ## v0.7.0.1 * Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the - message "Initializing mongo database... this may take a moment.". + message "Initializing mongo database... this may take a moment.". #1696 * Apply the Node patch to 0.10.24 as well (see the 0.7.0 section for details). -* Fix gratuitous IE7 incompatibility. +* Fix gratuitous IE7 incompatibility. #1690 ## v0.7.0 From 7341966f49da15302968678446bbbda8181e2db7 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:34:06 -0800 Subject: [PATCH 109/112] Fix incorrect Node version number Fixes #1701 --- tools/bundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bundler.js b/tools/bundler.js index e7acccdf43..9d93ddd75f 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1474,7 +1474,8 @@ var writeSiteArchive = function (targets, outputPath, options) { builder.write('README', { data: new Buffer( "This is a Meteor application bundle. It has only one dependency:\n" + "Node.js 0.10 (with the 'fibers' package). The current release of Meteor\n" + -"has been tested with Node 0.10.21. To run the application:\n" + +"has been tested with Node 0.10.22 and works best with 0.10.22 through\n" + +"0.10.24. To run the application:\n" + "\n" + " $ rm -r programs/server/node_modules/fibers\n" + " $ npm install fibers@1.0.1\n" + From c37d009d96feef10d975c80ee643944f003e42d3 Mon Sep 17 00:00:00 2001 From: Andrew Wilcox Date: Mon, 16 Dec 2013 10:51:51 -0500 Subject: [PATCH 110/112] Extract retry package from livedata --- packages/autoupdate/autoupdate_client.js | 4 +--- packages/autoupdate/package.js | 2 +- packages/livedata/livedata_common.js | 5 ----- packages/livedata/package.js | 4 ++-- packages/retry/.gitignore | 1 + packages/retry/package.js | 10 ++++++++++ packages/{livedata => retry}/retry.js | 23 +++++++++++------------ 7 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 packages/retry/.gitignore create mode 100644 packages/retry/package.js rename packages/{livedata => retry}/retry.js (70%) diff --git a/packages/autoupdate/autoupdate_client.js b/packages/autoupdate/autoupdate_client.js index dfd4063ddd..45179f8ba4 100644 --- a/packages/autoupdate/autoupdate_client.js +++ b/packages/autoupdate/autoupdate_client.js @@ -44,9 +44,7 @@ Autoupdate.newClientAvailable = function () { -// XXX Livedata exporting this via DDP is a hack. See -// packages/livedata/livedata_common.js -var retry = new DDP._Retry({ +var retry = new Retry({ // Unlike the stream reconnect use of Retry, which we want to be instant // in normal operation, this is a wacky failure. We don't want to retry // right away, we can start slowly. diff --git a/packages/autoupdate/package.js b/packages/autoupdate/package.js index f56a572b1a..fded7a3869 100644 --- a/packages/autoupdate/package.js +++ b/packages/autoupdate/package.js @@ -5,7 +5,7 @@ Package.describe({ Package.on_use(function (api) { api.use('webapp', 'server'); - api.use('deps', 'client'); + api.use(['deps', 'retry'], 'client'); api.use(['livedata', 'mongo-livedata'], ['client', 'server']); api.use('deps', 'client'); api.use('reload', 'client', {weak: true}); diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index 1420da5797..f9a873c6f7 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -115,8 +115,3 @@ stringifyDDP = function (msg) { // state in the DDP session. Meteor.setTimeout and friends clear // it. We can probably find a better way to factor this. DDP._CurrentInvocation = new Meteor.EnvironmentVariable; - - -// This is private and a hack. It is used by autoupdate_client. We -// should refactor. Maybe a separate 'exponential-backoff' package? -DDP._Retry = Retry; diff --git a/packages/livedata/package.js b/packages/livedata/package.js index 6c7778f808..d2b407ae13 100644 --- a/packages/livedata/package.js +++ b/packages/livedata/package.js @@ -6,7 +6,8 @@ Package.describe({ Npm.depends({sockjs: "0.3.8", websocket: "1.0.8"}); Package.on_use(function (api) { - api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps', 'logging'], + api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps', + 'logging', 'retry'], ['client', 'server']); // It is OK to use this package on a server architecture without making a @@ -33,7 +34,6 @@ Package.on_use(function (api) { // Transport api.use('reload', 'client', {weak: true}); api.add_files('common.js'); - api.add_files('retry.js', ['client', 'server']); api.add_files(['sockjs-0.3.4.js', 'stream_client_sockjs.js'], 'client'); api.add_files('stream_client_nodejs.js', 'server'); api.add_files('stream_client_common.js', ['client', 'server']); diff --git a/packages/retry/.gitignore b/packages/retry/.gitignore new file mode 100644 index 0000000000..677a6fc263 --- /dev/null +++ b/packages/retry/.gitignore @@ -0,0 +1 @@ +.build* diff --git a/packages/retry/package.js b/packages/retry/package.js new file mode 100644 index 0000000000..9c21873dcf --- /dev/null +++ b/packages/retry/package.js @@ -0,0 +1,10 @@ +Package.describe({ + summary: "Retry logic with exponential backoff", + internal: true +}); + +Package.on_use(function (api) { + api.use('underscore', ['client', 'server']); + api.export('Retry'); + api.add_files('retry.js', ['client', 'server']); +}); diff --git a/packages/livedata/retry.js b/packages/retry/retry.js similarity index 70% rename from packages/livedata/retry.js rename to packages/retry/retry.js index d5fdda4d66..a4407b5bdb 100644 --- a/packages/livedata/retry.js +++ b/packages/retry/retry.js @@ -1,24 +1,23 @@ // Retry logic with an exponential backoff. +// +// options: +// baseTimeout: time for initial reconnect attempt (ms). +// exponent: exponential factor to increase timeout each attempt. +// maxTimeout: maximum time between retries (ms). +// minCount: how many times to reconnect "instantly". +// minTimeout: time to wait for the first `minCount` retries (ms). +// fuzz: factor to randomize retry times by (to avoid retry storms). Retry = function (options) { var self = this; _.extend(self, _.defaults(_.clone(options || {}), { - // time for initial reconnect attempt. - baseTimeout: 1000, - // exponential factor to increase timeout each attempt. + baseTimeout: 1000, // 1 second exponent: 2.2, - // maximum time between reconnects. keep this intentionally - // high-ish to ensure a server can recover from a failure caused - // by load + // The default is high-ish to ensure a server can recover from a + // failure caused by load. maxTimeout: 5 * 60000, // 5 minutes - // time to wait for the first 2 retries. this helps page reload - // speed during dev mode restarts, but doesn't hurt prod too - // much (due to CONNECT_TIMEOUT) minTimeout: 10, - // how many times to try to reconnect 'instantly' minCount: 2, - // fuzz factor to randomize reconnect times by. avoid reconnect - // storms. fuzz: 0.5 // +- 25% })); self.retryTimer = null; From 178734b66d7ce9d059cbbe8bdb2788ab541a8632 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 23 Dec 2013 15:17:36 -0800 Subject: [PATCH 111/112] Fix test broken by 0f4a21f (thanks @awwx). --- tools/tests/test_bundler_npm.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/tests/test_bundler_npm.js b/tools/tests/test_bundler_npm.js index e4e1509bc0..188522184e 100644 --- a/tools/tests/test_bundler_npm.js +++ b/tools/tests/test_bundler_npm.js @@ -58,7 +58,7 @@ var _assertCorrectPackageNpmDir = function(deps) { // copy fields with values generated by shrinkwrap that can't be known to the // test author. We set keys on val always in this order so that comparison works well. var val = {}; - _.each(['version', 'from', 'resolved', 'dependencies'], function(key) { + _.each(['version', 'dependencies'], function(key) { if (expected[key]) val[key] = expected[key]; else if (actualMeteorNpmShrinkwrapDependencies[name] && actualMeteorNpmShrinkwrapDependencies[name][key]) @@ -264,7 +264,13 @@ assert.doesNotThrow(function () { var tmpOutputDir = tmpDir(); var result = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip', releaseStamp: 'none', library: lib}); assert.strictEqual(result.errors, false, result.errors && result.errors[0]); + try { _assertCorrectPackageNpmDir(deps); + } catch (e) { + console.log("ACTUAL", e.actual) + console.log("EXPECTED", e.expected) + throw e + } _assertCorrectBundleNpmContents(tmpOutputDir, deps); // Check that a string introduced by our fork is in the source. assert(/clientMaxAge = 604800000/.test( From c0e169dd6217ac11aac80f7cd5e0132d47ded47d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 23 Dec 2013 17:16:27 -0800 Subject: [PATCH 112/112] Place link to _observeDriver on multiplexer Instead of on *only the first* ObserveHandle --- packages/mongo-livedata/mongo_driver.js | 5 ++--- .../mongo-livedata/mongo_livedata_tests.js | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 9d8063fa56..baaa2eeab5 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -999,9 +999,8 @@ MongoConnection.prototype._observeChanges = function ( _testOnlyPollCallback: callbacks._testOnlyPollCallback }); - // This field is only set for the first ObserveHandle in an - // ObserveMultiplexer. It is only there for use tests. - observeHandle._observeDriver = observeDriver; + // This field is only set for use in tests. + multiplexer._observeDriver = observeDriver; } // Blocks until the initial adds have been sent. diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 1f163b4e4b..dbb4a65d36 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -391,13 +391,9 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } }); - // XXX What if there are multiple observe handles on the ObserveMultiplexer? - // There shouldn't be because the collection has a name unique to this - // run. if (Meteor.isServer) { - // For now, has to be polling (not oplog). - test.isTrue(obs._observeDriver); - test.isTrue(obs._observeDriver._suspendPolling); + // For now, has to be polling (not oplog) because it is ordered observe. + test.isTrue(obs._multiplexer._observeDriver._suspendPolling); } var step = 0; @@ -432,7 +428,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, finishObserve(function () { if (Meteor.isServer) - obs._observeDriver._suspendPolling(); + obs._multiplexer._observeDriver._suspendPolling(); // Do a batch of 1-10 operations var batch_count = rnd(10) + 1; @@ -465,7 +461,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } } if (Meteor.isServer) - obs._observeDriver._resumePolling(); + obs._multiplexer._observeDriver._resumePolling(); }); @@ -1886,14 +1882,12 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - _disableOplog", functi if (MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle) { var observeWithOplog = coll.find({x: 5}) .observeChanges({added: function () {}}); - test.isTrue(observeWithOplog._observeDriver); - test.isTrue(observeWithOplog._observeDriver._usesOplog); + test.isTrue(observeWithOplog._multiplexer._observeDriver._usesOplog); observeWithOplog.stop(); } var observeWithoutOplog = coll.find({x: 6}, {_disableOplog: true}) .observeChanges({added: function () {}}); - test.isTrue(observeWithoutOplog._observeDriver); - test.isFalse(observeWithoutOplog._observeDriver._usesOplog); + test.isFalse(observeWithoutOplog._multiplexer._observeDriver._usesOplog); observeWithoutOplog.stop(); });