From 859ff555fe440b2e3dec914e9ac2f99f57322168 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 5 Feb 2014 14:33:11 -0800
Subject: [PATCH 001/123] Limits in oplog code

- distinguish queries with limits and sorts
- separate the concept of matching set from published set
- keep a separate buffer for N documents outside of limit to reduce mongo
  lookups
---
 packages/minimongo/id_map.js                  |   2 +-
 packages/mongo-livedata/dummy-structure.js    |  44 ++++
 .../mongo-livedata/oplog_observe_driver.js    | 231 +++++++++++++++---
 packages/mongo-livedata/package.js            |   2 +
 4 files changed, 239 insertions(+), 40 deletions(-)
 create mode 100644 packages/mongo-livedata/dummy-structure.js

diff --git a/packages/minimongo/id_map.js b/packages/minimongo/id_map.js
index 5c2b628155..4e94273fac 100644
--- a/packages/minimongo/id_map.js
+++ b/packages/minimongo/id_map.js
@@ -68,7 +68,7 @@ _.extend(LocalCollection._IdMap.prototype, {
     var clone = new LocalCollection._IdMap;
     self.forEach(function (value, id) {
       clone.set(id, EJSON.clone(value));
-      });
+    });
     return clone;
   }
 });
diff --git a/packages/mongo-livedata/dummy-structure.js b/packages/mongo-livedata/dummy-structure.js
new file mode 100644
index 0000000000..ce008e56bd
--- /dev/null
+++ b/packages/mongo-livedata/dummy-structure.js
@@ -0,0 +1,44 @@
+// Implements the interface of IdMap and knows how to find Min or Max element
+DummyStructure = function (comparator) {
+  var self = this;
+  self.comparator = comparator;
+  self.idMap = new LocalCollection.IdMap;
+};
+
+_.each(['get', 'set', 'remove', 'has', 'empty', 'clear', 'forEach', 'size', 'setDefault'], function (method) {
+  DummyStructure.prototype[method] = function (/* arguments */) {
+    var self = this;
+    return self.idMap[method].apply(self, arguments);
+  };
+});
+
+DummyStructure.prototype.clone = function () {
+  var self = this;
+  var clone = new DummyStructure;
+  clone.comparator = self.comparator;
+  clone.idMap = self.idMap.clone();
+  return clone;
+};
+
+DummyStructure.prototype.minElementId = function () {
+  var self = this;
+  var minElementId = null;
+  self.idMap.forEach(function (value, key) {
+    if (minElement === null)
+      minElementId = key;
+    else if (self.comparator(value, self.idMap.get(minElementId)) < 0)
+      minElementId = key;
+  });
+
+  return minElementId;
+};
+
+DummyStructure.prototype.maxElementId = function () {
+  var self = this;
+  var comparator = self.comparator;
+  self.comparator = function (a, b) { return -comparator(a, b); };
+  var maxElementId = self.minElementId();
+  self.comparator = comparator;
+  return maxElementId;
+};
+
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 2cfa2a7775..cc79cf9e24 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -19,8 +19,21 @@ OplogObserveDriver = function (options) {
   self._cursorDescription = options.cursorDescription;
   self._mongoHandle = options.mongoHandle;
   self._multiplexer = options.multiplexer;
-  if (options.ordered)
-    throw Error("OplogObserveDriver only supports unordered observeChanges");
+
+  if (options.ordered) {
+    // XXX replace with doubly-heaps and shit once we get these working
+    var comparator = self._cursorDescription.sorter.getComparator();
+    self._limit = self._cursorDescription.limit;
+    self._comparator = comparator;
+    self._unpublishedBuffer = new DummyStructure(comparator);
+    // We need something that can find Max value in addition to IdMap interface
+    self._published = new DummyStructure(comparator);
+  } else {
+    self._limit = 0;
+    self._comparator = null;
+    self._unpublishedBuffer = null;
+    self._published = new LocalCollection._IdMap;
+  }
 
   self._stopped = false;
   self._stopHandles = [];
@@ -30,7 +43,6 @@ OplogObserveDriver = function (options) {
 
   self._registerPhaseChange(PHASE.QUERYING);
 
-  self._published = new LocalCollection._IdMap;
   var selector = self._cursorDescription.selector;
   self._matcher = options.matcher;
   var projection = self._cursorDescription.options.fields || {};
@@ -107,22 +119,111 @@ OplogObserveDriver = function (options) {
 };
 
 _.extend(OplogObserveDriver.prototype, {
-  _add: function (doc) {
+  _addPublished: function (id, doc) {
+    var self = this;
+    self._published.set(id, self._sharedProjectionFn(doc));
+    self._multiplexer.added(id, self._projectionFn(doc));
+
+    // After adding this document, the published set might be overflowed
+    // (exceeding capacity specified by limit). If so, push the maximum element
+    // to the buffer, we might want to save it in memory to reduce the amount of
+    // Mongo lookups in the future.
+    if (self._limit && self._published.size() > self._limit) {
+      // XXX in theory the size of published is no more than limit+1
+      if (self._published.size() !== self._limit + 1) {
+        // xcxc better error message
+        throw new Error("After adding to published, " +
+                        (self._limit - self._published.size()) +
+                        " documents are overflowing the set");
+      }
+
+      var overflowingDocId = self._published.getMaximumId();
+      var overflowingDoc = self._published.get(overflowingDocId);
+      self._published.remove(overflowingDocId);
+      self._multiplexer.removed(overflowingDocId);
+      self._addBuffered(overflowingDocId, overflowingDoc);
+    }
+  },
+  _removePublished: function (id) {
+    var self = this;
+    self._published.remove(id);
+    self._multiplexer.removed(id);
+    if (!self._unpublishedBuffer)
+      return;
+    // xcxc size on heaps should be cached to O(1)
+    if (self._published.size() < self._cursorDescription.limit) {
+      // The unpublished buffer is empty iff published contains the whole
+      // matching set, i.e. number of matching documents is less or equal to the
+      // queries limit.
+      if (!self._unpublishedBuffer.size())
+        return;
+
+      var newDocId = self._unpublishedBuffer.minElementId();
+      var newDoc = self._unpublishedBuffer.get(newDocId);
+      self._removeBuffered(newDocId);
+      self._addPublished(newDocId, newDoc);
+    }
+  },
+  _changePublished: function (id, oldDoc, newDoc) {
+    var self = this;
+    self._published.set(id, self._sharedProjectionFn(newDoc));
+    var changed = LocalCollection._makeChangedFields(_.clone(newDoc), oldDoc);
+    changed = self._projectionFn(changed);
+    if (!_.isEmpty(changed))
+      self._multiplexer.changed(id, changed);
+  },
+  _addBuffered: function (id, doc) {
+    var self = this;
+    self._unpublishedBuffer.add(id, self._sharedProjectionFn(fields));
+    if (self._unpublishedBuffer.size() > self._limit)
+      self._unpublishedBuffer.remove(self._unpublishedBuffer.getMaximumId());
+  },
+  _removeBuffered: function (id) {
+    var self = this;
+    self._unpublishedBuffer.remove(id);
+    if (! self._unpublishedBuffer.size())
+      self._needToPollQuery();
+  },
+  // Called when a document has joined the "Matching" results set.
+  // Takes responsibility of keeping _unpublishedBuffer in sync with _published
+  // and the effect of limit enforced.
+  _addMatching: function (doc) {
     var self = this;
     var id = doc._id;
     var fields = _.clone(doc);
     delete fields._id;
     if (self._published.has(id))
       throw Error("tried to add something already published " + id);
-    self._published.set(id, self._sharedProjectionFn(fields));
-    self._multiplexer.added(id, self._projectionFn(fields));
+    if (self._unpublishedBuffer && self._unpublishedBuffer.has(id))
+      throw Error("tried to add something already existed in buffer " + id); // xcxc error msg
+
+    var limit = self._limit;
+    var comparator = self._comparator;
+    var maxPublished = (limit && self._published.size() > 0) ? self._published.get(self._published.getMaximumId()) : null;
+    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ? self._unpublishedBuffer.get(self._unpublishedBuffer.getMaximumId()) : null;
+    // The query is unlimited or didn't publish enough documents yet or the new
+    // document would fit into published set pushing the maximum element out,
+    // then we need to publish the doc.
+    // Otherwise we might need to buffer it (only in case of limited query).
+    if (!limit || self._published.size() < limit || comparator(maxPublished, fields) === 1) {
+      self._addPublished(id, fields);
+    } else if (self._unpublishedBuffer.size() < limit || comparator(maxBuffered, fields) === 1) {
+      self._addBuffered(id, fields);
+    }
   },
-  _remove: function (id) {
+  // Called when a document leaves the "Matching" results set.
+  // Takes responsibility of keeping _unpublishedBuffer in sync with _published
+  // and the effect of limit enforced.
+  _removeMatching: function (id) {
     var self = this;
-    if (!self._published.has(id))
-      throw Error("tried to remove something unpublished " + id);
-    self._published.remove(id);
-    self._multiplexer.removed(id);
+    if (!self._published.has(id) && !self._unpublishedBuffer)
+      throw Error("tried to remove something unpublished " + id); // xcxc fix this error msg
+
+    if (self._published.has(id)) {
+      self._removePublished(id);
+    } else if (self._unpublishedBuffer.has(id)) {
+      self._removeBuffered(id);
+    }
   },
   _handleDoc: function (id, newDoc, mustMatchNow) {
     var self = this;
@@ -134,22 +235,40 @@ _.extend(OplogObserveDriver.prototype, {
                   + EJSON.stringify(self._cursorDescription));
     }
 
-    var matchedBefore = self._published.has(id);
+    var publishedBefore = self._published.has(id);
+    var bufferedBefore = self._unpublishedBuffer && self._unpublishedBuffer.has(id);
+    var cachedBefore = publishedBefore || bufferedBefore;
 
-    if (matchesNow && !matchedBefore) {
-      self._add(newDoc);
-    } else if (matchedBefore && !matchesNow) {
-      self._remove(id);
+    if (matchesNow && !cachedBefore) {
+      self._addMatching(newDoc);
+    } else if (cachedBefore && !matchesNow) {
+      self._removeMatching(id);
     } else if (matchesNow) {
-      var oldDoc = self._published.get(id);
-      if (!oldDoc)
-        throw Error("thought that " + id + " was there!");
       delete newDoc._id;
-      self._published.set(id, self._sharedProjectionFn(newDoc));
-      var changed = LocalCollection._makeChangedFields(_.clone(newDoc), oldDoc);
-      changed = self._projectionFn(changed);
-      if (!_.isEmpty(changed))
-        self._multiplexer.changed(id, changed);
+      var oldDoc = self._published.get(id);
+      var comparator = self._comparator;
+      var minBuffered = self._unpublishedBuffer && self._unpublishedBuffer.size() &&
+        self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId());
+
+      if (publishedBefore) {
+        // Unordered case where the document stays in published once it matches
+        // or the case when we don't have enough matching docs to publish or the
+        // changed but matching doc will stay in published anyways.
+        if (!self._limit || self._unpublishedBuffer.size() === 0 || comparator(newDoc, minBuffered) < 1) {
+          self._changePublished(id, oldDoc, newDoc);
+        } else {
+          // after the change doc doesn't stay in the published, remove it
+          self._removePublished(id);
+        }
+      } else if (bufferedBefore) {
+        // after the change we can't know if doc is still in the buffer limit
+        // w/o querying mongo, so just remove it from buffer
+        self._removeBuffered(id);
+        // but it can move into published now, check it
+        var maxPublished = self._published.get(self._published.getMaximumId());
+        if (comparator(newDoc, maxPublished) === -1)
+          self._addPublished(id, newDoc);
+      }
     }
   },
   _fetchModifiedDocuments: function () {
@@ -233,16 +352,17 @@ _.extend(OplogObserveDriver.prototype, {
     }
 
     if (op.op === 'd') {
-      if (self._published.has(id))
-        self._remove(id);
+      if (self._published.has(id) || (self._limit && self._unpublishedBuffer.has(id)))
+        self._removeMatching(id);
     } else if (op.op === 'i') {
+      // xcxc what if buffer has it?
       if (self._published.has(id))
         throw new Error("insert found for already-existing ID");
 
       // XXX what if selector yields?  for now it can't but later it could have
       // $where
       if (self._matcher.documentMatches(op.o).result)
-        self._add(op.o);
+        self._addMatching(op.o);
     } else if (op.op === 'u') {
       // Is this a modifier ($set/$unset, which may require us to poll the
       // database to figure out if the whole document matches the selector) or a
@@ -258,10 +378,13 @@ _.extend(OplogObserveDriver.prototype, {
 
       if (isReplace) {
         self._handleDoc(id, _.extend({_id: id}, op.o));
-      } else if (self._published.has(id) && canDirectlyModifyDoc) {
+      } else if ((self._published.has(id) || self._unpublishedBuffer.has(id)) && canDirectlyModifyDoc) {
         // Oh great, we actually know what the document is, so we can apply
         // this directly.
-        var newDoc = EJSON.clone(self._published.get(id));
+        if (self._published.has(id))
+          var newDoc = EJSON.clone(self._published.get(id));
+        else
+          var newDoc = EJSON.clone(self._unpublishedBuffer.get(id));
         newDoc._id = id;
         LocalCollection._modify(newDoc, op.o);
         self._handleDoc(id, self._sharedProjectionFn(newDoc));
@@ -280,9 +403,16 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._stopped)
       throw new Error("oplog stopped surprisingly early");
 
-    var initialCursor = self._cursorForQuery();
+    // Query 2x documents as the half excluded from the original query will go
+    // into unpublished buffer to reduce additional Mongo lookups in cases when
+    // documents are removed from the published set and need a replacement.
+    // XXX needs more thought on non-zero skip
+    // XXX "2" here is a "magic number"
+    var initialCursor = self._cursorForQuery({ limit: self._limit * 2 });
     initialCursor.forEach(function (initialDoc) {
-      self._add(initialDoc);
+      // self._addMatching knows how to correctly separate documents into the
+      // published set and the buffer.
+      self._addMatching(initialDoc);
     });
     if (self._stopped)
       throw new Error("oplog stopped quite early");
@@ -324,12 +454,18 @@ _.extend(OplogObserveDriver.prototype, {
       // subtle note: _published does not contain _id fields, but newResults
       // does
       var newResults = new LocalCollection._IdMap;
-      var cursor = self._cursorForQuery();
-      cursor.forEach(function (doc) {
-        newResults.set(doc._id, doc);
+      var newBuffer = new LocalCollection._IdMap;
+      // XXX 2 is a "magic number" meaning there is an extra chunk of docs for
+      // buffer if such is needed.
+      var cursor = self._cursorForQuery({ limit: self._limit * 2 });
+      cursor.forEach(function (doc, i) {
+        if (!self._limit || i < self._limit)
+          newResults.set(doc._id, doc);
+        else
+          newBuffer.set(doc._id, doc);
       });
 
-      self._publishNewResults(newResults);
+      self._publishNewResults(newResults, newBuffer);
 
       self._doneQuerying();
     });
@@ -379,7 +515,7 @@ _.extend(OplogObserveDriver.prototype, {
     }
   },
 
-  _cursorForQuery: function () {
+  _cursorForQuery: function (optionsOverwrite) {
     var self = this;
 
     // The query we run is almost the same as the cursor we are observing, with
@@ -388,6 +524,11 @@ _.extend(OplogObserveDriver.prototype, {
     // "shared" projection). And we don't want to apply any transform in the
     // cursor, because observeChanges shouldn't use the transform.
     var options = _.clone(self._cursorDescription.options);
+
+    // Allow the caller to modify the options. Useful to specify different skip
+    // and limit values.
+    _.extend(options, optionsOverwrite);
+
     options.fields = self._sharedProjection;
     delete options.transform;
     // We are NOT deep cloning fields or selector here, which should be OK.
@@ -401,13 +542,19 @@ _.extend(OplogObserveDriver.prototype, {
 
   // Replace self._published with newResults (both are IdMaps), invoking observe
   // callbacks on the multiplexer.
+  // Replace self._unpublishedBuffer with newBuffer.
   //
   // XXX This is very similar to LocalCollection._diffQueryUnorderedChanges. We
   // should really: (a) Unify IdMap and OrderedDict into Unordered/OrderedDict (b)
   // Rewrite diff.js to use these classes instead of arrays and objects.
-  _publishNewResults: function (newResults) {
+  _publishNewResults: function (newResults, newBuffer) {
     var self = this;
 
+    // If there is a buffer, shut down so it doesn't stay in a way
+    if (self._unpublishedBuffer) {
+      self._unpublishedBuffer.clear();
+    }
+
     // First remove anything that's gone. Be careful not to modify
     // self._published while iterating over it.
     var idsToRemove = [];
@@ -416,7 +563,7 @@ _.extend(OplogObserveDriver.prototype, {
         idsToRemove.push(id);
     });
     _.each(idsToRemove, function (id) {
-      self._remove(id);
+      self._removePublished(id);
     });
 
     // Now do adds and changes.
@@ -425,6 +572,11 @@ _.extend(OplogObserveDriver.prototype, {
       // selector.
       self._handleDoc(id, doc, true);
     });
+
+    // Finally, replace the buffer
+    newBuffer.forEach(function (doc, id) {
+      self._addBuffered(id, doc);
+    });
   },
 
   // This stop function is invoked from the onStop of the ObserveMultiplexer, so
@@ -451,6 +603,7 @@ _.extend(OplogObserveDriver.prototype, {
 
     // Proactively drop references to potentially big things.
     self._published = null;
+    self._unpublishedBuffer = null;
     self._needToFetch = null;
     self._currentlyFetching = null;
     self._oplogEntryHandle = null;
@@ -489,7 +642,7 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   // This option (which are mostly used for sorted cursors) require us to figure
   // out where a given document fits in an order to know if it's included or
   // not, and we don't track that information when doing oplog tailing.
-  if (options.limit || options.skip) return false;
+  if (options.limit && (options.skip || !options.sorter)) return false;
 
   // If a fields projection option is given check if it is supported by
   // minimongo (some operators are not supported).
diff --git a/packages/mongo-livedata/package.js b/packages/mongo-livedata/package.js
index f0b1f9e3d2..898258486a 100644
--- a/packages/mongo-livedata/package.js
+++ b/packages/mongo-livedata/package.js
@@ -47,6 +47,8 @@ Package.on_use(function (api) {
   // For tests only.
   api.export('MongoTest', 'server', {testOnly: true});
 
+  // xcxc temporary
+  api.add_files('dummy-structure.js', 'server');
   api.add_files(['mongo_driver.js', 'oplog_tailing.js',
                  'observe_multiplex.js', 'doc_fetcher.js',
                  'polling_observe_driver.js','oplog_observe_driver.js'],

From ad0c88f07183eaef7a1b3d5ada6e1fffbd6edc38 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 11 Feb 2014 16:45:47 -0800
Subject: [PATCH 002/123] Always use limit to separate ordered code vs
 unordered

---
 .../mongo-livedata/oplog_observe_driver.js    | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index cc79cf9e24..6b684222d9 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -21,6 +21,12 @@ OplogObserveDriver = function (options) {
   self._multiplexer = options.multiplexer;
 
   if (options.ordered) {
+    // There are several properties ordered driver implements:
+    // - _limit is a positive number
+    // - _comparator is a function-comparator by which the query is ordered
+    // - _unpublishedBuffer is non-null collection
+    // - _published implements maxElementId method in addition to IdMap methods
+
     // XXX replace with doubly-heaps and shit once we get these working
     var comparator = self._cursorDescription.sorter.getComparator();
     self._limit = self._cursorDescription.limit;
@@ -148,14 +154,14 @@ _.extend(OplogObserveDriver.prototype, {
     var self = this;
     self._published.remove(id);
     self._multiplexer.removed(id);
-    if (!self._unpublishedBuffer)
+    if (! self._limit)
       return;
     // xcxc size on heaps should be cached to O(1)
     if (self._published.size() < self._cursorDescription.limit) {
       // The unpublished buffer is empty iff published contains the whole
       // matching set, i.e. number of matching documents is less or equal to the
       // queries limit.
-      if (!self._unpublishedBuffer.size())
+      if (! self._unpublishedBuffer.size())
         return;
 
       var newDocId = self._unpublishedBuffer.minElementId();
@@ -194,7 +200,7 @@ _.extend(OplogObserveDriver.prototype, {
     delete fields._id;
     if (self._published.has(id))
       throw Error("tried to add something already published " + id);
-    if (self._unpublishedBuffer && self._unpublishedBuffer.has(id))
+    if (self._limit && self._unpublishedBuffer.has(id))
       throw Error("tried to add something already existed in buffer " + id); // xcxc error msg
 
     var limit = self._limit;
@@ -216,7 +222,7 @@ _.extend(OplogObserveDriver.prototype, {
   // and the effect of limit enforced.
   _removeMatching: function (id) {
     var self = this;
-    if (!self._published.has(id) && !self._unpublishedBuffer)
+    if (!self._published.has(id) && !self._limit)
       throw Error("tried to remove something unpublished " + id); // xcxc fix this error msg
 
     if (self._published.has(id)) {
@@ -236,7 +242,7 @@ _.extend(OplogObserveDriver.prototype, {
     }
 
     var publishedBefore = self._published.has(id);
-    var bufferedBefore = self._unpublishedBuffer && self._unpublishedBuffer.has(id);
+    var bufferedBefore = self._limit && self._unpublishedBuffer.has(id);
     var cachedBefore = publishedBefore || bufferedBefore;
 
     if (matchesNow && !cachedBefore) {
@@ -247,7 +253,7 @@ _.extend(OplogObserveDriver.prototype, {
       delete newDoc._id;
       var oldDoc = self._published.get(id);
       var comparator = self._comparator;
-      var minBuffered = self._unpublishedBuffer && self._unpublishedBuffer.size() &&
+      var minBuffered = self._limit && self._unpublishedBuffer.size() &&
         self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId());
 
       if (publishedBefore) {
@@ -550,8 +556,9 @@ _.extend(OplogObserveDriver.prototype, {
   _publishNewResults: function (newResults, newBuffer) {
     var self = this;
 
-    // If there is a buffer, shut down so it doesn't stay in a way
-    if (self._unpublishedBuffer) {
+    // If the query is ordered and there is a buffer, shut down so it doesn't
+    // stay in a way.
+    if (self._limit) {
       self._unpublishedBuffer.clear();
     }
 

From b8429fa0b18cf68573441d97c64a7fbb8cd1b139 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 11 Feb 2014 21:57:49 -0800
Subject: [PATCH 003/123] Export sorter in Minimongo namespace

---
 packages/minimongo/sort.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js
index 718ec1537e..60c3dd60b2 100644
--- a/packages/minimongo/sort.js
+++ b/packages/minimongo/sort.js
@@ -110,6 +110,7 @@ Sorter.prototype.getComparator = function (options) {
   }]);
 };
 
+Minimongo.Sorter = Sorter;
 MinimongoTest.Sorter = Sorter;
 
 // Given an array of comparators

From 818a770ed93d02c8a930b8fd4aaf6cb93df20888 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 11 Feb 2014 21:58:09 -0800
Subject: [PATCH 004/123] Fix Dummy Structure

actually Dummy-structure was supposed to be so dummy, it would be trivially easy
to write a bug-free 20-line code for it. But I failed. To dummy structure...
---
 packages/mongo-livedata/dummy-structure.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/mongo-livedata/dummy-structure.js b/packages/mongo-livedata/dummy-structure.js
index ce008e56bd..e0622a6d80 100644
--- a/packages/mongo-livedata/dummy-structure.js
+++ b/packages/mongo-livedata/dummy-structure.js
@@ -2,13 +2,13 @@
 DummyStructure = function (comparator) {
   var self = this;
   self.comparator = comparator;
-  self.idMap = new LocalCollection.IdMap;
+  self.idMap = new LocalCollection._IdMap;
 };
 
 _.each(['get', 'set', 'remove', 'has', 'empty', 'clear', 'forEach', 'size', 'setDefault'], function (method) {
   DummyStructure.prototype[method] = function (/* arguments */) {
     var self = this;
-    return self.idMap[method].apply(self, arguments);
+    return self.idMap[method].apply(self.idMap, arguments);
   };
 });
 

From c389dadc460728c5618fea75b2bca1415f2ae472 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 11 Feb 2014 21:59:13 -0800
Subject: [PATCH 005/123] Correctly distinguish queries with limits and sorts

---
 packages/mongo-livedata/oplog_observe_driver.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 6b684222d9..d0679fff58 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -20,7 +20,7 @@ OplogObserveDriver = function (options) {
   self._mongoHandle = options.mongoHandle;
   self._multiplexer = options.multiplexer;
 
-  if (options.ordered) {
+  if (options.cursorDescription.options.limit) {
     // There are several properties ordered driver implements:
     // - _limit is a positive number
     // - _comparator is a function-comparator by which the query is ordered
@@ -28,7 +28,8 @@ OplogObserveDriver = function (options) {
     // - _published implements maxElementId method in addition to IdMap methods
 
     // XXX replace with doubly-heaps and shit once we get these working
-    var comparator = self._cursorDescription.sorter.getComparator();
+    var sorter = new Minimongo.Sorter(options.cursorDescription.options.sort);
+    var comparator = sorter.getComparator();
     self._limit = self._cursorDescription.limit;
     self._comparator = comparator;
     self._unpublishedBuffer = new DummyStructure(comparator);
@@ -649,7 +650,7 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   // This option (which are mostly used for sorted cursors) require us to figure
   // out where a given document fits in an order to know if it's included or
   // not, and we don't track that information when doing oplog tailing.
-  if (options.limit && (options.skip || !options.sorter)) return false;
+  if (options.limit && (options.skip || !options.sort)) return false;
 
   // If a fields projection option is given check if it is supported by
   // minimongo (some operators are not supported).

From 7493890bb2855affbd2d7e71e4dd6a490380035e Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 11 Feb 2014 23:27:07 -0800
Subject: [PATCH 006/123] Fix typos

---
 packages/mongo-livedata/dummy-structure.js      |  2 +-
 packages/mongo-livedata/oplog_observe_driver.js | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/packages/mongo-livedata/dummy-structure.js b/packages/mongo-livedata/dummy-structure.js
index e0622a6d80..0b0708c210 100644
--- a/packages/mongo-livedata/dummy-structure.js
+++ b/packages/mongo-livedata/dummy-structure.js
@@ -24,7 +24,7 @@ DummyStructure.prototype.minElementId = function () {
   var self = this;
   var minElementId = null;
   self.idMap.forEach(function (value, key) {
-    if (minElement === null)
+    if (minElementId === null)
       minElementId = key;
     else if (self.comparator(value, self.idMap.get(minElementId)) < 0)
       minElementId = key;
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index d0679fff58..7a1db5ae59 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -30,7 +30,7 @@ OplogObserveDriver = function (options) {
     // XXX replace with doubly-heaps and shit once we get these working
     var sorter = new Minimongo.Sorter(options.cursorDescription.options.sort);
     var comparator = sorter.getComparator();
-    self._limit = self._cursorDescription.limit;
+    self._limit = self._cursorDescription.options.limit;
     self._comparator = comparator;
     self._unpublishedBuffer = new DummyStructure(comparator);
     // We need something that can find Max value in addition to IdMap interface
@@ -144,7 +144,7 @@ _.extend(OplogObserveDriver.prototype, {
                         " documents are overflowing the set");
       }
 
-      var overflowingDocId = self._published.getMaximumId();
+      var overflowingDocId = self._published.maxElementId();
       var overflowingDoc = self._published.get(overflowingDocId);
       self._published.remove(overflowingDocId);
       self._multiplexer.removed(overflowingDocId);
@@ -181,9 +181,9 @@ _.extend(OplogObserveDriver.prototype, {
   },
   _addBuffered: function (id, doc) {
     var self = this;
-    self._unpublishedBuffer.add(id, self._sharedProjectionFn(fields));
+    self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc));
     if (self._unpublishedBuffer.size() > self._limit)
-      self._unpublishedBuffer.remove(self._unpublishedBuffer.getMaximumId());
+      self._unpublishedBuffer.remove(self._unpublishedBuffer.maxElementId());
   },
   _removeBuffered: function (id) {
     var self = this;
@@ -206,8 +206,8 @@ _.extend(OplogObserveDriver.prototype, {
 
     var limit = self._limit;
     var comparator = self._comparator;
-    var maxPublished = (limit && self._published.size() > 0) ? self._published.get(self._published.getMaximumId()) : null;
-    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ? self._unpublishedBuffer.get(self._unpublishedBuffer.getMaximumId()) : null;
+    var maxPublished = (limit && self._published.size() > 0) ? self._published.get(self._published.maxElementId()) : null;
+    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ? self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) : null;
     // The query is unlimited or didn't publish enough documents yet or the new
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
@@ -272,7 +272,7 @@ _.extend(OplogObserveDriver.prototype, {
         // w/o querying mongo, so just remove it from buffer
         self._removeBuffered(id);
         // but it can move into published now, check it
-        var maxPublished = self._published.get(self._published.getMaximumId());
+        var maxPublished = self._published.get(self._published.maxElementId());
         if (comparator(newDoc, maxPublished) === -1)
           self._addPublished(id, newDoc);
       }

From 4e095325cd38c074efc5a1ca1fb6d4e5720bdc70 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 12 Feb 2014 00:13:20 -0800
Subject: [PATCH 007/123] Typo + always use inequalities with C-style
 comparators

---
 packages/mongo-livedata/oplog_observe_driver.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 7a1db5ae59..2ee1ea5f1b 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -158,7 +158,7 @@ _.extend(OplogObserveDriver.prototype, {
     if (! self._limit)
       return;
     // xcxc size on heaps should be cached to O(1)
-    if (self._published.size() < self._cursorDescription.limit) {
+    if (self._published.size() < self._limit) {
       // The unpublished buffer is empty iff published contains the whole
       // matching set, i.e. number of matching documents is less or equal to the
       // queries limit.
@@ -212,9 +212,9 @@ _.extend(OplogObserveDriver.prototype, {
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
     // Otherwise we might need to buffer it (only in case of limited query).
-    if (!limit || self._published.size() < limit || comparator(maxPublished, fields) === 1) {
+    if (!limit || self._published.size() < limit || comparator(maxPublished, fields) > 0) {
       self._addPublished(id, fields);
-    } else if (self._unpublishedBuffer.size() < limit || comparator(maxBuffered, fields) === 1) {
+    } else if (self._unpublishedBuffer.size() < limit || comparator(maxBuffered, fields) > 0) {
       self._addBuffered(id, fields);
     }
   },
@@ -273,7 +273,7 @@ _.extend(OplogObserveDriver.prototype, {
         self._removeBuffered(id);
         // but it can move into published now, check it
         var maxPublished = self._published.get(self._published.maxElementId());
-        if (comparator(newDoc, maxPublished) === -1)
+        if (comparator(newDoc, maxPublished) < 0)
           self._addPublished(id, newDoc);
       }
     }

From 690f7c485bce0e1686e3fb5bff71c0caa0d24be1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 12 Feb 2014 00:14:16 -0800
Subject: [PATCH 008/123] A simple test for limits

and it does pass!
---
 .../mongo-livedata/mongo_livedata_tests.js    | 77 +++++++++++++++++++
 1 file changed, 77 insertions(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 032e0dbaa7..c6100a41d5 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -745,6 +745,83 @@ if (Meteor.isServer) {
     });
     x++;
   });
+
+  Tinytest.addAsync("mongo-livedata - observe sorted, limited " + idGeneration, function (test, onComplete) {
+    var run = test.runId();
+    var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
+
+    var observer = function () {
+      var output = [];
+      var callbacks = {
+        changed: function (newDoc) {
+          output.push({changed: newDoc._id});
+        },
+        added: function (newDoc) {
+          output.push({added: newDoc._id});
+        },
+        removed: function (oldDoc) {
+          output.push({removed: oldDoc._id});
+        }
+      };
+      var handle = coll.find({foo: 22},
+                             {sort: {bar: 1}, limit: 3}).observe(callbacks);
+
+      return {output: output, handle: handle};
+    };
+
+    var ins = function (doc) {
+      var id;
+      runInFence(function () {
+        id = coll.insert(doc); });
+      return id;
+    };
+
+    var rem = function (sel) { runInFence(function () { coll.remove(sel); }); };
+
+    // Insert a doc and start observing.
+    var docId1 = ins({foo: 22, bar: 5});
+    var o = observer();
+    // Initial add.
+    test.length(o.output, 1);
+    test.equal(o.output.shift(), {added: docId1});
+
+    // Insert another doc (blocking until observes have fired).
+    var docId2 = ins({foo: 22, bar: 6});
+    // Observed add.
+    test.length(o.output, 1);
+    test.equal(o.output.shift(), {added: docId2});
+
+    var docId3 = ins({ foo: 22, bar: 3 });
+    test.length(o.output, 1);
+    test.equal(o.output.shift(), {added: docId3});
+
+    // Add a non-matching document
+    ins({ foo: 13 });
+    // It shouldn't be added
+    test.length(o.output, 0);
+
+    // Add something that matches but is too big to fit in
+    var docId4 = ins({ foo: 22, bar: 7 });
+    // It shouldn't be added
+    test.length(o.output, 0);
+
+    // Let's add something small enough to fit in
+    var docId5 = ins({ foo: 22, bar: -1 });
+    // We should get an added and a removed events
+    test.length(o.output, 2);
+    test.equal(o.output.shift(), {added: docId5});
+    // doc 2 was removed from the published set as it is too big to be in
+    test.equal(o.output.shift(), {removed: docId2});
+
+    // Now remove something and that doc 2 should be right back
+    rem(docId5);
+    test.length(o.output, 2);
+    test.equal(o.output.shift(), {removed: docId5});
+    test.equal(o.output.shift(), {added: docId2});
+
+    o.handle.stop();
+    onComplete();
+  });
 }
 
 

From cfadc51b91c9682d904a363b56dc314dc5282a4e Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 12 Feb 2014 01:44:51 -0800
Subject: [PATCH 009/123] More tests on updates. Clarifying comment

---
 .../mongo-livedata/mongo_livedata_tests.js    | 55 +++++++++++++++++--
 .../mongo-livedata/oplog_observe_driver.js    |  4 ++
 2 files changed, 55 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index c6100a41d5..12618963fa 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -770,13 +770,21 @@ if (Meteor.isServer) {
     };
 
     var ins = function (doc) {
-      var id;
-      runInFence(function () {
-        id = coll.insert(doc); });
+      var id; runInFence(function () { id = coll.insert(doc); });
       return id;
     };
-
     var rem = function (sel) { runInFence(function () { coll.remove(sel); }); };
+    var upd = function (sel, mod, opt) {
+      runInFence(function () {
+        coll.update(sel, mod, opt);
+      });
+    };
+    // compares arrays a and b w/o looking at order
+    var setsEqual = function (a, b) {
+      a = _.map(a, EJSON.stringify);
+      b = _.map(b, EJSON.stringify);
+      return _.isEmpty(_.difference(a, b));
+    };
 
     // Insert a doc and start observing.
     var docId1 = ins({foo: 22, bar: 5});
@@ -819,6 +827,45 @@ if (Meteor.isServer) {
     test.equal(o.output.shift(), {removed: docId5});
     test.equal(o.output.shift(), {added: docId2});
 
+    // Current state is [3 5 6] 7]
+    // Add some negative numbers overflowing the buffer.
+    // New documents will take the published place, [3 5 6] will take the buffer
+    // and 7 will be outside of the buffer in MongoDB.
+    var docId6 = ins({ foo: 22, bar: -1 });
+    var docId7 = ins({ foo: 22, bar: -2 });
+    var docId8 = ins({ foo: 22, bar: -3 });
+    test.length(o.output, 6);
+    var expected = [{added: docId6}, {removed: docId2},
+                    {added: docId7}, {removed: docId1},
+                    {added: docId8}, {removed: docId3}];
+
+    test.equal(o.output, expected);
+    o.output.splice(0, 6);
+
+    // Now the state is [-3 -2 -1] 3 5 6] 7
+    // If we update first 3 docs (increment them by 20), it would be
+    // interesting.
+    upd({ bar: { $lt: 0 }}, { $inc: { bar: 20 } }, { multi: true });
+
+    // The updated documents can't find their place in published and they can't
+    // be buffered as we are not aware of the situation outside of the buffer.
+    // But since our buffer becomes empty, it will be refilled partially with
+    // updated documents.
+    test.length(o.output, 6);
+    expected = [{removed: docId6}, {added: docId4},
+                {removed: docId7}, {added: docId1},
+                {removed: docId8}, {added: docId2}];
+
+    // Note: since we are updating multiple things, the order of updates may
+    // differ from launch to launch. That's why we compare even positions
+    // (removes) w/o looking at ordering.
+    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]],
+                          [expected[0], expected[2], expected[4]]));
+    test.equal([o.output[1], o.output[3], o.output[5]],
+               [expected[1], expected[3], expected[5]]);
+    // The new arrangment is [3 5 6] 7 17 18] 19
+
+
     o.handle.stop();
     onComplete();
   });
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 2ee1ea5f1b..0febd194e3 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -261,6 +261,10 @@ _.extend(OplogObserveDriver.prototype, {
         // Unordered case where the document stays in published once it matches
         // or the case when we don't have enough matching docs to publish or the
         // changed but matching doc will stay in published anyways.
+        // XXX: We rely on the emptiness of buffer. Be sure to maintain the fact
+        // that buffer can't be empty if there are matching documents not
+        // published. Notably, we don't want to schedule repoll and continue
+        // relying on this property.
         if (!self._limit || self._unpublishedBuffer.size() === 0 || comparator(newDoc, minBuffered) < 1) {
           self._changePublished(id, oldDoc, newDoc);
         } else {

From 8be6928b4c74ea29986caa7762157769ebb9b2fb Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 12 Feb 2014 17:34:44 -0800
Subject: [PATCH 010/123] wip test

---
 packages/mongo-livedata/mongo_livedata_tests.js | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 12618963fa..14ba50208c 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -852,7 +852,7 @@ if (Meteor.isServer) {
     // But since our buffer becomes empty, it will be refilled partially with
     // updated documents.
     test.length(o.output, 6);
-    expected = [{removed: docId6}, {added: docId4},
+    expected = [{removed: docId6}, {added: docId3},
                 {removed: docId7}, {added: docId1},
                 {removed: docId8}, {added: docId2}];
 
@@ -863,8 +863,21 @@ if (Meteor.isServer) {
                           [expected[0], expected[2], expected[4]]));
     test.equal([o.output[1], o.output[3], o.output[5]],
                [expected[1], expected[3], expected[5]]);
-    // The new arrangment is [3 5 6] 7 17 18] 19
+    o.output.splice(0, 6);
 
+    // The new arrangment is [3 5 6] 7 17 18] 19
+    // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
+    // Remove first 4 docs (3, 1, 2, 4) forcing buffer to become empty and
+    // schedule a repoll.
+    debugger
+    rem({ bar: { $lt: 10 } });
+    var expectedRemoves = [{removed: docId3}, {removed: docId1},
+                           {removed: docId2}, {removed: docId4}];
+    var expectedAdds = [{added: docId6}, {added: docId7}, {added: docId8}];
+
+    test.length(o.output, 7);
+    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]]));
+    test.equal(o.output, expectedRemoves);
 
     o.handle.stop();
     onComplete();

From 3ac9cf88b74842d965a41932d1108b2f06d3eaa7 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 12 Feb 2014 21:12:33 -0800
Subject: [PATCH 011/123] Fix the test on limit-requery ; keep docs in
 _published and _unpublishedBuffer consistently w/o _id

---
 packages/mongo-livedata/mongo_livedata_tests.js | 10 +++++-----
 packages/mongo-livedata/oplog_observe_driver.js |  2 ++
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 14ba50208c..4c55df86c1 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -869,15 +869,15 @@ if (Meteor.isServer) {
     // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
     // Remove first 4 docs (3, 1, 2, 4) forcing buffer to become empty and
     // schedule a repoll.
-    debugger
     rem({ bar: { $lt: 10 } });
     var expectedRemoves = [{removed: docId3}, {removed: docId1},
                            {removed: docId2}, {removed: docId4}];
-    var expectedAdds = [{added: docId6}, {added: docId7}, {added: docId8}];
+    var expectedAdds = [{added: docId4}, {added: docId8},
+                        {added: docId7}, {added: docId6}];
 
-    test.length(o.output, 7);
-    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]]));
-    test.equal(o.output, expectedRemoves);
+    test.length(o.output, 8);
+    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]], expectedRemoves));
+    test.equal([o.output[1], o.output[3], o.output[5], o.output[7]], expectedAdds);
 
     o.handle.stop();
     onComplete();
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 0febd194e3..3407ea702a 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -578,6 +578,7 @@ _.extend(OplogObserveDriver.prototype, {
       self._removePublished(id);
     });
 
+    // xcxc this should be sorted?
     // Now do adds and changes.
     newResults.forEach(function (doc, id) {
       // "true" here means to throw if we think this doc doesn't match the
@@ -587,6 +588,7 @@ _.extend(OplogObserveDriver.prototype, {
 
     // Finally, replace the buffer
     newBuffer.forEach(function (doc, id) {
+      delete doc._id;
       self._addBuffered(id, doc);
     });
   },

From 10d97c70c50e3aa7e73115556fa43cd57eeade42 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 14 Feb 2014 18:58:19 -0800
Subject: [PATCH 012/123] wip fixing the appends to buffer

---
 .../mongo-livedata/mongo_livedata_tests.js    | 37 +++++++++++++++++--
 .../mongo-livedata/oplog_observe_driver.js    | 27 ++++++++++----
 2 files changed, 53 insertions(+), 11 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 4c55df86c1..143516182d 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -746,6 +746,8 @@ if (Meteor.isServer) {
     x++;
   });
 
+  // This test mainly checks the correctness of oplog code dealing with limited
+  // queries. Compitablity with poll-diff is added as well.
   Tinytest.addAsync("mongo-livedata - observe sorted, limited " + idGeneration, function (test, onComplete) {
     var run = test.runId();
     var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
@@ -770,6 +772,7 @@ if (Meteor.isServer) {
     };
 
     var ins = function (doc) {
+      if (doc.bar) doc._id = doc.bar.toString();
       var id; runInFence(function () { id = coll.insert(doc); });
       return id;
     };
@@ -824,8 +827,8 @@ if (Meteor.isServer) {
     // Now remove something and that doc 2 should be right back
     rem(docId5);
     test.length(o.output, 2);
-    test.equal(o.output.shift(), {removed: docId5});
-    test.equal(o.output.shift(), {added: docId2});
+    test.isTrue(setsEqual(o.output, [{removed: docId5}, {added: docId2}]));
+    o.output.slice(0, 2);
 
     // Current state is [3 5 6] 7]
     // Add some negative numbers overflowing the buffer.
@@ -865,7 +868,7 @@ if (Meteor.isServer) {
                [expected[1], expected[3], expected[5]]);
     o.output.splice(0, 6);
 
-    // The new arrangment is [3 5 6] 7 17 18] 19
+    // The new arrangement is [3 5 6] 7 17 18] 19
     // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
     // Remove first 4 docs (3, 1, 2, 4) forcing buffer to become empty and
     // schedule a repoll.
@@ -878,6 +881,34 @@ if (Meteor.isServer) {
     test.length(o.output, 8);
     test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]], expectedRemoves));
     test.equal([o.output[1], o.output[3], o.output[5], o.output[7]], expectedAdds);
+    o.output.splice(0, 8);
+
+    // The new arrangement is [17 18 19] or [docId6 docId7 docId8]
+    var docId9 = ins({ foo: 22, bar: 21 });
+    var docId10 = ins({ foo: 22, bar: 31 });
+    var docId11 = ins({ foo: 22, bar: 41 });
+    var docId12 = ins({ foo: 22, bar: 51 });
+
+    console.log("----------------------d");
+    console.log(o.handle._multiplexer._observeDriver._published.idMap._map);
+    console.log(o.handle._multiplexer._observeDriver._unpublishedBuffer.idMap._map);
+    debugger;
+    upd({ bar: { $lt: 20 } }, { $inc: { bar: 5 } }, { multi: true });
+    // Becomes [21 22 23] 24 31 41] 51
+    test.length(o.output, 4);
+    test.equal(o.output.shift(), { removed: docId6 });
+    test.equal(o.output.shift(), { added: docId9 });
+    test.equal(o.output.shift(), { changed: docId7 });
+    test.equal(o.output.shift(), { changed: docId8 });
+    console.log(o.handle._multiplexer._observeDriver._published.idMap._map);
+    console.log(o.handle._multiplexer._observeDriver._unpublishedBuffer.idMap._map);
+    console.log("----------------------p");
+
+    rem(docId9);
+    // Becomes [22 23 24] 31 41] 51
+    test.length(o.output, 2);
+    test.equal(o.output.shift(), { removed: docId9 });
+    test.equal(o.output.shift(), { added: docId6 });
 
     o.handle.stop();
     onComplete();
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 3407ea702a..3b9a438b9c 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -41,6 +41,7 @@ OplogObserveDriver = function (options) {
     self._unpublishedBuffer = null;
     self._published = new LocalCollection._IdMap;
   }
+  self._justUpdatedBuffer = false;
 
   self._stopped = false;
   self._stopHandles = [];
@@ -182,8 +183,10 @@ _.extend(OplogObserveDriver.prototype, {
   _addBuffered: function (id, doc) {
     var self = this;
     self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc));
-    if (self._unpublishedBuffer.size() > self._limit)
+    if (self._unpublishedBuffer.size() > self._limit) {
       self._unpublishedBuffer.remove(self._unpublishedBuffer.maxElementId());
+      self._justUpdatedBuffer = false;
+    }
   },
   _removeBuffered: function (id) {
     var self = this;
@@ -212,9 +215,12 @@ _.extend(OplogObserveDriver.prototype, {
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
     // Otherwise we might need to buffer it (only in case of limited query).
+    // Buffering a new document is allowed only if it is inserted in the middle
+    // or the beginning of it as we cannot determine if there are documents
+    // outside of the buffer easily.
     if (!limit || self._published.size() < limit || comparator(maxPublished, fields) > 0) {
       self._addPublished(id, fields);
-    } else if (self._unpublishedBuffer.size() < limit || comparator(maxBuffered, fields) > 0) {
+    } else if (self._justUpdatedBuffer || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
       self._addBuffered(id, fields);
     }
   },
@@ -420,11 +426,15 @@ _.extend(OplogObserveDriver.prototype, {
     // XXX needs more thought on non-zero skip
     // XXX "2" here is a "magic number"
     var initialCursor = self._cursorForQuery({ limit: self._limit * 2 });
-    initialCursor.forEach(function (initialDoc) {
-      // self._addMatching knows how to correctly separate documents into the
-      // published set and the buffer.
-      self._addMatching(initialDoc);
+    initialCursor.forEach(function (initialDoc, i) {
+      var id = initialDoc._id;
+      delete initialDoc._id;
+      if (!self._limit || i < self._limit)
+        self._addPublished(id, initialDoc);
+      else
+        self._addBuffered(id, initialDoc);
     });
+    self._justUpdatedBuffer = true;
     if (self._stopped)
       throw new Error("oplog stopped quite early");
     // Allow observeChanges calls to return. (After this, it's possible for
@@ -477,7 +487,6 @@ _.extend(OplogObserveDriver.prototype, {
       });
 
       self._publishNewResults(newResults, newBuffer);
-
       self._doneQuerying();
     });
   },
@@ -578,8 +587,9 @@ _.extend(OplogObserveDriver.prototype, {
       self._removePublished(id);
     });
 
-    // xcxc this should be sorted?
     // Now do adds and changes.
+    // If self has a buffer and limit, the new fetched result will be
+    // ordered correctly as the query has sort specifier.
     newResults.forEach(function (doc, id) {
       // "true" here means to throw if we think this doc doesn't match the
       // selector.
@@ -591,6 +601,7 @@ _.extend(OplogObserveDriver.prototype, {
       delete doc._id;
       self._addBuffered(id, doc);
     });
+    self._justUpdatedBuffer = true;
   },
 
   // This stop function is invoked from the onStop of the ObserveMultiplexer, so

From 92d38af16dc4d199f95f8e9201be32eeb60b38ee Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sat, 15 Feb 2014 01:32:05 -0800
Subject: [PATCH 013/123] wip changed published document can go into buffer

---
 packages/mongo-livedata/oplog_observe_driver.js | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 3b9a438b9c..02f0909165 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -276,6 +276,12 @@ _.extend(OplogObserveDriver.prototype, {
         } else {
           // after the change doc doesn't stay in the published, remove it
           self._removePublished(id);
+          // but it can move into buffered now, check it
+          var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
+          if (self._justUpdatedBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0))
+            self._addPublished(id, newDoc);
+          else
+            self._justUpdatedBuffer = false;
         }
       } else if (bufferedBefore) {
         // after the change we can't know if doc is still in the buffer limit

From 1071b37ed445225c19f9ff96dac7f95e8632fbd5 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 13:47:26 -0800
Subject: [PATCH 014/123] Fix tests for oplog

---
 .../mongo-livedata/mongo_livedata_tests.js    | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 143516182d..c0c1fe6dc6 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -770,6 +770,7 @@ if (Meteor.isServer) {
 
       return {output: output, handle: handle};
     };
+    var clearOutput = function (o) { o.output.splice(0, o.output.length); };
 
     var ins = function (doc) {
       if (doc.bar) doc._id = doc.bar.toString();
@@ -828,7 +829,7 @@ if (Meteor.isServer) {
     rem(docId5);
     test.length(o.output, 2);
     test.isTrue(setsEqual(o.output, [{removed: docId5}, {added: docId2}]));
-    o.output.slice(0, 2);
+    clearOutput(o);
 
     // Current state is [3 5 6] 7]
     // Add some negative numbers overflowing the buffer.
@@ -843,7 +844,7 @@ if (Meteor.isServer) {
                     {added: docId8}, {removed: docId3}];
 
     test.equal(o.output, expected);
-    o.output.splice(0, 6);
+    clearOutput(o);
 
     // Now the state is [-3 -2 -1] 3 5 6] 7
     // If we update first 3 docs (increment them by 20), it would be
@@ -866,7 +867,7 @@ if (Meteor.isServer) {
                           [expected[0], expected[2], expected[4]]));
     test.equal([o.output[1], o.output[3], o.output[5]],
                [expected[1], expected[3], expected[5]]);
-    o.output.splice(0, 6);
+    clearOutput(o);
 
     // The new arrangement is [3 5 6] 7 17 18] 19
     // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
@@ -881,7 +882,7 @@ if (Meteor.isServer) {
     test.length(o.output, 8);
     test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]], expectedRemoves));
     test.equal([o.output[1], o.output[3], o.output[5], o.output[7]], expectedAdds);
-    o.output.splice(0, 8);
+    clearOutput(o);
 
     // The new arrangement is [17 18 19] or [docId6 docId7 docId8]
     var docId9 = ins({ foo: 22, bar: 21 });
@@ -889,10 +890,7 @@ if (Meteor.isServer) {
     var docId11 = ins({ foo: 22, bar: 41 });
     var docId12 = ins({ foo: 22, bar: 51 });
 
-    console.log("----------------------d");
-    console.log(o.handle._multiplexer._observeDriver._published.idMap._map);
-    console.log(o.handle._multiplexer._observeDriver._unpublishedBuffer.idMap._map);
-    debugger;
+    test.length(o.output, 0);
     upd({ bar: { $lt: 20 } }, { $inc: { bar: 5 } }, { multi: true });
     // Becomes [21 22 23] 24 31 41] 51
     test.length(o.output, 4);
@@ -900,15 +898,14 @@ if (Meteor.isServer) {
     test.equal(o.output.shift(), { added: docId9 });
     test.equal(o.output.shift(), { changed: docId7 });
     test.equal(o.output.shift(), { changed: docId8 });
-    console.log(o.handle._multiplexer._observeDriver._published.idMap._map);
-    console.log(o.handle._multiplexer._observeDriver._unpublishedBuffer.idMap._map);
-    console.log("----------------------p");
+    clearOutput(o);
 
     rem(docId9);
     // Becomes [22 23 24] 31 41] 51
     test.length(o.output, 2);
     test.equal(o.output.shift(), { removed: docId9 });
     test.equal(o.output.shift(), { added: docId6 });
+    clearOutput(o);
 
     o.handle.stop();
     onComplete();

From 5b0663993abf2eac076ba162f3db75c0721f4bf1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 13:47:38 -0800
Subject: [PATCH 015/123] Fix typo: addPublished -> addBuffered;

Revert the population of initialQuerying back to _addMatching everything.
---
 packages/mongo-livedata/oplog_observe_driver.js | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 02f0909165..ac33779eae 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -279,7 +279,7 @@ _.extend(OplogObserveDriver.prototype, {
           // but it can move into buffered now, check it
           var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
           if (self._justUpdatedBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0))
-            self._addPublished(id, newDoc);
+            self._addBuffered(id, newDoc);
           else
             self._justUpdatedBuffer = false;
         }
@@ -432,13 +432,8 @@ _.extend(OplogObserveDriver.prototype, {
     // XXX needs more thought on non-zero skip
     // XXX "2" here is a "magic number"
     var initialCursor = self._cursorForQuery({ limit: self._limit * 2 });
-    initialCursor.forEach(function (initialDoc, i) {
-      var id = initialDoc._id;
-      delete initialDoc._id;
-      if (!self._limit || i < self._limit)
-        self._addPublished(id, initialDoc);
-      else
-        self._addBuffered(id, initialDoc);
+    initialCursor.forEach(function (initialDoc) {
+      self._addMatching(initialDoc);
     });
     self._justUpdatedBuffer = true;
     if (self._stopped)

From 8b931e64d6dbe811e4cc48472625b9551e91d216 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 15:00:51 -0800
Subject: [PATCH 016/123] Tests behave correctly in both oplog and non-oplog
 mode

---
 .../mongo-livedata/mongo_livedata_tests.js    | 55 ++++++++++++-------
 1 file changed, 35 insertions(+), 20 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index c0c1fe6dc6..46abc8e78b 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -856,17 +856,19 @@ if (Meteor.isServer) {
     // But since our buffer becomes empty, it will be refilled partially with
     // updated documents.
     test.length(o.output, 6);
-    expected = [{removed: docId6}, {added: docId3},
-                {removed: docId7}, {added: docId1},
-                {removed: docId8}, {added: docId2}];
+    var expectedRemoves = [{removed: docId6},
+                           {removed: docId7},
+                           {removed: docId8}];
+    var expectedAdds = [{added: docId3},
+                        {added: docId1},
+                        {added: docId2}];
 
     // Note: since we are updating multiple things, the order of updates may
     // differ from launch to launch. That's why we compare even positions
     // (removes) w/o looking at ordering.
-    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]],
-                          [expected[0], expected[2], expected[4]]));
-    test.equal([o.output[1], o.output[3], o.output[5]],
-               [expected[1], expected[3], expected[5]]);
+    test.isTrue(setsEqual(_.filter(o.output, function (e) {return e.removed;}),
+                          expectedRemoves));
+    test.equal(_.filter(o.output, function (e){return e.added;}), expectedAdds);
     clearOutput(o);
 
     // The new arrangement is [3 5 6] 7 17 18] 19
@@ -874,14 +876,28 @@ if (Meteor.isServer) {
     // Remove first 4 docs (3, 1, 2, 4) forcing buffer to become empty and
     // schedule a repoll.
     rem({ bar: { $lt: 10 } });
-    var expectedRemoves = [{removed: docId3}, {removed: docId1},
-                           {removed: docId2}, {removed: docId4}];
-    var expectedAdds = [{added: docId4}, {added: docId8},
-                        {added: docId7}, {added: docId6}];
 
-    test.length(o.output, 8);
-    test.isTrue(setsEqual([o.output[0], o.output[2], o.output[4]], expectedRemoves));
-    test.equal([o.output[1], o.output[3], o.output[5], o.output[7]], expectedAdds);
+    // XXX the oplog code analyzes the events one by one: one remove after
+    // another. Poll-n-diff code, on the other side, analyzes the batch action
+    // of multiple remove. Because of that difference, expected outputs differ.
+    if (o.handle._multiplexer._observeDriver._usesOplog) {
+      var expectedRemoves = [{removed: docId3}, {removed: docId1},
+                             {removed: docId2}, {removed: docId4}];
+      var expectedAdds = [{added: docId4}, {added: docId8},
+                          {added: docId7}, {added: docId6}];
+
+      test.length(o.output, 8);
+    } else {
+      var expectedRemoves = [{removed: docId3}, {removed: docId1},
+                             {removed: docId2}];
+      var expectedAdds = [{added: docId8}, {added: docId7}, {added: docId6}];
+
+      test.length(o.output, 6);
+    }
+
+    test.isTrue(setsEqual(_.filter(o.output, function (e) {return e.removed;}),
+                          expectedRemoves));
+    test.equal(_.filter(o.output, function (e) {return e.added;}), expectedAdds);
     clearOutput(o);
 
     // The new arrangement is [17 18 19] or [docId6 docId7 docId8]
@@ -894,17 +910,16 @@ if (Meteor.isServer) {
     upd({ bar: { $lt: 20 } }, { $inc: { bar: 5 } }, { multi: true });
     // Becomes [21 22 23] 24 31 41] 51
     test.length(o.output, 4);
-    test.equal(o.output.shift(), { removed: docId6 });
-    test.equal(o.output.shift(), { added: docId9 });
-    test.equal(o.output.shift(), { changed: docId7 });
-    test.equal(o.output.shift(), { changed: docId8 });
+    test.isTrue(setsEqual(o.output, [{removed: docId6},
+                                     {added: docId9},
+                                     {changed: docId7},
+                                     {changed: docId8}]));
     clearOutput(o);
 
     rem(docId9);
     // Becomes [22 23 24] 31 41] 51
     test.length(o.output, 2);
-    test.equal(o.output.shift(), { removed: docId9 });
-    test.equal(o.output.shift(), { added: docId6 });
+    test.isTrue(setsEqual(o.output, [{removed: docId9}, {added: docId6}]));
     clearOutput(o);
 
     o.handle.stop();

From a5ddd71b1d14c70123875a7e6cb78ab1b07c6cc9 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 15:05:18 -0800
Subject: [PATCH 017/123] Tests on newly supported cursor types

---
 packages/mongo-livedata/oplog_tests.js | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/packages/mongo-livedata/oplog_tests.js b/packages/mongo-livedata/oplog_tests.js
index a94527f322..5d28c01a44 100644
--- a/packages/mongo-livedata/oplog_tests.js
+++ b/packages/mongo-livedata/oplog_tests.js
@@ -1,8 +1,8 @@
 var OplogCollection = new Meteor.Collection("oplog-" + Random.id());
 
 Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) {
-  var supported = function (expected, selector) {
-    var cursor = OplogCollection.find(selector);
+  var supported = function (expected, selector, options) {
+    var cursor = OplogCollection.find(selector, options);
     var handle = cursor.observeChanges({added: function () {}});
     test.equal(!!handle._multiplexer._observeDriver._usesOplog, expected);
     handle.stop();
@@ -38,4 +38,9 @@ Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) {
   // Nothing Minimongo doesn't understand.  (Minimongo happens to fail to
   // implement $elemMatch inside $all which MongoDB supports.)
   supported(false, {x: {$all: [{$elemMatch: {y: 2}}]}});
+
+  supported(true, {}, { sort: {x:1} });
+  supported(true, {}, { sort: {x:1}, limit: 5 });
+  supported(false, {}, { limit: 5 });
+  supported(false, {}, { skip: 2, limit: 5 });
 });

From c03701ef1c98838a52d6030ea7cf334612dc32dc Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 17:01:10 -0800
Subject: [PATCH 018/123] Remove Sorter class from MinimongoTest as it is
 already exported in Minimongo symbol

---
 packages/minimongo/minimongo_tests.js | 8 ++++----
 packages/minimongo/sort.js            | 1 -
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js
index 4bb58d2b94..8560139017 100644
--- a/packages/minimongo/minimongo_tests.js
+++ b/packages/minimongo/minimongo_tests.js
@@ -1549,7 +1549,7 @@ Tinytest.add("minimongo - ordering", function (test) {
   // document ordering under a sort specification
   var verify = function (sorts, docs) {
     _.each(_.isArray(sorts) ? sorts : [sorts], function (sort) {
-      var sorter = new MinimongoTest.Sorter(sort);
+      var sorter = new Minimongo.Sorter(sort);
       assert_ordering(test, sorter.getComparator(), docs);
     });
   };
@@ -1577,15 +1577,15 @@ Tinytest.add("minimongo - ordering", function (test) {
          [{c: 1}, {a: 1, b: 2}, {a: 1, b: 3}, {a: 2, b: 0}]);
 
   test.throws(function () {
-    new MinimongoTest.Sorter("a");
+    new Minimongo.Sorter("a");
   });
 
   test.throws(function () {
-    new MinimongoTest.Sorter(123);
+    new Minimongo.Sorter(123);
   });
 
   // No sort spec implies everything equal.
-  test.equal(new MinimongoTest.Sorter({}).getComparator()({a:1}, {a:2}), 0);
+  test.equal(new Minimongo.Sorter({}).getComparator()({a:1}, {a:2}), 0);
 
   // All sorts of array edge cases!
   // Increasing sort sorts by the smallest element it finds; 1 < 2.
diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js
index 60c3dd60b2..5a1e57a3db 100644
--- a/packages/minimongo/sort.js
+++ b/packages/minimongo/sort.js
@@ -111,7 +111,6 @@ Sorter.prototype.getComparator = function (options) {
 };
 
 Minimongo.Sorter = Sorter;
-MinimongoTest.Sorter = Sorter;
 
 // Given an array of comparators
 // (functions (a,b)->(negative or positive or zero)), returns a single

From 07a18984b15a581e07ae4efcba01ee8b6dba2a38 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 17:25:01 -0800
Subject: [PATCH 019/123] More comments, explicity, fixed ambiguities

---
 .../mongo-livedata/oplog_observe_driver.js    | 35 +++++++++++++------
 1 file changed, 25 insertions(+), 10 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index ac33779eae..dbfbcbf930 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -20,14 +20,22 @@ OplogObserveDriver = function (options) {
   self._mongoHandle = options.mongoHandle;
   self._multiplexer = options.multiplexer;
 
+  if (options.ordered) {
+    throw Error("OplogObserveDriver only supports unordered observeChanges");
+  }
+
   if (options.cursorDescription.options.limit) {
     // There are several properties ordered driver implements:
     // - _limit is a positive number
     // - _comparator is a function-comparator by which the query is ordered
-    // - _unpublishedBuffer is non-null collection
+    // - _unpublishedBuffer is non-null collection,
+    //                      the empty buffer in STEADY phase implies that the
+    //                      everything that matches the queries selector fits
+    //                      into published set.
     // - _published implements maxElementId method in addition to IdMap methods
 
-    // XXX replace with doubly-heaps and shit once we get these working
+    // We don't support $near and other geo-queries so it's OK to initialize the
+    // comparator only once in the constructor.
     var sorter = new Minimongo.Sorter(options.cursorDescription.options.sort);
     var comparator = sorter.getComparator();
     self._limit = self._cursorDescription.options.limit;
@@ -191,6 +199,9 @@ _.extend(OplogObserveDriver.prototype, {
   _removeBuffered: function (id) {
     var self = this;
     self._unpublishedBuffer.remove(id);
+    // To keep the contract "buffer is never empty in STEADY phase unless the
+    // everything matching fits into published" true, we poll everything as soon
+    // as we see the buffer becoming empty.
     if (! self._unpublishedBuffer.size())
       self._needToPollQuery();
   },
@@ -256,7 +267,7 @@ _.extend(OplogObserveDriver.prototype, {
       self._addMatching(newDoc);
     } else if (cachedBefore && !matchesNow) {
       self._removeMatching(id);
-    } else if (matchesNow) {
+    } else if (cachedBefore && matchesNow) {
       delete newDoc._id;
       var oldDoc = self._published.get(id);
       var comparator = self._comparator;
@@ -264,14 +275,14 @@ _.extend(OplogObserveDriver.prototype, {
         self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId());
 
       if (publishedBefore) {
-        // Unordered case where the document stays in published once it matches
+        // Unlimited case where the document stays in published once it matches
         // or the case when we don't have enough matching docs to publish or the
         // changed but matching doc will stay in published anyways.
         // XXX: We rely on the emptiness of buffer. Be sure to maintain the fact
         // that buffer can't be empty if there are matching documents not
         // published. Notably, we don't want to schedule repoll and continue
         // relying on this property.
-        if (!self._limit || self._unpublishedBuffer.size() === 0 || comparator(newDoc, minBuffered) < 1) {
+        if (!self._limit || self._unpublishedBuffer.size() === 0 || comparator(newDoc, minBuffered) <= 0) {
           self._changePublished(id, oldDoc, newDoc);
         } else {
           // after the change doc doesn't stay in the published, remove it
@@ -291,6 +302,8 @@ _.extend(OplogObserveDriver.prototype, {
         var maxPublished = self._published.get(self._published.maxElementId());
         if (comparator(newDoc, maxPublished) < 0)
           self._addPublished(id, newDoc);
+      } else {
+        throw new Error("cachedBefore implies either of publishedBefore or bufferedBefore is true.");
       }
     }
   },
@@ -571,7 +584,7 @@ _.extend(OplogObserveDriver.prototype, {
   _publishNewResults: function (newResults, newBuffer) {
     var self = this;
 
-    // If the query is ordered and there is a buffer, shut down so it doesn't
+    // If the query is limited and there is a buffer, shut down so it doesn't
     // stay in a way.
     if (self._limit) {
       self._unpublishedBuffer.clear();
@@ -590,7 +603,7 @@ _.extend(OplogObserveDriver.prototype, {
 
     // Now do adds and changes.
     // If self has a buffer and limit, the new fetched result will be
-    // ordered correctly as the query has sort specifier.
+    // limited correctly as the query has sort specifier.
     newResults.forEach(function (doc, id) {
       // "true" here means to throw if we think this doc doesn't match the
       // selector.
@@ -667,8 +680,8 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
 
   // This option (which are mostly used for sorted cursors) require us to figure
   // out where a given document fits in an order to know if it's included or
-  // not, and we don't track that information when doing oplog tailing.
-  if (options.limit && (options.skip || !options.sort)) return false;
+  // not. We do it only if skip is not defined or 0.
+  if (options.skip || (options.limit && !options.sort)) return false;
 
   // If a fields projection option is given check if it is supported by
   // minimongo (some operators are not supported).
@@ -688,7 +701,9 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   //             as Mongo, and can yield!)
   //   - $near (has "interesting" properties in MongoDB, like the possibility
   //            of returning an ID multiple times, though even polling maybe
-  //            have a bug there
+  //            have a bug there)
+  //           XXX: once we support it, we would need to think more on how we
+  //           initialize the comparators when we create the driver.
   return !matcher.hasWhere() && !matcher.hasGeoQuery();
 };
 

From f336c61eda75174e75879af2132c62071b64d8aa Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 17:30:46 -0800
Subject: [PATCH 020/123] More self-checks with throws (asserts)

---
 packages/mongo-livedata/oplog_observe_driver.js | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index dbfbcbf930..8edffca8e5 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -147,14 +147,18 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._limit && self._published.size() > self._limit) {
       // XXX in theory the size of published is no more than limit+1
       if (self._published.size() !== self._limit + 1) {
-        // xcxc better error message
         throw new Error("After adding to published, " +
-                        (self._limit - self._published.size()) +
+                        (self._published.size() - self._limit) +
                         " documents are overflowing the set");
       }
 
       var overflowingDocId = self._published.maxElementId();
       var overflowingDoc = self._published.get(overflowingDocId);
+
+      if (_.isEqual(overflowingDocId, id)) {
+        throw new Error("The document just added is overflowing the published set");
+      }
+
       self._published.remove(overflowingDocId);
       self._multiplexer.removed(overflowingDocId);
       self._addBuffered(overflowingDocId, overflowingDoc);
@@ -192,7 +196,13 @@ _.extend(OplogObserveDriver.prototype, {
     var self = this;
     self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc));
     if (self._unpublishedBuffer.size() > self._limit) {
-      self._unpublishedBuffer.remove(self._unpublishedBuffer.maxElementId());
+      var maxBufferedId = self._unpublishedBuffer.maxElementId();
+
+      if (_.isEqual(maxBufferedId, id)) {
+        throw new Error("The document just added to buffer is overflowing the buffer");
+      }
+
+      self._unpublishedBuffer.remove(maxBufferedId);
       self._justUpdatedBuffer = false;
     }
   },

From e4eb3e3c75d525d8e2d878f97b1b5c87f303dc28 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 17:49:39 -0800
Subject: [PATCH 021/123] Never put something into buffer if it will drop out
 in a moment after

---
 packages/mongo-livedata/oplog_observe_driver.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 8edffca8e5..ea0fa7fd5a 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -241,7 +241,7 @@ _.extend(OplogObserveDriver.prototype, {
     // outside of the buffer easily.
     if (!limit || self._published.size() < limit || comparator(maxPublished, fields) > 0) {
       self._addPublished(id, fields);
-    } else if (self._justUpdatedBuffer || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
+    } else if ((self._justUpdatedBuffer && self._unpublishedBuffer.size() < limit) || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
       self._addBuffered(id, fields);
     }
   },

From 8cad412ea199242943b943d0fc9dd72d1b9e778d Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 15:50:50 -0800
Subject: [PATCH 022/123] init commit for BinaryHeap package

Nothing here yet
---
 packages/binary-heap/binary-heap-tests.js |  0
 packages/binary-heap/binary-heap.js       | 40 +++++++++++++++++++++++
 packages/binary-heap/package.js           | 17 ++++++++++
 3 files changed, 57 insertions(+)
 create mode 100644 packages/binary-heap/binary-heap-tests.js
 create mode 100644 packages/binary-heap/binary-heap.js
 create mode 100644 packages/binary-heap/package.js

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
new file mode 100644
index 0000000000..0d112965b5
--- /dev/null
+++ b/packages/binary-heap/binary-heap.js
@@ -0,0 +1,40 @@
+BinaryHeap = function (comparator, initData) {
+  if (! _.isFunction(comparator))
+    throw new Error('Passed comparator is invalid, should be a comparison function');
+  var self = this;
+  self._comparator = comparator;
+  self._map = {};
+  self._heap = [];
+
+  if (_.isArray(initData))
+    self._initFromData(initData);
+};
+
+var idStringify, idParse;
+if (Package.minimongo) {
+  idStringify = Package.minimongo.LocalCollection._idStringify;
+  idParse = Package.minimongo.LocalCollection._idParse;
+} else {
+  // XXX: These can't deal with special strings like '__proto__'
+  // XXX: or '{ looksLike: "object" }' or numbers.
+  idStringify = function (id) { return JSON.stringify(id); };
+  idParse = function (id) { return JSON.parse(id); }
+}
+
+_.extend(BinaryHeap.prototype, {
+  _initFromData: function (data) {},
+
+  get: function (id) {},
+  set: function (id, value) {},
+  remove: function (id) {},
+  has: function (id) {},
+  empty: function (id) {},
+  clear: function () {},
+  forEach: function (iterator) {},
+  size: function () {},
+  setDefault: function () {},
+  clone: function () {},
+
+  maxElementId: function () {}
+});
+
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
new file mode 100644
index 0000000000..fab25f5e1e
--- /dev/null
+++ b/packages/binary-heap/package.js
@@ -0,0 +1,17 @@
+Package.describe({
+  summary: "Binary Heap datastructure implementation",
+  internal: true
+});
+
+Package.on_use(function (api) {
+  api.export('BinaryHeap');
+  api.use(['underscore']);
+  api.use(['minimongo'], { weak: true });
+  api.add_files('binary-heap.js');
+});
+
+Package.on_test(function (api) {
+  api.use('tinytest');
+  api.add_files('binary-heap-tests.js');
+});
+

From 7f2c6bb09b75e6e943fc5bfabd523766627529c2 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 20:47:41 -0800
Subject: [PATCH 023/123] The implementation of Heap

---
 packages/binary-heap/binary-heap.js | 172 +++++++++++++++++++++++++---
 1 file changed, 156 insertions(+), 16 deletions(-)

diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
index 0d112965b5..9ad1731c2d 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/binary-heap.js
@@ -1,40 +1,180 @@
+// Constructor of Heap
+// - comparator - Function - given two items returns a number
+// - initData - Array - the initial data in a format:
+//      Object:
+//        - id - String - unique id of the item
+//        - value - Any - the data value
+//    the contents of initData is retained
 BinaryHeap = function (comparator, initData) {
   if (! _.isFunction(comparator))
     throw new Error('Passed comparator is invalid, should be a comparison function');
   var self = this;
   self._comparator = comparator;
-  self._map = {};
+  self._heapIdx = {};
   self._heap = [];
 
   if (_.isArray(initData))
     self._initFromData(initData);
 };
 
-var idStringify, idParse;
+var idStringify;
 if (Package.minimongo) {
   idStringify = Package.minimongo.LocalCollection._idStringify;
-  idParse = Package.minimongo.LocalCollection._idParse;
 } else {
   // XXX: These can't deal with special strings like '__proto__'
   // XXX: or '{ looksLike: "object" }' or numbers.
   idStringify = function (id) { return JSON.stringify(id); };
-  idParse = function (id) { return JSON.parse(id); }
 }
 
 _.extend(BinaryHeap.prototype, {
-  _initFromData: function (data) {},
+  _initFromData: function (data) {
+    var self = this;
 
-  get: function (id) {},
-  set: function (id, value) {},
-  remove: function (id) {},
-  has: function (id) {},
-  empty: function (id) {},
-  clear: function () {},
-  forEach: function (iterator) {},
-  size: function () {},
-  setDefault: function () {},
-  clone: function () {},
+    self._heap = _.clone(data);
 
-  maxElementId: function () {}
+    _.each(data, function (o, i) {
+      self._heapIdx[idStringify(o.id)] = i;
+    });
+
+    for (var i = parentIdx(data.length); i >= 0; i--)
+      self._heapify(i);
+  },
+
+  _heapify: function (idx) {
+    var self = this;
+
+    while (leftChildIdx(idx) < self.size()) {
+      var left = leftChildIdx(idx);
+      var right = rightChildIdx(idx);
+      var largest = idx;
+
+      if (left < self.size() &&
+          self._comparator(self._get(left), self._get(largest)) > 0) {
+        largest = left;
+      }
+      if (right < self.size() &&
+          self._comparator(self._get(right), self._get(largest)) > 0) {
+        largest = right;
+      }
+
+      if (largest === idx)
+        break;
+
+      self._swap(largest, idx);
+      idx = largest;
+    }
+  },
+
+  _upHeap: function (idx) {
+    var self = this;
+    var value = self._get(idx);
+    var ancestor = idx;
+
+    while (ancestor > 0) {
+      var parent = parentIdx(ancestor);
+      if (self._comparator(self._get(parent), value) < 0)
+        ancestor = parent;
+      else
+        break;
+    }
+
+    if (ancestor !== idx)
+      self._swap(ancestor, idx);
+  },
+
+  // Internal: gets raw data object placed on idxth place in heap
+  _get: function (idx) {
+    var self = this;
+    return self._heap[idx].value;
+  },
+
+  _swap: function (idxA, idxB) {
+    var self = this;
+    var A = self._heap[idxA];
+    var B = self._heap[idxB];
+
+    self._heapIdx[idStringify(A.id)] = idxB;
+    self._heapIdx[idStringify(B.id)] = idxA;
+
+    self._heap[idxA] = B;
+    self._heap[idxB] = A;
+  },
+
+  get: function (id) {
+    var self = this;
+    return self._get(self._heapIdx[idStringify(id)]);
+  },
+  set: function (id, value) {
+    var self = this;
+
+    if (self.has(id)) {
+      if (self.get(id) === value)
+        return;
+      else
+        self.remove(id);
+    }
+
+    self._heapIdx[idStringify(id)] = self._heap.length;
+    self._heap.push({ id: id, value: value });
+    self._upHeap(self._heap.length - 1);
+  },
+  remove: function (id) {
+    var self = this;
+    var strId = idStringify(id);
+
+    if (! self.has(id)) {
+      var last = self._heap.length - 1;
+      var idx = self._heapIdx[strId];
+      self._swap(idx, last);
+      self._heap.pop();
+      self._heapify(idx);
+      delete self._heapIdx[strId];
+    }
+  },
+  has: function (id) {
+    var self = this;
+    return self._heapIdx[idStringify(id)] !== undefined;
+  },
+  empty: function (id) {
+    var self = this;
+    return !self.size();
+  },
+  clear: function () {
+    var self = this;
+    self._heap = [];
+    self._heapIdx = {};
+  },
+  forEach: function (iterator) {
+    var self = this;
+    _.each(self._heap, function (obj) {
+      return iterator(obj.value, obj.id);
+    });
+  },
+  size: function () {
+    var self = this;
+    return self._heap.length;
+  },
+  setDefault: function (id, def) {
+    var self = this;
+    if (self.has(id))
+      return self.get(id);
+    self.set(id, def);
+    return def;
+  },
+  clone: function () {
+    var self = this;
+    var clone = new BinaryHeap(self._comparator);
+    clone._heap = EJSON.clone(self._heap);
+    clone._heapIdx = EJSON.clone(self._heapIdx);
+  },
+
+  maxElementId: function () {
+    var self = this;
+    return self.size() ? self._heap[0].id : null;
+  }
 });
 
+function leftChildIdx (i) { return i * 2 + 1; }
+function rightChildIdx (i) { return i * i + 2; }
+function parentIdx (i) { return (i - 1) >> 1; }
+

From 678d607332622ca28fd0449e5e320b3cdd974fb1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 21:18:03 -0800
Subject: [PATCH 024/123] Fix remove and get of non-existent id

---
 packages/binary-heap/binary-heap.js | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
index 9ad1731c2d..1792cce554 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/binary-heap.js
@@ -102,6 +102,8 @@ _.extend(BinaryHeap.prototype, {
 
   get: function (id) {
     var self = this;
+    if (! self.has(id))
+      return null;
     return self._get(self._heapIdx[idStringify(id)]);
   },
   set: function (id, value) {
@@ -122,12 +124,18 @@ _.extend(BinaryHeap.prototype, {
     var self = this;
     var strId = idStringify(id);
 
-    if (! self.has(id)) {
+    if (self.has(id)) {
       var last = self._heap.length - 1;
       var idx = self._heapIdx[strId];
-      self._swap(idx, last);
-      self._heap.pop();
-      self._heapify(idx);
+
+      if (idx !== last) {
+        self._swap(idx, last);
+        self._heap.pop();
+        self._heapify(idx);
+      } else {
+        self._heap.pop();
+      }
+
       delete self._heapIdx[strId];
     }
   },

From 70a4d6cba1a256b773cb788e9f9c537b13ff3a9f Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 21:18:29 -0800
Subject: [PATCH 025/123] Tests for BinaryHeap

---
 packages/binary-heap/binary-heap-tests.js | 30 +++++++++++++++++++++++
 packages/binary-heap/package.js           |  1 +
 2 files changed, 31 insertions(+)

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index e69de29bb2..16e4232156 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -0,0 +1,30 @@
+Tinytest.add("binary-heap - simple heap tests", function (test) {
+  var h = new BinaryHeap(function (a, b) { return a-b; });
+  h.set("a", 1);
+  h.set("b", 233);
+  h.set("c", -122);
+  h.set("d", 0);
+  h.set("e", 0);
+
+  test.equal(h.size(), 5);
+  test.equal(h.maxElementId(), "b");
+  test.equal(h.get("b"), 233);
+
+  h.remove("b");
+  test.equal(h.size(), 4);
+  test.equal(h.maxElementId(), "a");
+  h.set("e", 44);
+  test.equal(h.maxElementId(), "e");
+  test.equal(h.get("b"), null);
+  test.isTrue(h.has("a"));
+  test.isFalse(h.has("dd"));
+
+  h.clear();
+  test.isFalse(h.has("a"));
+  test.equal(h.size(), 0);
+  test.equal(h.setDefault("a", 12345), 12345);
+  test.equal(h.setDefault("a", 55555), 12345);
+  test.equal(h.size(), 1);
+  test.equal(h.maxElementId(), "a");
+});
+
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
index fab25f5e1e..b832d4e524 100644
--- a/packages/binary-heap/package.js
+++ b/packages/binary-heap/package.js
@@ -12,6 +12,7 @@ Package.on_use(function (api) {
 
 Package.on_test(function (api) {
   api.use('tinytest');
+  api.use('binary-heap');
   api.add_files('binary-heap-tests.js');
 });
 

From 3c4af7cd8f792b1ef8702c1d18f2cc62236f14b8 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 17 Feb 2014 21:19:13 -0800
Subject: [PATCH 026/123] Rename heapify to downHeap to keep it consistent with
 upHeap

Most text-books use "heapify" but in one text-book I remember the use of
"downHeap", so I didn't make it up.
---
 packages/binary-heap/binary-heap.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
index 1792cce554..993d598186 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/binary-heap.js
@@ -37,10 +37,10 @@ _.extend(BinaryHeap.prototype, {
     });
 
     for (var i = parentIdx(data.length); i >= 0; i--)
-      self._heapify(i);
+      self._downHeap(i);
   },
 
-  _heapify: function (idx) {
+  _downHeap: function (idx) {
     var self = this;
 
     while (leftChildIdx(idx) < self.size()) {
@@ -131,7 +131,7 @@ _.extend(BinaryHeap.prototype, {
       if (idx !== last) {
         self._swap(idx, last);
         self._heap.pop();
-        self._heapify(idx);
+        self._downHeap(idx);
       } else {
         self._heap.pop();
       }

From 4dfb2fee4dfbfa1c43bebd5c01a60b579827ead1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 18 Feb 2014 00:32:23 -0800
Subject: [PATCH 027/123] A step to MaxMinHeap

---
 packages/binary-heap/.gitignore           |  1 +
 packages/binary-heap/binary-heap-tests.js | 32 ++++++++++++++++
 packages/binary-heap/binary-heap.js       |  1 +
 packages/binary-heap/max-min-heap.js      | 46 +++++++++++++++++++++++
 packages/binary-heap/package.js           |  3 +-
 5 files changed, 82 insertions(+), 1 deletion(-)
 create mode 100644 packages/binary-heap/.gitignore
 create mode 100644 packages/binary-heap/max-min-heap.js

diff --git a/packages/binary-heap/.gitignore b/packages/binary-heap/.gitignore
new file mode 100644
index 0000000000..677a6fc263
--- /dev/null
+++ b/packages/binary-heap/.gitignore
@@ -0,0 +1 @@
+.build*
diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index 16e4232156..aa026e8148 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -28,3 +28,35 @@ Tinytest.add("binary-heap - simple heap tests", function (test) {
   test.equal(h.maxElementId(), "a");
 });
 
+Tinytest.add("binary-heap - max-min heap tests", function (test) {
+  var h = new MaxMinHeap(function (a, b) { return a-b; });
+  h.set("a", 1);
+  h.set("b", 233);
+  h.set("c", -122);
+  h.set("d", 0);
+  h.set("e", 0);
+
+  test.equal(h.size(), 5);
+  test.equal(h.maxElementId(), "b");
+  test.equal(h.get("b"), 233);
+  test.equal(h.minElementId(), "c");
+
+  h.remove("b");
+  test.equal(h.size(), 4);
+  test.equal(h.minElementId(), "c");
+  h.set("e", -123);
+  test.equal(h.minElementId(), "e");
+  test.equal(h.get("b"), null);
+  test.isTrue(h.has("a"));
+  test.isFalse(h.has("dd"));
+
+  h.clear();
+  test.isFalse(h.has("a"));
+  test.equal(h.size(), 0);
+  test.equal(h.setDefault("a", 12345), 12345);
+  test.equal(h.setDefault("a", 55555), 12345);
+  test.equal(h.size(), 1);
+  test.equal(h.maxElementId(), "a");
+  test.equal(h.minElementId(), "a");
+});
+
diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
index 993d598186..8158ad419d 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/binary-heap.js
@@ -174,6 +174,7 @@ _.extend(BinaryHeap.prototype, {
     var clone = new BinaryHeap(self._comparator);
     clone._heap = EJSON.clone(self._heap);
     clone._heapIdx = EJSON.clone(self._heapIdx);
+    return clone;
   },
 
   maxElementId: function () {
diff --git a/packages/binary-heap/max-min-heap.js b/packages/binary-heap/max-min-heap.js
new file mode 100644
index 0000000000..1b8fe7dc83
--- /dev/null
+++ b/packages/binary-heap/max-min-heap.js
@@ -0,0 +1,46 @@
+MaxMinHeap = function (comparator, initialData) {
+  var self = this;
+
+  self.prototype.constructor.call(self, comparator, initialData);
+  self._minHeap = new BinaryHeap(function (a, b) {
+    return -comparator(a, b);
+  }, initialData);
+};
+
+MaxMinHeap.prototype = Object.create(BinaryHeap);
+
+_.extend(MaxMinHeap.prototype, {
+  set: function (id, value) {
+    var self = this;
+    self._minHeap.set(id, value);
+    BinaryHeap.prototype.set.apply(self, arguments);
+  },
+  remove: function (id) {
+    var self = this;
+    self._minHeap.remove(id);
+    BinaryHeap.prototype.remove.apply(self, arguments);
+  },
+  clear: function () {
+    var self = this;
+    self._minHeap.clear(id);
+    BinaryHeap.prototype.clear.apply(self, arguments);
+  },
+  setDefault: function (id, def) {
+    var self = this;
+    self._minHeap.setDefault(id, def);
+    return BinaryHeap.prototype.setDefault.apply(self, arguments);
+  },
+  clone: function () {
+    var self = this;
+    var clone = new MaxMinHeap(self._comparator);
+    clone._heap = EJSON.clone(self._heap);
+    clone._heapIdx = EJSON.clone(self._heapIdx);
+    clone._minHeap = self._minHeap.clone();
+    return clone;
+  },
+  minElementId: function () {
+    var self = this;
+    return self._minHeap.maxElementId();
+  }
+});
+
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
index b832d4e524..9d1504eae3 100644
--- a/packages/binary-heap/package.js
+++ b/packages/binary-heap/package.js
@@ -5,9 +5,10 @@ Package.describe({
 
 Package.on_use(function (api) {
   api.export('BinaryHeap');
+  api.export('MaxMinHeap');
   api.use(['underscore']);
   api.use(['minimongo'], { weak: true });
-  api.add_files('binary-heap.js');
+  api.add_files(['binary-heap.js', 'max-min-heap.js']);
 });
 
 Package.on_test(function (api) {

From d59200f9517b4f20a86cb612c1bf4210c5fd56f6 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 18 Feb 2014 14:39:58 -0800
Subject: [PATCH 028/123] Do OOP right

---
 packages/binary-heap/max-min-heap.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/binary-heap/max-min-heap.js b/packages/binary-heap/max-min-heap.js
index 1b8fe7dc83..4221d2f9df 100644
--- a/packages/binary-heap/max-min-heap.js
+++ b/packages/binary-heap/max-min-heap.js
@@ -1,13 +1,13 @@
 MaxMinHeap = function (comparator, initialData) {
   var self = this;
 
-  self.prototype.constructor.call(self, comparator, initialData);
+  BinaryHeap.call(self, comparator, initialData);
   self._minHeap = new BinaryHeap(function (a, b) {
     return -comparator(a, b);
   }, initialData);
 };
 
-MaxMinHeap.prototype = Object.create(BinaryHeap);
+MaxMinHeap.prototype = Object.create(BinaryHeap.prototype);
 
 _.extend(MaxMinHeap.prototype, {
   set: function (id, value) {

From 4631f2437760af5265a092da5eebd871d74da871 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 18 Feb 2014 15:26:44 -0800
Subject: [PATCH 029/123] Fix _upHeap and fix oop once again

---
 packages/binary-heap/binary-heap.js  | 16 +++++++---------
 packages/binary-heap/max-min-heap.js | 10 +++++-----
 2 files changed, 12 insertions(+), 14 deletions(-)

diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/binary-heap.js
index 8158ad419d..cd35e3f1df 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/binary-heap.js
@@ -68,18 +68,16 @@ _.extend(BinaryHeap.prototype, {
   _upHeap: function (idx) {
     var self = this;
     var value = self._get(idx);
-    var ancestor = idx;
 
-    while (ancestor > 0) {
-      var parent = parentIdx(ancestor);
-      if (self._comparator(self._get(parent), value) < 0)
-        ancestor = parent;
-      else
+    while (idx > 0) {
+      var parent = parentIdx(idx);
+      if (self._comparator(self._get(parent), value) < 0) {
+        self._swap(parent, idx)
+        idx = parent;
+      } else {
         break;
+      }
     }
-
-    if (ancestor !== idx)
-      self._swap(ancestor, idx);
   },
 
   // Internal: gets raw data object placed on idxth place in heap
diff --git a/packages/binary-heap/max-min-heap.js b/packages/binary-heap/max-min-heap.js
index 4221d2f9df..4d7af28a03 100644
--- a/packages/binary-heap/max-min-heap.js
+++ b/packages/binary-heap/max-min-heap.js
@@ -12,23 +12,23 @@ MaxMinHeap.prototype = Object.create(BinaryHeap.prototype);
 _.extend(MaxMinHeap.prototype, {
   set: function (id, value) {
     var self = this;
-    self._minHeap.set(id, value);
     BinaryHeap.prototype.set.apply(self, arguments);
+    self._minHeap.set(id, value);
   },
   remove: function (id) {
     var self = this;
-    self._minHeap.remove(id);
     BinaryHeap.prototype.remove.apply(self, arguments);
+    self._minHeap.remove(id);
   },
   clear: function () {
     var self = this;
-    self._minHeap.clear(id);
     BinaryHeap.prototype.clear.apply(self, arguments);
+    self._minHeap.clear();
   },
   setDefault: function (id, def) {
     var self = this;
-    self._minHeap.setDefault(id, def);
-    return BinaryHeap.prototype.setDefault.apply(self, arguments);
+    BinaryHeap.prototype.setDefault.apply(self, arguments);
+    return self._minHeap.setDefault(id, def);
   },
   clone: function () {
     var self = this;

From 3871c92181f549eeb93df36b613c3f331e309f78 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 18 Feb 2014 16:37:31 -0800
Subject: [PATCH 030/123] Don't throw an updated-buffered doc right away. It
 can go to published or back to buffer.

---
 .../mongo-livedata/oplog_observe_driver.js    | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index ea0fa7fd5a..50fcd10564 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -305,13 +305,22 @@ _.extend(OplogObserveDriver.prototype, {
             self._justUpdatedBuffer = false;
         }
       } else if (bufferedBefore) {
-        // after the change we can't know if doc is still in the buffer limit
-        // w/o querying mongo, so just remove it from buffer
-        self._removeBuffered(id);
-        // but it can move into published now, check it
+        oldDoc = self._unpublishedBuffer.get(id);
+        // remove the old version manually so we don't trigger the querying
+        // immediately
+        self._unpublishedBuffer.remove(id);
+
         var maxPublished = self._published.get(self._published.maxElementId());
-        if (comparator(newDoc, maxPublished) < 0)
+        var maxBuffered = self._unpublishedBuffer.size() && self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
+        // the buffered doc was updated, it could move to published
+        if (comparator(newDoc, maxPublished) < 0) {
           self._addPublished(id, newDoc);
+        } else if (self._justUpdatedBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0)) {
+          // stays in buffer
+          self._unpublishedBuffer.set(id, newDoc);
+        } else {
+          self._justUpdatedBuffer = false;
+        }
       } else {
         throw new Error("cachedBefore implies either of publishedBefore or bufferedBefore is true.");
       }

From 6144213649c91bf176f7cfd23ca4344a7d9b5276 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 18 Feb 2014 16:38:03 -0800
Subject: [PATCH 031/123] Tests for updated-buffered which stays in buffer or
 comes to published

---
 .../mongo-livedata/mongo_livedata_tests.js    | 26 ++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 46abc8e78b..ed318b4a27 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -752,17 +752,21 @@ if (Meteor.isServer) {
     var run = test.runId();
     var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
 
+    var currentState = {};
     var observer = function () {
       var output = [];
       var callbacks = {
         changed: function (newDoc) {
           output.push({changed: newDoc._id});
+          currentState[newDoc._id] = newDoc;
         },
         added: function (newDoc) {
           output.push({added: newDoc._id});
+          currentState[newDoc._id] = newDoc;
         },
         removed: function (oldDoc) {
           output.push({removed: oldDoc._id});
+          delete currentState[oldDoc._id];
         }
       };
       var handle = coll.find({foo: 22},
@@ -773,7 +777,6 @@ if (Meteor.isServer) {
     var clearOutput = function (o) { o.output.splice(0, o.output.length); };
 
     var ins = function (doc) {
-      if (doc.bar) doc._id = doc.bar.toString();
       var id; runInFence(function () { id = coll.insert(doc); });
       return id;
     };
@@ -922,6 +925,27 @@ if (Meteor.isServer) {
     test.isTrue(setsEqual(o.output, [{removed: docId9}, {added: docId6}]));
     clearOutput(o);
 
+    upd({ bar: { $gt: 25 } }, { $inc: { bar: -7.5 } }, { multi: true });
+    // Becomes [22 23 23.5] 24 33.5] 43.5
+    test.length(o.output, 2);
+    test.isTrue(setsEqual(o.output, [{removed: docId6}, {added: docId10}]));
+    clearOutput(o);
+
+    // Force buffer objects to be moved into published set so we can check them
+    rem(docId7);
+    rem(docId8);
+    rem(docId10);
+    // Becomes [24 33.5 43.5]
+    test.length(o.output, 6);
+    test.isTrue(setsEqual(o.output, [{removed: docId7}, {removed: docId8},
+                                     {removed: docId10}, {added: docId6},
+                                     {added: docId11}, {added: docId12}]));
+
+    test.equal(currentState[docId6], { _id: docId6, foo: 22, bar: 24 });
+    test.equal(currentState[docId11], { _id: docId11, foo: 22, bar: 33.5 });
+    test.equal(currentState[docId12], { _id: docId12, foo: 22, bar: 43.5 });
+    clearOutput(o);
+
     o.handle.stop();
     onComplete();
   });

From 23ec5007e98638877d6ed44a3f9f5a368aa2f349 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 19 Feb 2014 12:44:19 -0800
Subject: [PATCH 032/123] _justUpdatedBuffer => _safeAppendToBuffer

---
 .../mongo-livedata/oplog_observe_driver.js    | 25 ++++++++++++-------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 50fcd10564..098cf10eed 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -49,7 +49,11 @@ OplogObserveDriver = function (options) {
     self._unpublishedBuffer = null;
     self._published = new LocalCollection._IdMap;
   }
-  self._justUpdatedBuffer = false;
+
+  // Indicates if it is safe to insert a new document at the end of the buffer
+  // for this query. i.e. it is known that there are no documents matching the
+  // selector those are not in published or buffer.
+  self._safeAppendToBuffer = false;
 
   self._stopped = false;
   self._stopHandles = [];
@@ -203,7 +207,7 @@ _.extend(OplogObserveDriver.prototype, {
       }
 
       self._unpublishedBuffer.remove(maxBufferedId);
-      self._justUpdatedBuffer = false;
+      self._safeAppendToBuffer = false;
     }
   },
   _removeBuffered: function (id) {
@@ -241,7 +245,7 @@ _.extend(OplogObserveDriver.prototype, {
     // outside of the buffer easily.
     if (!limit || self._published.size() < limit || comparator(maxPublished, fields) > 0) {
       self._addPublished(id, fields);
-    } else if ((self._justUpdatedBuffer && self._unpublishedBuffer.size() < limit) || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
+    } else if ((self._safeAppendToBuffer && self._unpublishedBuffer.size() < limit) || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
       self._addBuffered(id, fields);
     }
   },
@@ -299,10 +303,10 @@ _.extend(OplogObserveDriver.prototype, {
           self._removePublished(id);
           // but it can move into buffered now, check it
           var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
-          if (self._justUpdatedBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0))
+          if (self._safeAppendToBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0))
             self._addBuffered(id, newDoc);
           else
-            self._justUpdatedBuffer = false;
+            self._safeAppendToBuffer = false;
         }
       } else if (bufferedBefore) {
         oldDoc = self._unpublishedBuffer.get(id);
@@ -315,11 +319,11 @@ _.extend(OplogObserveDriver.prototype, {
         // the buffered doc was updated, it could move to published
         if (comparator(newDoc, maxPublished) < 0) {
           self._addPublished(id, newDoc);
-        } else if (self._justUpdatedBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0)) {
+        } else if (self._safeAppendToBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0)) {
           // stays in buffer
           self._unpublishedBuffer.set(id, newDoc);
         } else {
-          self._justUpdatedBuffer = false;
+          self._safeAppendToBuffer = false;
         }
       } else {
         throw new Error("cachedBefore implies either of publishedBefore or bufferedBefore is true.");
@@ -467,7 +471,9 @@ _.extend(OplogObserveDriver.prototype, {
     initialCursor.forEach(function (initialDoc) {
       self._addMatching(initialDoc);
     });
-    self._justUpdatedBuffer = true;
+
+    self._safeAppendToBuffer = initialCursor.count() < self._limit * 2;
+
     if (self._stopped)
       throw new Error("oplog stopped quite early");
     // Allow observeChanges calls to return. (After this, it's possible for
@@ -634,7 +640,8 @@ _.extend(OplogObserveDriver.prototype, {
       delete doc._id;
       self._addBuffered(id, doc);
     });
-    self._justUpdatedBuffer = true;
+
+    self._safeAppendToBuffer = newBuffer.size() < self._limit;
   },
 
   // This stop function is invoked from the onStop of the ObserveMultiplexer, so

From b76fd3c3af3eaad18ad2007971852a40aa915f1c Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 13:46:39 -0800
Subject: [PATCH 033/123] Rename BinaryHeap to MaxHeap and MaxMinHeap to
 MinMaxHeap

---
 packages/binary-heap/binary-heap-tests.js     |  8 ++++----
 .../{binary-heap.js => max-heap.js}           |  6 +++---
 .../{max-min-heap.js => min-max-heap.js}      | 20 +++++++++----------
 packages/binary-heap/package.js               |  6 +++---
 4 files changed, 20 insertions(+), 20 deletions(-)
 rename packages/binary-heap/{binary-heap.js => max-heap.js} (97%)
 rename packages/binary-heap/{max-min-heap.js => min-max-heap.js} (58%)

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index aa026e8148..86aabea08f 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -1,5 +1,5 @@
-Tinytest.add("binary-heap - simple heap tests", function (test) {
-  var h = new BinaryHeap(function (a, b) { return a-b; });
+Tinytest.add("binary-heap - simple max-heap tests", function (test) {
+  var h = new MaxHeap(function (a, b) { return a-b; });
   h.set("a", 1);
   h.set("b", 233);
   h.set("c", -122);
@@ -28,8 +28,8 @@ Tinytest.add("binary-heap - simple heap tests", function (test) {
   test.equal(h.maxElementId(), "a");
 });
 
-Tinytest.add("binary-heap - max-min heap tests", function (test) {
-  var h = new MaxMinHeap(function (a, b) { return a-b; });
+Tinytest.add("binary-heap - min-max heap tests", function (test) {
+  var h = new MinMaxHeap(function (a, b) { return a-b; });
   h.set("a", 1);
   h.set("b", 233);
   h.set("c", -122);
diff --git a/packages/binary-heap/binary-heap.js b/packages/binary-heap/max-heap.js
similarity index 97%
rename from packages/binary-heap/binary-heap.js
rename to packages/binary-heap/max-heap.js
index cd35e3f1df..c2904b70c9 100644
--- a/packages/binary-heap/binary-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -5,7 +5,7 @@
 //        - id - String - unique id of the item
 //        - value - Any - the data value
 //    the contents of initData is retained
-BinaryHeap = function (comparator, initData) {
+MaxHeap = function (comparator, initData) {
   if (! _.isFunction(comparator))
     throw new Error('Passed comparator is invalid, should be a comparison function');
   var self = this;
@@ -26,7 +26,7 @@ if (Package.minimongo) {
   idStringify = function (id) { return JSON.stringify(id); };
 }
 
-_.extend(BinaryHeap.prototype, {
+_.extend(MaxHeap.prototype, {
   _initFromData: function (data) {
     var self = this;
 
@@ -169,7 +169,7 @@ _.extend(BinaryHeap.prototype, {
   },
   clone: function () {
     var self = this;
-    var clone = new BinaryHeap(self._comparator);
+    var clone = new MaxHeap(self._comparator);
     clone._heap = EJSON.clone(self._heap);
     clone._heapIdx = EJSON.clone(self._heapIdx);
     return clone;
diff --git a/packages/binary-heap/max-min-heap.js b/packages/binary-heap/min-max-heap.js
similarity index 58%
rename from packages/binary-heap/max-min-heap.js
rename to packages/binary-heap/min-max-heap.js
index 4d7af28a03..9503162261 100644
--- a/packages/binary-heap/max-min-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -1,38 +1,38 @@
-MaxMinHeap = function (comparator, initialData) {
+MinMaxHeap = function (comparator, initialData) {
   var self = this;
 
-  BinaryHeap.call(self, comparator, initialData);
-  self._minHeap = new BinaryHeap(function (a, b) {
+  MaxHeap.call(self, comparator, initialData);
+  self._minHeap = new MaxHeap(function (a, b) {
     return -comparator(a, b);
   }, initialData);
 };
 
-MaxMinHeap.prototype = Object.create(BinaryHeap.prototype);
+MinMaxHeap.prototype = Object.create(MaxHeap.prototype);
 
-_.extend(MaxMinHeap.prototype, {
+_.extend(MinMaxHeap.prototype, {
   set: function (id, value) {
     var self = this;
-    BinaryHeap.prototype.set.apply(self, arguments);
+    MaxHeap.prototype.set.apply(self, arguments);
     self._minHeap.set(id, value);
   },
   remove: function (id) {
     var self = this;
-    BinaryHeap.prototype.remove.apply(self, arguments);
+    MaxHeap.prototype.remove.apply(self, arguments);
     self._minHeap.remove(id);
   },
   clear: function () {
     var self = this;
-    BinaryHeap.prototype.clear.apply(self, arguments);
+    MaxHeap.prototype.clear.apply(self, arguments);
     self._minHeap.clear();
   },
   setDefault: function (id, def) {
     var self = this;
-    BinaryHeap.prototype.setDefault.apply(self, arguments);
+    MaxHeap.prototype.setDefault.apply(self, arguments);
     return self._minHeap.setDefault(id, def);
   },
   clone: function () {
     var self = this;
-    var clone = new MaxMinHeap(self._comparator);
+    var clone = new MinMaxHeap(self._comparator);
     clone._heap = EJSON.clone(self._heap);
     clone._heapIdx = EJSON.clone(self._heapIdx);
     clone._minHeap = self._minHeap.clone();
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
index 9d1504eae3..a340d7c8c5 100644
--- a/packages/binary-heap/package.js
+++ b/packages/binary-heap/package.js
@@ -4,11 +4,11 @@ Package.describe({
 });
 
 Package.on_use(function (api) {
-  api.export('BinaryHeap');
-  api.export('MaxMinHeap');
+  api.export('MaxHeap');
+  api.export('MinMaxHeap');
   api.use(['underscore']);
   api.use(['minimongo'], { weak: true });
-  api.add_files(['binary-heap.js', 'max-min-heap.js']);
+  api.add_files(['max-heap.js', 'min-max-heap.js']);
 });
 
 Package.on_test(function (api) {

From b4e94c3b2a5095fbee9f1e22a5b8dc56ad33d780 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 14:04:27 -0800
Subject: [PATCH 034/123] A randomized test for max-heap

---
 packages/binary-heap/binary-heap-tests.js | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index 86aabea08f..9a709b6fe4 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -28,6 +28,27 @@ Tinytest.add("binary-heap - simple max-heap tests", function (test) {
   test.equal(h.maxElementId(), "a");
 });
 
+Tinytest.add("binary-heap - big test for max-heap", function (test) {
+  var positiveNumbers = _.shuffle(_.range(1, 41));
+  var negativeNumbers = _.shuffle(_.range(-1, -41, -1));
+  var allNumbers = negativeNumbers.concat(positiveNumbers);
+
+  var heap = new MaxHeap(function (a, b) { return a-b; });
+  var output = [];
+
+  _.each(allNumbers, function (n) { heap.set(n, n); });
+
+  _.times(positiveNumbers.length + negativeNumbers.length, function () {
+    var maxId = heap.maxElementId();
+    output.push(heap.get(maxId));
+    heap.remove(maxId);
+  });
+
+  allNumbers.sort(function (a, b) { return b-a; });
+
+  test.equal(output, allNumbers);
+});
+
 Tinytest.add("binary-heap - min-max heap tests", function (test) {
   var h = new MinMaxHeap(function (a, b) { return a-b; });
   h.set("a", 1);

From 3fa5fd9342f3e5b81c270ed10a7bfc0d854ce6d5 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 14:05:12 -0800
Subject: [PATCH 035/123] binary-heap: Fix rightChildIdx()

---
 packages/binary-heap/max-heap.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index c2904b70c9..cd614ebe2c 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -182,6 +182,6 @@ _.extend(MaxHeap.prototype, {
 });
 
 function leftChildIdx (i) { return i * 2 + 1; }
-function rightChildIdx (i) { return i * i + 2; }
+function rightChildIdx (i) { return i * 2 + 2; }
 function parentIdx (i) { return (i - 1) >> 1; }
 

From 8c079efcdac835aeda67175a22c9c82dc46195dc Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 16:00:08 -0800
Subject: [PATCH 036/123] Extracted IdMap to a separate package

---
 packages/id-map/.gitignore    |  1 +
 packages/id-map/id-map.js     | 77 +++++++++++++++++++++++++++++++++++
 packages/id-map/package.js    | 11 +++++
 packages/minimongo/id_map.js  | 71 +-------------------------------
 packages/minimongo/package.js |  2 +-
 5 files changed, 92 insertions(+), 70 deletions(-)
 create mode 100644 packages/id-map/.gitignore
 create mode 100644 packages/id-map/id-map.js
 create mode 100644 packages/id-map/package.js

diff --git a/packages/id-map/.gitignore b/packages/id-map/.gitignore
new file mode 100644
index 0000000000..677a6fc263
--- /dev/null
+++ b/packages/id-map/.gitignore
@@ -0,0 +1 @@
+.build*
diff --git a/packages/id-map/id-map.js b/packages/id-map/id-map.js
new file mode 100644
index 0000000000..888fdee63d
--- /dev/null
+++ b/packages/id-map/id-map.js
@@ -0,0 +1,77 @@
+IdMap = function (idStringify, idParse) {
+  var self = this;
+  self._map = {};
+  self._idStringify = idStringify || JSON.stringify;
+  self._idParse = idParse || JSON.parse;
+};
+
+// Some of these methods are designed to match methods on OrderedDict, since
+// (eg) ObserveMultiplex and _CachingChangeObserver use them interchangeably.
+// (Conceivably, this should be replaced with "UnorderedDict" with a specific
+// set of methods that overlap between the two.)
+
+_.extend(IdMap.prototype, {
+  get: function (id) {
+    var self = this;
+    var key = self._idStringify(id);
+    return self._map[key];
+  },
+  set: function (id, value) {
+    var self = this;
+    var key = self._idStringify(id);
+    self._map[key] = value;
+  },
+  remove: function (id) {
+    var self = this;
+    var key = self._idStringify(id);
+    delete self._map[key];
+  },
+  has: function (id) {
+    var self = this;
+    var key = self._idStringify(id);
+    return _.has(self._map, key);
+  },
+  empty: function () {
+    var self = this;
+    return _.isEmpty(self._map);
+  },
+  clear: function () {
+    var self = this;
+    self._map = {};
+  },
+  // Iterates over the items in the map. Return `false` to break the loop.
+  forEach: function (iterator) {
+    var self = this;
+    // don't use _.each, because we can't break out of it.
+    var keys = _.keys(self._map);
+    for (var i = 0; i < keys.length; i++) {
+      var breakIfFalse = iterator.call(null, self._map[keys[i]],
+                                       self._idParse(keys[i]));
+      if (breakIfFalse === false)
+        return;
+    }
+  },
+  size: function () {
+    var self = this;
+    return _.size(self._map);
+  },
+  setDefault: function (id, def) {
+    var self = this;
+    var key = self._idStringify(id);
+    if (_.has(self._map, key))
+      return self._map[key];
+    self._map[key] = def;
+    return def;
+  },
+  // Assumes that values are EJSON-cloneable, and that we don't need to clone
+  // IDs (ie, that nobody is going to mutate an ObjectId).
+  clone: function () {
+    var self = this;
+    var clone = new IdMap(self._idStringify, self._idParse);
+    self.forEach(function (value, id) {
+      clone.set(id, EJSON.clone(value));
+    });
+    return clone;
+  }
+});
+
diff --git a/packages/id-map/package.js b/packages/id-map/package.js
new file mode 100644
index 0000000000..d3ef1db346
--- /dev/null
+++ b/packages/id-map/package.js
@@ -0,0 +1,11 @@
+Package.describe({
+  summary: "Dictionary data structure: a wrapper for a raw object",
+  internal: true
+});
+
+Package.on_use(function (api) {
+  api.export('IdMap');
+  api.use(['underscore', 'json', 'ejson']);
+  api.add_files([ 'id-map.js' ]);
+});
+
diff --git a/packages/minimongo/id_map.js b/packages/minimongo/id_map.js
index 4e94273fac..7fc10d8add 100644
--- a/packages/minimongo/id_map.js
+++ b/packages/minimongo/id_map.js
@@ -1,74 +1,7 @@
 LocalCollection._IdMap = function () {
   var self = this;
-  self._map = {};
+  IdMap.call(self, LocalCollection._idStringify, LocalCollection._idParse);
 };
 
-// Some of these methods are designed to match methods on OrderedDict, since
-// (eg) ObserveMultiplex and _CachingChangeObserver use them interchangeably.
-// (Conceivably, this should be replaced with "UnorderedDict" with a specific
-// set of methods that overlap between the two.)
+LocalCollection._IdMap.prototype = Object.create(IdMap.prototype);
 
-_.extend(LocalCollection._IdMap.prototype, {
-  get: function (id) {
-    var self = this;
-    var key = LocalCollection._idStringify(id);
-    return self._map[key];
-  },
-  set: function (id, value) {
-    var self = this;
-    var key = LocalCollection._idStringify(id);
-    self._map[key] = value;
-  },
-  remove: function (id) {
-    var self = this;
-    var key = LocalCollection._idStringify(id);
-    delete self._map[key];
-  },
-  has: function (id) {
-    var self = this;
-    var key = LocalCollection._idStringify(id);
-    return _.has(self._map, key);
-  },
-  empty: function () {
-    var self = this;
-    return _.isEmpty(self._map);
-  },
-  clear: function () {
-    var self = this;
-    self._map = {};
-  },
-  // Iterates over the items in the map. Return `false` to break the loop.
-  forEach: function (iterator) {
-    var self = this;
-    // don't use _.each, because we can't break out of it.
-    var keys = _.keys(self._map);
-    for (var i = 0; i < keys.length; i++) {
-      var breakIfFalse = iterator.call(null, self._map[keys[i]],
-                                       LocalCollection._idParse(keys[i]));
-      if (breakIfFalse === false)
-        return;
-    }
-  },
-  size: function () {
-    var self = this;
-    return _.size(self._map);
-  },
-  setDefault: function (id, def) {
-    var self = this;
-    var key = LocalCollection._idStringify(id);
-    if (_.has(self._map, key))
-      return self._map[key];
-    self._map[key] = def;
-    return def;
-  },
-  // Assumes that values are EJSON-cloneable, and that we don't need to clone
-  // IDs (ie, that nobody is going to mutate an ObjectId).
-  clone: function () {
-    var self = this;
-    var clone = new LocalCollection._IdMap;
-    self.forEach(function (value, id) {
-      clone.set(id, EJSON.clone(value));
-    });
-    return clone;
-  }
-});
diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js
index 8791cbfe77..58783070fb 100644
--- a/packages/minimongo/package.js
+++ b/packages/minimongo/package.js
@@ -7,7 +7,7 @@ Package.on_use(function (api) {
   api.export('LocalCollection');
   api.export('Minimongo');
   api.export('MinimongoTest', { testOnly: true });
-  api.use(['underscore', 'json', 'ejson', 'ordered-dict', 'deps',
+  api.use(['underscore', 'json', 'ejson', 'id-map', 'ordered-dict', 'deps',
            'random', 'ordered-dict']);
   // This package is used for geo-location queries such as $near
   api.use('geojson-utils');

From 939d8d7e7f8f10bfbaba3f4c5758aba526ab82dd Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 16:29:01 -0800
Subject: [PATCH 037/123] es3 friendly inheritance

---
 packages/binary-heap/min-max-heap.js | 4 +++-
 packages/minimongo/id_map.js         | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/binary-heap/min-max-heap.js b/packages/binary-heap/min-max-heap.js
index 9503162261..332dc33f07 100644
--- a/packages/binary-heap/min-max-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -7,7 +7,9 @@ MinMaxHeap = function (comparator, initialData) {
   }, initialData);
 };
 
-MinMaxHeap.prototype = Object.create(MaxHeap.prototype);
+var F = function () {};
+F.prototype = MaxHeap.prototype;
+MinMaxHeap.prototype = new F;
 
 _.extend(MinMaxHeap.prototype, {
   set: function (id, value) {
diff --git a/packages/minimongo/id_map.js b/packages/minimongo/id_map.js
index 7fc10d8add..cf098dbee1 100644
--- a/packages/minimongo/id_map.js
+++ b/packages/minimongo/id_map.js
@@ -3,5 +3,5 @@ LocalCollection._IdMap = function () {
   IdMap.call(self, LocalCollection._idStringify, LocalCollection._idParse);
 };
 
-LocalCollection._IdMap.prototype = Object.create(IdMap.prototype);
+LocalCollection._IdMap.prototype = IdMap.prototype;
 

From cd918bbf17f34e29c8044d9ada4708a64def419c Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 17:13:52 -0800
Subject: [PATCH 038/123] binary-heap uses id-map

---
 packages/binary-heap/max-heap.js | 37 ++++++++++++++------------------
 packages/binary-heap/package.js  |  2 +-
 2 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index cd614ebe2c..5eb05747ab 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -10,22 +10,18 @@ MaxHeap = function (comparator, initData) {
     throw new Error('Passed comparator is invalid, should be a comparison function');
   var self = this;
   self._comparator = comparator;
-  self._heapIdx = {};
+
+  if (Package && Package.minimongo)
+    self._heapIdx = new Package.minimongo.LocalCollection._IdMap;
+  else
+    self._heapIdx = new IdMap;
+
   self._heap = [];
 
   if (_.isArray(initData))
     self._initFromData(initData);
 };
 
-var idStringify;
-if (Package.minimongo) {
-  idStringify = Package.minimongo.LocalCollection._idStringify;
-} else {
-  // XXX: These can't deal with special strings like '__proto__'
-  // XXX: or '{ looksLike: "object" }' or numbers.
-  idStringify = function (id) { return JSON.stringify(id); };
-}
-
 _.extend(MaxHeap.prototype, {
   _initFromData: function (data) {
     var self = this;
@@ -33,7 +29,7 @@ _.extend(MaxHeap.prototype, {
     self._heap = _.clone(data);
 
     _.each(data, function (o, i) {
-      self._heapIdx[idStringify(o.id)] = i;
+      self._heapIdx.set(o.id, i);
     });
 
     for (var i = parentIdx(data.length); i >= 0; i--)
@@ -91,8 +87,8 @@ _.extend(MaxHeap.prototype, {
     var A = self._heap[idxA];
     var B = self._heap[idxB];
 
-    self._heapIdx[idStringify(A.id)] = idxB;
-    self._heapIdx[idStringify(B.id)] = idxA;
+    self._heapIdx.set(A.id, idxB);
+    self._heapIdx.set(B.id, idxA);
 
     self._heap[idxA] = B;
     self._heap[idxB] = A;
@@ -102,7 +98,7 @@ _.extend(MaxHeap.prototype, {
     var self = this;
     if (! self.has(id))
       return null;
-    return self._get(self._heapIdx[idStringify(id)]);
+    return self._get(self._heapIdx.get(id));
   },
   set: function (id, value) {
     var self = this;
@@ -114,17 +110,16 @@ _.extend(MaxHeap.prototype, {
         self.remove(id);
     }
 
-    self._heapIdx[idStringify(id)] = self._heap.length;
+    self._heapIdx.set(id, self._heap.length);
     self._heap.push({ id: id, value: value });
     self._upHeap(self._heap.length - 1);
   },
   remove: function (id) {
     var self = this;
-    var strId = idStringify(id);
 
     if (self.has(id)) {
       var last = self._heap.length - 1;
-      var idx = self._heapIdx[strId];
+      var idx = self._heapIdx.get(id);
 
       if (idx !== last) {
         self._swap(idx, last);
@@ -134,12 +129,12 @@ _.extend(MaxHeap.prototype, {
         self._heap.pop();
       }
 
-      delete self._heapIdx[strId];
+      self._heapIdx.remove(id);
     }
   },
   has: function (id) {
     var self = this;
-    return self._heapIdx[idStringify(id)] !== undefined;
+    return self._heapIdx.has(id);
   },
   empty: function (id) {
     var self = this;
@@ -148,7 +143,7 @@ _.extend(MaxHeap.prototype, {
   clear: function () {
     var self = this;
     self._heap = [];
-    self._heapIdx = {};
+    self._heapIdx.clear();
   },
   forEach: function (iterator) {
     var self = this;
@@ -171,7 +166,7 @@ _.extend(MaxHeap.prototype, {
     var self = this;
     var clone = new MaxHeap(self._comparator);
     clone._heap = EJSON.clone(self._heap);
-    clone._heapIdx = EJSON.clone(self._heapIdx);
+    clone._heapIdx = self._heapIdx.clone();
     return clone;
   },
 
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
index a340d7c8c5..5788fc1973 100644
--- a/packages/binary-heap/package.js
+++ b/packages/binary-heap/package.js
@@ -6,7 +6,7 @@ Package.describe({
 Package.on_use(function (api) {
   api.export('MaxHeap');
   api.export('MinMaxHeap');
-  api.use(['underscore']);
+  api.use(['underscore', 'id-map']);
   api.use(['minimongo'], { weak: true });
   api.add_files(['max-heap.js', 'min-max-heap.js']);
 });

From 006e307f1542b97c254fac7e0b4e8e7fa410f453 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 21:38:09 -0800
Subject: [PATCH 039/123] More comments on Heap implementation, linear
 initialization doesn't retain anything but the references to the values.

---
 packages/binary-heap/max-heap.js | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 5eb05747ab..6327e58e3d 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -9,24 +9,40 @@ MaxHeap = function (comparator, initData) {
   if (! _.isFunction(comparator))
     throw new Error('Passed comparator is invalid, should be a comparison function');
   var self = this;
+
+  // a C-style comparator that is given two values and returns a number,
+  // negative if the first value is less than the second, positive if the second
+  // value is greater than the first and zero if they are equal.
   self._comparator = comparator;
 
+  // _heapIdx maps an id to an index in the Heap array the corresponding value
+  // is located on.
   if (Package && Package.minimongo)
     self._heapIdx = new Package.minimongo.LocalCollection._IdMap;
   else
     self._heapIdx = new IdMap;
 
+  // The Heap data-structure implemented as a 0-based contiguous array where
+  // every item on index idx is a node in a complete binary tree. Every node can
+  // have leaves on indexes idx*2+1 and idx*2+2, except for the lists. Every
+  // node has a parent on index (idx-1)/2;
   self._heap = [];
 
+  // If the initial array is passed, we can build the heap in linear time
+  // complexity (O(N)) compared to linearithmetic time complexity (O(nlogn)) if
+  // we push elements one by one.
   if (_.isArray(initData))
     self._initFromData(initData);
 };
 
 _.extend(MaxHeap.prototype, {
+  // Builds a new heap in-place in linear time based on passed data
   _initFromData: function (data) {
     var self = this;
 
-    self._heap = _.clone(data);
+    self._heap = _.map(data, function (o) {
+      return { id: EJSON.clone(o.id), value: o.value };
+    });
 
     _.each(data, function (o, i) {
       self._heapIdx.set(o.id, i);

From 19eeff650a29369fed01e8f036fb2d67467ff56a Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 21:45:28 -0800
Subject: [PATCH 040/123] Implement heap clone with passing an array of the
 original's data

---
 packages/binary-heap/max-heap.js | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 6327e58e3d..37fc90c941 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -180,9 +180,7 @@ _.extend(MaxHeap.prototype, {
   },
   clone: function () {
     var self = this;
-    var clone = new MaxHeap(self._comparator);
-    clone._heap = EJSON.clone(self._heap);
-    clone._heapIdx = self._heapIdx.clone();
+    var clone = new MaxHeap(self._comparator, self._heap);
     return clone;
   },
 

From d87b18a56e23cf1d9ac8fbdbdeb78db5dbe1d8bb Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 22:38:56 -0800
Subject: [PATCH 041/123] Fix building heap from array

---
 packages/binary-heap/max-heap.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 37fc90c941..349dde8030 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -48,7 +48,8 @@ _.extend(MaxHeap.prototype, {
       self._heapIdx.set(o.id, i);
     });
 
-    for (var i = parentIdx(data.length); i >= 0; i--)
+    // start from the first non-leaf - the parent of the last leaf
+    for (var i = parentIdx(data.length - 1); i >= 0; i--)
       self._downHeap(i);
   },
 

From 7bad26e79613c5d912ed086d32e30bb53f3cfc59 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 14:25:06 -0800
Subject: [PATCH 042/123] Make min-max-heap's clone work in the same way it
 works for max-heap

---
 packages/binary-heap/min-max-heap.js | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/packages/binary-heap/min-max-heap.js b/packages/binary-heap/min-max-heap.js
index 332dc33f07..957d6014f4 100644
--- a/packages/binary-heap/min-max-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -34,10 +34,7 @@ _.extend(MinMaxHeap.prototype, {
   },
   clone: function () {
     var self = this;
-    var clone = new MinMaxHeap(self._comparator);
-    clone._heap = EJSON.clone(self._heap);
-    clone._heapIdx = EJSON.clone(self._heapIdx);
-    clone._minHeap = self._minHeap.clone();
+    var clone = new MinMaxHeap(self._comparator, self._heap);
     return clone;
   },
   minElementId: function () {

From 503c5f263f07d4fa8598234bba6022ff4565ef37 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 14:27:20 -0800
Subject: [PATCH 043/123] Extra safety belt for binary-heap constructor

---
 packages/binary-heap/max-heap.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 349dde8030..ce74539d1a 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -48,6 +48,9 @@ _.extend(MaxHeap.prototype, {
       self._heapIdx.set(o.id, i);
     });
 
+    if (! data.length)
+      return;
+
     // start from the first non-leaf - the parent of the last leaf
     for (var i = parentIdx(data.length - 1); i >= 0; i--)
       self._downHeap(i);

From e853ced77c5f42eeba06149b3525ddc0a54a346d Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 14:35:01 -0800
Subject: [PATCH 044/123] Binary-heap: refactor out the _maxIndex method

---
 packages/binary-heap/max-heap.js | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index ce74539d1a..e8a2b0445a 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -64,12 +64,10 @@ _.extend(MaxHeap.prototype, {
       var right = rightChildIdx(idx);
       var largest = idx;
 
-      if (left < self.size() &&
-          self._comparator(self._get(left), self._get(largest)) > 0) {
+      if (left < self.size() && self._maxIndex(largest, left) === left) {
         largest = left;
       }
-      if (right < self.size() &&
-          self._comparator(self._get(right), self._get(largest)) > 0) {
+      if (right < self.size() && self._maxIndex(largest, right) === right) {
         largest = right;
       }
 
@@ -83,11 +81,10 @@ _.extend(MaxHeap.prototype, {
 
   _upHeap: function (idx) {
     var self = this;
-    var value = self._get(idx);
 
     while (idx > 0) {
       var parent = parentIdx(idx);
-      if (self._comparator(self._get(parent), value) < 0) {
+      if (self._maxIndex(parent, idx) === idx) {
         self._swap(parent, idx)
         idx = parent;
       } else {
@@ -96,6 +93,13 @@ _.extend(MaxHeap.prototype, {
     }
   },
 
+  _maxIndex: function (idxA, idxB) {
+    var self = this;
+    var valueA = self._get(idxA);
+    var valueB = self._get(idxB);
+    return self._comparator(valueA, valueB) > 0 ? idxA : idxB;
+  },
+
   // Internal: gets raw data object placed on idxth place in heap
   _get: function (idx) {
     var self = this;

From 74553efa304f32bed43f8b5cbbee0116640e2c36 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 14:35:55 -0800
Subject: [PATCH 045/123] Binary-heap: avoid capitalized names for non-classes

---
 packages/binary-heap/max-heap.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index e8a2b0445a..891e9b3771 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -108,14 +108,14 @@ _.extend(MaxHeap.prototype, {
 
   _swap: function (idxA, idxB) {
     var self = this;
-    var A = self._heap[idxA];
-    var B = self._heap[idxB];
+    var recA = self._heap[idxA];
+    var recB = self._heap[idxB];
 
-    self._heapIdx.set(A.id, idxB);
-    self._heapIdx.set(B.id, idxA);
+    self._heapIdx.set(recA.id, idxB);
+    self._heapIdx.set(recB.id, idxA);
 
-    self._heap[idxA] = B;
-    self._heap[idxB] = A;
+    self._heap[idxA] = recB;
+    self._heap[idxB] = recA;
   },
 
   get: function (id) {

From b01ed021eaee519c035b707cfdfb77501e32a7bb Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 17:48:22 -0800
Subject: [PATCH 046/123] New giant randomized test with self-checks

---
 packages/binary-heap/binary-heap-tests.js | 34 +++++++++++++++++++++++
 packages/binary-heap/max-heap.js          |  9 ++++++
 2 files changed, 43 insertions(+)

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index 9a709b6fe4..496af9c0c3 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -81,3 +81,37 @@ Tinytest.add("binary-heap - min-max heap tests", function (test) {
   test.equal(h.minElementId(), "a");
 });
 
+Tinytest.add("binary-heap - big test for min-max-heap", function (test) {
+  var N = 500;
+  var positiveNumbers = _.shuffle(_.range(1, N + 1));
+  var negativeNumbers = _.shuffle(_.range(-1, -N - 1, -1));
+  var allNumbers = positiveNumbers.concat(negativeNumbers);
+
+  var heap = new MinMaxHeap(function (a, b) { return a-b; });
+  var output = [];
+
+  _.each(allNumbers, function (n) {
+    heap.set(n, n);
+    heap._selfCheck();
+    heap._minHeap._selfCheck();
+  });
+
+  allNumbers = _.shuffle(allNumbers);
+  _.each(allNumbers, function (n) {
+    heap.set(-n, n);
+    heap._selfCheck();
+    heap._minHeap._selfCheck();
+  });
+
+  _.times(positiveNumbers.length + negativeNumbers.length, function () {
+    var minId = heap.minElementId();
+    output.push(heap.get(minId));
+    heap.remove(minId);
+    heap._selfCheck(); heap._minHeap._selfCheck();
+  });
+
+  allNumbers.sort(function (a, b) { return a-b; });
+
+  test.equal(output, allNumbers);
+});
+
diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 891e9b3771..2d70e3509d 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -195,6 +195,15 @@ _.extend(MaxHeap.prototype, {
   maxElementId: function () {
     var self = this;
     return self.size() ? self._heap[0].id : null;
+  },
+
+  _selfCheck: function () {
+    var self = this;
+    for (var i = 1; i < self._heap.length; i++)
+      if (self._maxIndex(parentIdx(i), i) !== parentIdx(i))
+          throw new Error("An item with id " + self._heap[i].id +
+                          " has a parent younger than him: " +
+                          self._heap[parentIdx(i)].id);
   }
 });
 

From 133428e6e513ff358b41a2a8e32e1aed8448c087 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 17:48:37 -0800
Subject: [PATCH 047/123] Fix the removal and the priority of _maxIndex.

Removing an arbitary element you might need to upHeap the position of removed
element instead of downHeap. It is not the case with the maximum, because root
can't be upHeaped.
---
 packages/binary-heap/max-heap.js | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 2d70e3509d..7215e92d5f 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -97,7 +97,7 @@ _.extend(MaxHeap.prototype, {
     var self = this;
     var valueA = self._get(idxA);
     var valueB = self._get(idxB);
-    return self._comparator(valueA, valueB) > 0 ? idxA : idxB;
+    return self._comparator(valueA, valueB) >= 0 ? idxA : idxB;
   },
 
   // Internal: gets raw data object placed on idxth place in heap
@@ -130,8 +130,9 @@ _.extend(MaxHeap.prototype, {
     if (self.has(id)) {
       if (self.get(id) === value)
         return;
-      else
+      else {
         self.remove(id);
+      }
     }
 
     self._heapIdx.set(id, self._heap.length);
@@ -149,6 +150,7 @@ _.extend(MaxHeap.prototype, {
         self._swap(idx, last);
         self._heap.pop();
         self._downHeap(idx);
+        self._upHeap(idx);
       } else {
         self._heap.pop();
       }

From 1dc172696f10fb99f51f292b0da6ca645a7e4aed Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 18:12:35 -0800
Subject: [PATCH 048/123] minor stylish tweak

---
 packages/binary-heap/max-heap.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 7215e92d5f..137d7271b4 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -149,13 +149,13 @@ _.extend(MaxHeap.prototype, {
       if (idx !== last) {
         self._swap(idx, last);
         self._heap.pop();
+        self._heapIdx.remove(id);
         self._downHeap(idx);
         self._upHeap(idx);
       } else {
         self._heap.pop();
+        self._heapIdx.remove(id);
       }
-
-      self._heapIdx.remove(id);
     }
   },
   has: function (id) {

From f38294b17301be7b7c3f0da0498cf6ea62f107dd Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 18:19:13 -0800
Subject: [PATCH 049/123] Binary-Heap: update in-place

---
 packages/binary-heap/max-heap.js | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index 137d7271b4..ccc4ff3f36 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -130,14 +130,16 @@ _.extend(MaxHeap.prototype, {
     if (self.has(id)) {
       if (self.get(id) === value)
         return;
-      else {
-        self.remove(id);
-      }
-    }
 
-    self._heapIdx.set(id, self._heap.length);
-    self._heap.push({ id: id, value: value });
-    self._upHeap(self._heap.length - 1);
+      var idx = self._heapIdx.get(id);
+      self._heap[idx].value = value;
+      self._upHeap(idx);
+      self._downHeap(idx);
+    } else {
+      self._heapIdx.set(id, self._heap.length);
+      self._heap.push({ id: id, value: value });
+      self._upHeap(self._heap.length - 1);
+    }
   },
   remove: function (id) {
     var self = this;

From a6410c4af750c1027756c1822c515de0bdccf8e7 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 18:23:58 -0800
Subject: [PATCH 050/123] A comment on Min-Max-Heap implementation

---
 packages/binary-heap/min-max-heap.js | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/packages/binary-heap/min-max-heap.js b/packages/binary-heap/min-max-heap.js
index 957d6014f4..b491c61bb1 100644
--- a/packages/binary-heap/min-max-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -1,3 +1,15 @@
+// This implementation of Min/Max-Heap is just a subclass of Max-Heap
+// with a Min-Heap as an encapsulated property.
+//
+// Most of the operations are just proxy methods to call the same method on both
+// heaps.
+//
+// This implementation takes 2*N memory but is fairly simple to write and
+// understand. And the constant factor of a simple Heap is usually smaller
+// compared to other two-way priority queues like Min/Max Heaps
+// (http://www.cs.otago.ac.nz/staffpriv/mike/Papers/MinMaxHeaps/MinMaxHeaps.pdf)
+// and Interval Heaps
+// (http://www.cise.ufl.edu/~sahni/dsaac/enrich/c13/double.htm)
 MinMaxHeap = function (comparator, initialData) {
   var self = this;
 

From 9ccdc0b80c6ddc6526f7ce3bad09c5c67dccd6dc Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 19:47:23 -0800
Subject: [PATCH 051/123] Don't repoll if everything fits into buffer; Use
 EJSON.equals Better comments

---
 .../mongo-livedata/oplog_observe_driver.js    | 21 +++++++++++--------
 1 file changed, 12 insertions(+), 9 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 098cf10eed..50f33aa679 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -28,11 +28,11 @@ OplogObserveDriver = function (options) {
     // There are several properties ordered driver implements:
     // - _limit is a positive number
     // - _comparator is a function-comparator by which the query is ordered
-    // - _unpublishedBuffer is non-null collection,
+    // - _unpublishedBuffer is non-null Min/Max Heap,
     //                      the empty buffer in STEADY phase implies that the
     //                      everything that matches the queries selector fits
     //                      into published set.
-    // - _published implements maxElementId method in addition to IdMap methods
+    // - _published - Min Heap (also implements IdMap methods)
 
     // We don't support $near and other geo-queries so it's OK to initialize the
     // comparator only once in the constructor.
@@ -159,7 +159,7 @@ _.extend(OplogObserveDriver.prototype, {
       var overflowingDocId = self._published.maxElementId();
       var overflowingDoc = self._published.get(overflowingDocId);
 
-      if (_.isEqual(overflowingDocId, id)) {
+      if (EJSON.equals(overflowingDocId, id)) {
         throw new Error("The document just added is overflowing the published set");
       }
 
@@ -202,7 +202,7 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._unpublishedBuffer.size() > self._limit) {
       var maxBufferedId = self._unpublishedBuffer.maxElementId();
 
-      if (_.isEqual(maxBufferedId, id)) {
+      if (EJSON.equals(maxBufferedId, id)) {
         throw new Error("The document just added to buffer is overflowing the buffer");
       }
 
@@ -216,7 +216,7 @@ _.extend(OplogObserveDriver.prototype, {
     // To keep the contract "buffer is never empty in STEADY phase unless the
     // everything matching fits into published" true, we poll everything as soon
     // as we see the buffer becoming empty.
-    if (! self._unpublishedBuffer.size())
+    if (! self._unpublishedBuffer.size() && ! self._safeAppendToBuffer)
       self._needToPollQuery();
   },
   // Called when a document has joined the "Matching" results set.
@@ -468,11 +468,13 @@ _.extend(OplogObserveDriver.prototype, {
     // XXX needs more thought on non-zero skip
     // XXX "2" here is a "magic number"
     var initialCursor = self._cursorForQuery({ limit: self._limit * 2 });
+    var fetchedDocsCount = 0;
     initialCursor.forEach(function (initialDoc) {
       self._addMatching(initialDoc);
+      fetchedDocsCount++;
     });
 
-    self._safeAppendToBuffer = initialCursor.count() < self._limit * 2;
+    self._safeAppendToBuffer = fetchedDocsCount < self._limit * 2;
 
     if (self._stopped)
       throw new Error("oplog stopped quite early");
@@ -704,9 +706,10 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   if (options._disableOplog)
     return false;
 
-  // This option (which are mostly used for sorted cursors) require us to figure
-  // out where a given document fits in an order to know if it's included or
-  // not. We do it only if skip is not defined or 0.
+  // skip is not supported: to support it we would need to keep track of all
+  // "skipped" documents or at least their ids.
+  // limit w/o a sort specifier is not supported: current implementation needs a
+  // determent way to order documents.
   if (options.skip || (options.limit && !options.sort)) return false;
 
   // If a fields projection option is given check if it is supported by

From 622d61f0a7b977b6a6b09d6be6e07f4213d4ad66 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 21 Feb 2014 20:03:22 -0800
Subject: [PATCH 052/123] fix tests' setsEqual

---
 .../mongo-livedata/mongo_livedata_tests.js     | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index ed318b4a27..9f0665c11e 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -752,27 +752,27 @@ if (Meteor.isServer) {
     var run = test.runId();
     var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
 
-    var currentState = {};
     var observer = function () {
+      var state = {};
       var output = [];
       var callbacks = {
         changed: function (newDoc) {
           output.push({changed: newDoc._id});
-          currentState[newDoc._id] = newDoc;
+          state[newDoc._id] = newDoc;
         },
         added: function (newDoc) {
           output.push({added: newDoc._id});
-          currentState[newDoc._id] = newDoc;
+          state[newDoc._id] = newDoc;
         },
         removed: function (oldDoc) {
           output.push({removed: oldDoc._id});
-          delete currentState[oldDoc._id];
+          delete state[oldDoc._id];
         }
       };
       var handle = coll.find({foo: 22},
                              {sort: {bar: 1}, limit: 3}).observe(callbacks);
 
-      return {output: output, handle: handle};
+      return {output: output, handle: handle, state: state};
     };
     var clearOutput = function (o) { o.output.splice(0, o.output.length); };
 
@@ -790,7 +790,7 @@ if (Meteor.isServer) {
     var setsEqual = function (a, b) {
       a = _.map(a, EJSON.stringify);
       b = _.map(b, EJSON.stringify);
-      return _.isEmpty(_.difference(a, b));
+      return _.isEmpty(_.difference(a, b)) && _.isEmpty(_.difference(b, a));
     };
 
     // Insert a doc and start observing.
@@ -941,9 +941,9 @@ if (Meteor.isServer) {
                                      {removed: docId10}, {added: docId6},
                                      {added: docId11}, {added: docId12}]));
 
-    test.equal(currentState[docId6], { _id: docId6, foo: 22, bar: 24 });
-    test.equal(currentState[docId11], { _id: docId11, foo: 22, bar: 33.5 });
-    test.equal(currentState[docId12], { _id: docId12, foo: 22, bar: 43.5 });
+    test.equal(o.state[docId6], { _id: docId6, foo: 22, bar: 24 });
+    test.equal(o.state[docId11], { _id: docId11, foo: 22, bar: 33.5 });
+    test.equal(o.state[docId12], { _id: docId12, foo: 22, bar: 43.5 });
     clearOutput(o);
 
     o.handle.stop();

From 9d4783e4fc6bb9f4dc5f60d9ecd4b2a31a62155d Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sun, 23 Feb 2014 17:51:51 -0800
Subject: [PATCH 053/123] Additional assert for _safeAppendToBuffer

---
 packages/mongo-livedata/oplog_observe_driver.js | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 50f33aa679..65e26bf1c6 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -174,13 +174,16 @@ _.extend(OplogObserveDriver.prototype, {
     self._multiplexer.removed(id);
     if (! self._limit)
       return;
-    // xcxc size on heaps should be cached to O(1)
     if (self._published.size() < self._limit) {
       // The unpublished buffer is empty iff published contains the whole
       // matching set, i.e. number of matching documents is less or equal to the
       // queries limit.
-      if (! self._unpublishedBuffer.size())
+      if (! self._unpublishedBuffer.size()) {
+        // Assertion of the statement above
+        if (! self._safeAppendToBuffer && self._phase !== PHASE.QUERYING)
+          throw new Error("At this phase, buffer can be empty only if published contains the whole matching set");
         return;
+      }
 
       var newDocId = self._unpublishedBuffer.minElementId();
       var newDoc = self._unpublishedBuffer.get(newDocId);

From f0d13f6678ee1fff0f25a26d6489756efd458822 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sun, 23 Feb 2014 17:52:17 -0800
Subject: [PATCH 054/123] Restructure and update comments for
 _addMatching-limits behavior

The logic remains the same.
Also remove `fields` - nothing uses the term `fields` except for the fields projection.
---
 .../mongo-livedata/oplog_observe_driver.js    | 29 +++++++++++++------
 1 file changed, 20 insertions(+), 9 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 65e26bf1c6..c9d6ce6ed2 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -228,8 +228,8 @@ _.extend(OplogObserveDriver.prototype, {
   _addMatching: function (doc) {
     var self = this;
     var id = doc._id;
-    var fields = _.clone(doc);
-    delete fields._id;
+    var doc = _.clone(doc);
+    delete doc._id;
     if (self._published.has(id))
       throw Error("tried to add something already published " + id);
     if (self._limit && self._unpublishedBuffer.has(id))
@@ -242,14 +242,25 @@ _.extend(OplogObserveDriver.prototype, {
     // The query is unlimited or didn't publish enough documents yet or the new
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
+    var toPublish = ! limit || self._published.size() < limit ||
+                    comparator(maxPublished, doc) > 0;
+
     // Otherwise we might need to buffer it (only in case of limited query).
-    // Buffering a new document is allowed only if it is inserted in the middle
-    // or the beginning of it as we cannot determine if there are documents
-    // outside of the buffer easily.
-    if (!limit || self._published.size() < limit || comparator(maxPublished, fields) > 0) {
-      self._addPublished(id, fields);
-    } else if ((self._safeAppendToBuffer && self._unpublishedBuffer.size() < limit) || (maxBuffered && comparator(maxBuffered, fields) > 0)) {
-      self._addBuffered(id, fields);
+    // Buffering is allowed if the buffer is not filled up yet and all matching
+    // docs are either in the published set or in the buffer.
+    var canAppendToBuffer = self._safeAppendToBuffer &&
+                            self._unpublishedBuffer.size() < limit;
+
+    // Or if it is small enough to be safely inserted to the middle or the
+    // beginning of the buffer.
+    var canInsertIntoBuffer = maxBuffered && comparator(maxBuffered, doc) > 0;
+
+    var toBuffer = canAppendToBuffer || canInsertIntoBuffer;
+
+    if (toPublish) {
+      self._addPublished(id, doc);
+    } else if (toBuffer) {
+      self._addBuffered(id, doc);
     }
   },
   // Called when a document leaves the "Matching" results set.

From 90ba50acb74470d76b58b7aa5bd8531e0343bfd1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sun, 23 Feb 2014 18:19:57 -0800
Subject: [PATCH 055/123] Restructure and comment on _handleDoc logic for
 limits

Logic remains the same.
---
 .../mongo-livedata/oplog_observe_driver.js    | 38 ++++++++++++++-----
 1 file changed, 29 insertions(+), 9 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index c9d6ce6ed2..9dfca00e83 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -233,12 +233,14 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._published.has(id))
       throw Error("tried to add something already published " + id);
     if (self._limit && self._unpublishedBuffer.has(id))
-      throw Error("tried to add something already existed in buffer " + id); // xcxc error msg
+      throw Error("tried to add something already existed in buffer " + id);
 
     var limit = self._limit;
     var comparator = self._comparator;
-    var maxPublished = (limit && self._published.size() > 0) ? self._published.get(self._published.maxElementId()) : null;
-    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ? self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) : null;
+    var maxPublished = (limit && self._published.size() > 0) ?
+      self._published.get(self._published.maxElementId()) : null;
+    var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ?
+      self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) : null;
     // The query is unlimited or didn't publish enough documents yet or the new
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
@@ -310,17 +312,27 @@ _.extend(OplogObserveDriver.prototype, {
         // that buffer can't be empty if there are matching documents not
         // published. Notably, we don't want to schedule repoll and continue
         // relying on this property.
-        if (!self._limit || self._unpublishedBuffer.size() === 0 || comparator(newDoc, minBuffered) <= 0) {
+        var staysInPublished = ! self._limit ||
+                               self._unpublishedBuffer.size() === 0 ||
+                               comparator(newDoc, minBuffered) <= 0;
+
+        if (staysInPublished) {
           self._changePublished(id, oldDoc, newDoc);
         } else {
           // after the change doc doesn't stay in the published, remove it
           self._removePublished(id);
           // but it can move into buffered now, check it
           var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
-          if (self._safeAppendToBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0))
+
+          var toBuffer = self._safeAppendToBuffer ||
+                         (maxBuffered && comparator(newDoc, maxBuffered) < 0);
+
+          if (toBuffer) {
             self._addBuffered(id, newDoc);
-          else
+          } else {
+            // Throw away from both published set and buffer
             self._safeAppendToBuffer = false;
+          }
         }
       } else if (bufferedBefore) {
         oldDoc = self._unpublishedBuffer.get(id);
@@ -330,13 +342,21 @@ _.extend(OplogObserveDriver.prototype, {
 
         var maxPublished = self._published.get(self._published.maxElementId());
         var maxBuffered = self._unpublishedBuffer.size() && self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
+
         // the buffered doc was updated, it could move to published
-        if (comparator(newDoc, maxPublished) < 0) {
+        var toPublish = comparator(newDoc, maxPublished) < 0;
+
+        // or stays in buffer even after the change
+        var staysInBuffer = self._safeAppendToBuffer ||
+          (maxBuffered && comparator(newDoc, maxBuffered) < 0);
+
+        if (toPublish) {
           self._addPublished(id, newDoc);
-        } else if (self._safeAppendToBuffer || (maxBuffered && comparator(newDoc, maxBuffered) < 0)) {
-          // stays in buffer
+        } else if (staysInBuffer) {
+          // stays in buffer but changes
           self._unpublishedBuffer.set(id, newDoc);
         } else {
+          // Throw away from both published set and buffer
           self._safeAppendToBuffer = false;
         }
       } else {

From 7c202f6004bc697261cf58d126eb12009f62f4f5 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sun, 23 Feb 2014 20:03:38 -0800
Subject: [PATCH 056/123] Better assertions in the oplog code

---
 packages/mongo-livedata/oplog_observe_driver.js | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 9dfca00e83..86f35b2d6b 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -270,8 +270,8 @@ _.extend(OplogObserveDriver.prototype, {
   // and the effect of limit enforced.
   _removeMatching: function (id) {
     var self = this;
-    if (!self._published.has(id) && !self._limit)
-      throw Error("tried to remove something unpublished " + id); // xcxc fix this error msg
+    if (! self._published.has(id) && ! self._limit)
+      throw Error("tried to remove something matching but not cached " + id);
 
     if (self._published.has(id)) {
       self._removePublished(id);
@@ -448,9 +448,10 @@ _.extend(OplogObserveDriver.prototype, {
       if (self._published.has(id) || (self._limit && self._unpublishedBuffer.has(id)))
         self._removeMatching(id);
     } else if (op.op === 'i') {
-      // xcxc what if buffer has it?
       if (self._published.has(id))
-        throw new Error("insert found for already-existing ID");
+        throw new Error("insert found for already-existing ID in published");
+      if (self._unpublishedBuffer && self._unpublishedBuffer.has(id))
+        throw new Error("insert found for already-existing ID in buffer");
 
       // XXX what if selector yields?  for now it can't but later it could have
       // $where

From 90a0c2d5c2a3f9df98122bc90525f722c6043fd8 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 00:25:38 -0800
Subject: [PATCH 057/123] Add missing `_safeAppendToBuffer = false` and add
 clarity with comments

---
 .../mongo-livedata/oplog_observe_driver.js    | 19 +++++++++++++++----
 1 file changed, 15 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 86f35b2d6b..16d5b7f04b 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -202,6 +202,8 @@ _.extend(OplogObserveDriver.prototype, {
   _addBuffered: function (id, doc) {
     var self = this;
     self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc));
+
+    // If something is overflowing the buffer, we just remove it from cache
     if (self._unpublishedBuffer.size() > self._limit) {
       var maxBufferedId = self._unpublishedBuffer.maxElementId();
 
@@ -210,9 +212,14 @@ _.extend(OplogObserveDriver.prototype, {
       }
 
       self._unpublishedBuffer.remove(maxBufferedId);
+
+      // Since something matching is removed from cache (both published set and
+      // buffer), set flag to false
       self._safeAppendToBuffer = false;
     }
   },
+  // Is called either to remove the doc completely from matching set or to move
+  // it to the published set later.
   _removeBuffered: function (id) {
     var self = this;
     self._unpublishedBuffer.remove(id);
@@ -263,6 +270,9 @@ _.extend(OplogObserveDriver.prototype, {
       self._addPublished(id, doc);
     } else if (toBuffer) {
       self._addBuffered(id, doc);
+    } else {
+      // dropping it and not saving to the cache
+      self._safeAppendToBuffer = false;
     }
   },
   // Called when a document leaves the "Matching" results set.
@@ -475,10 +485,11 @@ _.extend(OplogObserveDriver.prototype, {
       } else if ((self._published.has(id) || self._unpublishedBuffer.has(id)) && canDirectlyModifyDoc) {
         // Oh great, we actually know what the document is, so we can apply
         // this directly.
-        if (self._published.has(id))
-          var newDoc = EJSON.clone(self._published.get(id));
-        else
-          var newDoc = EJSON.clone(self._unpublishedBuffer.get(id));
+        var newDoc = self._published.has(id) ?
+          self._published.get(id) :
+          self._unpublishedBuffer.get(id);
+        newDoc = EJSON.clone(newDoc);
+
         newDoc._id = id;
         LocalCollection._modify(newDoc, op.o);
         self._handleDoc(id, self._sharedProjectionFn(newDoc));

From 036dc4885edcf429490609c036034d6ffdc08ac7 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 00:26:11 -0800
Subject: [PATCH 058/123] Tiny changes to tests formatting

---
 .../mongo-livedata/mongo_livedata_tests.js     | 18 +++++++++++++++---
 1 file changed, 15 insertions(+), 3 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 9f0665c11e..832eb35416 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -793,9 +793,20 @@ if (Meteor.isServer) {
       return _.isEmpty(_.difference(a, b)) && _.isEmpty(_.difference(b, a));
     };
 
+    // tests '_id' subfields for all documents in oplog buffer
+    var testOplogBufferIds = function (ids) {
+      var bufferIds = [];
+      o.handle._multiplexer._observeDriver._unpublishedBuffer.forEach(function (x, id) {
+        bufferIds.push(id);
+      });
+
+      test.isTrue(setsEqual(ids, bufferIds));
+    };
+
     // Insert a doc and start observing.
     var docId1 = ins({foo: 22, bar: 5});
     var o = observer();
+    var usesOplog = o.handle._multiplexer._observeDriver._usesOplog;
     // Initial add.
     test.length(o.output, 1);
     test.equal(o.output.shift(), {added: docId1});
@@ -833,8 +844,9 @@ if (Meteor.isServer) {
     test.length(o.output, 2);
     test.isTrue(setsEqual(o.output, [{removed: docId5}, {added: docId2}]));
     clearOutput(o);
+    usesOplog && testOplogBufferIds([docId4]);
 
-    // Current state is [3 5 6] 7]
+    // Current state is [3 5 6 | 7]
     // Add some negative numbers overflowing the buffer.
     // New documents will take the published place, [3 5 6] will take the buffer
     // and 7 will be outside of the buffer in MongoDB.
@@ -849,7 +861,7 @@ if (Meteor.isServer) {
     test.equal(o.output, expected);
     clearOutput(o);
 
-    // Now the state is [-3 -2 -1] 3 5 6] 7
+    // Now the state is [-3 -2 -1 | 3 5 6] 7
     // If we update first 3 docs (increment them by 20), it would be
     // interesting.
     upd({ bar: { $lt: 0 }}, { $inc: { bar: 20 } }, { multi: true });
@@ -883,7 +895,7 @@ if (Meteor.isServer) {
     // XXX the oplog code analyzes the events one by one: one remove after
     // another. Poll-n-diff code, on the other side, analyzes the batch action
     // of multiple remove. Because of that difference, expected outputs differ.
-    if (o.handle._multiplexer._observeDriver._usesOplog) {
+    if (usesOplog) {
       var expectedRemoves = [{removed: docId3}, {removed: docId1},
                              {removed: docId2}, {removed: docId4}];
       var expectedAdds = [{added: docId4}, {added: docId8},

From 97f7ccfa63b5418495b88155efd20be3ff77095e Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 20:22:24 -0800
Subject: [PATCH 059/123] Glasser's comments

---
 packages/binary-heap/max-heap.js | 27 +++++++++++++++++----------
 1 file changed, 17 insertions(+), 10 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index ccc4ff3f36..ddc01733fd 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -4,7 +4,7 @@
 //      Object:
 //        - id - String - unique id of the item
 //        - value - Any - the data value
-//    the contents of initData is retained
+//    each value is retained
 MaxHeap = function (comparator, initData) {
   if (! _.isFunction(comparator))
     throw new Error('Passed comparator is invalid, should be a comparison function');
@@ -24,12 +24,12 @@ MaxHeap = function (comparator, initData) {
 
   // The Heap data-structure implemented as a 0-based contiguous array where
   // every item on index idx is a node in a complete binary tree. Every node can
-  // have leaves on indexes idx*2+1 and idx*2+2, except for the lists. Every
+  // have children on indexes idx*2+1 and idx*2+2, except for the leaves. Every
   // node has a parent on index (idx-1)/2;
   self._heap = [];
 
   // If the initial array is passed, we can build the heap in linear time
-  // complexity (O(N)) compared to linearithmetic time complexity (O(nlogn)) if
+  // complexity (O(N)) compared to linearithmic time complexity (O(nlogn)) if
   // we push elements one by one.
   if (_.isArray(initData))
     self._initFromData(initData);
@@ -41,7 +41,7 @@ _.extend(MaxHeap.prototype, {
     var self = this;
 
     self._heap = _.map(data, function (o) {
-      return { id: EJSON.clone(o.id), value: o.value };
+      return { id: o.id, value: o.value };
     });
 
     _.each(data, function (o, i) {
@@ -64,11 +64,11 @@ _.extend(MaxHeap.prototype, {
       var right = rightChildIdx(idx);
       var largest = idx;
 
-      if (left < self.size() && self._maxIndex(largest, left) === left) {
-        largest = left;
+      if (left < self.size()) {
+        largest = self._maxIndex(largest, left);
       }
-      if (right < self.size() && self._maxIndex(largest, right) === right) {
-        largest = right;
+      if (right < self.size()) {
+        largest = self._maxIndex(largest, right);
       }
 
       if (largest === idx)
@@ -133,7 +133,11 @@ _.extend(MaxHeap.prototype, {
 
       var idx = self._heapIdx.get(id);
       self._heap[idx].value = value;
+
+      // Fix the new value's position
+      // Either bubble new value up if it is greater than its parent
       self._upHeap(idx);
+      // or bubble it down if it is smaller than one of its children
       self._downHeap(idx);
     } else {
       self._heapIdx.set(id, self._heap.length);
@@ -152,8 +156,10 @@ _.extend(MaxHeap.prototype, {
         self._swap(idx, last);
         self._heap.pop();
         self._heapIdx.remove(id);
-        self._downHeap(idx);
+
+        // Fix the swapped value's position
         self._upHeap(idx);
+        self._downHeap(idx);
       } else {
         self._heap.pop();
         self._heapIdx.remove(id);
@@ -173,6 +179,7 @@ _.extend(MaxHeap.prototype, {
     self._heap = [];
     self._heapIdx.clear();
   },
+  // iterate over values in no particular order
   forEach: function (iterator) {
     var self = this;
     _.each(self._heap, function (obj) {
@@ -206,7 +213,7 @@ _.extend(MaxHeap.prototype, {
     for (var i = 1; i < self._heap.length; i++)
       if (self._maxIndex(parentIdx(i), i) !== parentIdx(i))
           throw new Error("An item with id " + self._heap[i].id +
-                          " has a parent younger than him: " +
+                          " has a parent younger than it: " +
                           self._heap[parentIdx(i)].id);
   }
 });

From 9451416f2767314266e024638bf23db472eec444 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 20:28:45 -0800
Subject: [PATCH 060/123] Use options object in MaxHeap constructor and allow
 passing a custom IdMap class

---
 packages/binary-heap/max-heap.js     | 26 ++++++++++++++------------
 packages/binary-heap/min-max-heap.js |  6 +++---
 packages/binary-heap/package.js      |  1 -
 3 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/packages/binary-heap/max-heap.js b/packages/binary-heap/max-heap.js
index ddc01733fd..aa08a3fad5 100644
--- a/packages/binary-heap/max-heap.js
+++ b/packages/binary-heap/max-heap.js
@@ -1,11 +1,14 @@
 // Constructor of Heap
 // - comparator - Function - given two items returns a number
-// - initData - Array - the initial data in a format:
-//      Object:
-//        - id - String - unique id of the item
-//        - value - Any - the data value
-//    each value is retained
-MaxHeap = function (comparator, initData) {
+// - options:
+//   - initData - Array - Optional - the initial data in a format:
+//        Object:
+//          - id - String - unique id of the item
+//          - value - Any - the data value
+//      each value is retained
+//   - IdMap - Constructor - Optional - custom IdMap class to store id->index
+//       mappings internally. Standard IdMap is used by default.
+MaxHeap = function (comparator, options) {
   if (! _.isFunction(comparator))
     throw new Error('Passed comparator is invalid, should be a comparison function');
   var self = this;
@@ -15,12 +18,11 @@ MaxHeap = function (comparator, initData) {
   // value is greater than the first and zero if they are equal.
   self._comparator = comparator;
 
+  options = _.defaults(options || {}, { IdMap: IdMap });
+
   // _heapIdx maps an id to an index in the Heap array the corresponding value
   // is located on.
-  if (Package && Package.minimongo)
-    self._heapIdx = new Package.minimongo.LocalCollection._IdMap;
-  else
-    self._heapIdx = new IdMap;
+  self._heapIdx = new options.IdMap;
 
   // The Heap data-structure implemented as a 0-based contiguous array where
   // every item on index idx is a node in a complete binary tree. Every node can
@@ -31,8 +33,8 @@ MaxHeap = function (comparator, initData) {
   // If the initial array is passed, we can build the heap in linear time
   // complexity (O(N)) compared to linearithmic time complexity (O(nlogn)) if
   // we push elements one by one.
-  if (_.isArray(initData))
-    self._initFromData(initData);
+  if (_.isArray(options.initData))
+    self._initFromData(options.initData);
 };
 
 _.extend(MaxHeap.prototype, {
diff --git a/packages/binary-heap/min-max-heap.js b/packages/binary-heap/min-max-heap.js
index b491c61bb1..49911b4785 100644
--- a/packages/binary-heap/min-max-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -10,13 +10,13 @@
 // (http://www.cs.otago.ac.nz/staffpriv/mike/Papers/MinMaxHeaps/MinMaxHeaps.pdf)
 // and Interval Heaps
 // (http://www.cise.ufl.edu/~sahni/dsaac/enrich/c13/double.htm)
-MinMaxHeap = function (comparator, initialData) {
+MinMaxHeap = function (comparator, options) {
   var self = this;
 
-  MaxHeap.call(self, comparator, initialData);
+  MaxHeap.call(self, comparator, options);
   self._minHeap = new MaxHeap(function (a, b) {
     return -comparator(a, b);
-  }, initialData);
+  }, options);
 };
 
 var F = function () {};
diff --git a/packages/binary-heap/package.js b/packages/binary-heap/package.js
index 5788fc1973..f8c4ae4613 100644
--- a/packages/binary-heap/package.js
+++ b/packages/binary-heap/package.js
@@ -7,7 +7,6 @@ Package.on_use(function (api) {
   api.export('MaxHeap');
   api.export('MinMaxHeap');
   api.use(['underscore', 'id-map']);
-  api.use(['minimongo'], { weak: true });
   api.add_files(['max-heap.js', 'min-max-heap.js']);
 });
 

From 6dfd5bfc29c3e30ff6787772c438207dc0f5c72a Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 21:25:17 -0800
Subject: [PATCH 061/123] Additional generated tests for min-max-heap

---
 packages/binary-heap/binary-heap-tests.js | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/packages/binary-heap/binary-heap-tests.js b/packages/binary-heap/binary-heap-tests.js
index 496af9c0c3..0a487b7b2f 100644
--- a/packages/binary-heap/binary-heap-tests.js
+++ b/packages/binary-heap/binary-heap-tests.js
@@ -90,6 +90,7 @@ Tinytest.add("binary-heap - big test for min-max-heap", function (test) {
   var heap = new MinMaxHeap(function (a, b) { return a-b; });
   var output = [];
 
+  var initialSets = _.clone(allNumbers);
   _.each(allNumbers, function (n) {
     heap.set(n, n);
     heap._selfCheck();
@@ -97,6 +98,8 @@ Tinytest.add("binary-heap - big test for min-max-heap", function (test) {
   });
 
   allNumbers = _.shuffle(allNumbers);
+  var secondarySets = _.clone(allNumbers);
+
   _.each(allNumbers, function (n) {
     heap.set(-n, n);
     heap._selfCheck();
@@ -110,8 +113,26 @@ Tinytest.add("binary-heap - big test for min-max-heap", function (test) {
     heap._selfCheck(); heap._minHeap._selfCheck();
   });
 
+  test.equal(heap.size(), 0);
+
   allNumbers.sort(function (a, b) { return a-b; });
 
-  test.equal(output, allNumbers);
+  var initialTestText = "initial sets: " + initialSets.toString() +
+    "; secondary sets: " + secondarySets.toString();
+  test.equal(output, allNumbers, initialTestText);
+
+  _.each(initialSets, function (n) { heap.set(n, n); })
+  _.each(secondarySets, function (n) { heap.set(-n, n); });
+
+  allNumbers.sort(function (a, b) { return b-a; });
+  output = [];
+  _.times(positiveNumbers.length + negativeNumbers.length, function () {
+    var maxId = heap.maxElementId();
+    output.push(heap.get(maxId));
+    heap.remove(maxId);
+    heap._selfCheck(); heap._minHeap._selfCheck();
+  });
+
+  test.equal(output, allNumbers, initialTestText);
 });
 

From a51bc653727b3f4fdd7e7c4bc7279999aecd6896 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 23:55:16 -0800
Subject: [PATCH 062/123] Meteor._inherits - prototypical inheritance
 implementation

---
 packages/meteor/helpers.js | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/packages/meteor/helpers.js b/packages/meteor/helpers.js
index 9b05246669..341df557b9 100644
--- a/packages/meteor/helpers.js
+++ b/packages/meteor/helpers.js
@@ -111,5 +111,26 @@ _.extend(Meteor, {
         return fut.wait();
       return result;
     };
+  },
+
+  // Sets child's prototype to a new object whose prototype is parent's
+  // prototype. Used as:
+  //   Meteor._inherit(ClassB, ClassA).
+  //   _.extend(ClassB.prototype, { ... })
+  // Inspired by CoffeeScript's `extend` and Google Closure's `goog.inherits`.
+  _inherits: function (Child, Parent) {
+    // copy static fields
+    _.each(Parent, function (prop, field) {
+      Child[field] = prop;
+    });
+
+    // a middle member of prototype chain: takes the prototype from the Parent
+    var Middle = function () {
+      this.constructor = Child;
+    };
+    Middle.prototype = Parent.prototype;
+    Child.prototype = new Middle();
+    Child.__super__ = Parent.prototype;
+    return Child;
   }
 });

From 7bd160192c7d7f2b81bf9c616f50f015a9e83425 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 23:55:35 -0800
Subject: [PATCH 063/123] Replace ad-hoc inheritance with Meteor._inherit calls

---
 packages/binary-heap/min-max-heap.js | 4 +---
 packages/minimongo/id_map.js         | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/packages/binary-heap/min-max-heap.js b/packages/binary-heap/min-max-heap.js
index 49911b4785..990a71402d 100644
--- a/packages/binary-heap/min-max-heap.js
+++ b/packages/binary-heap/min-max-heap.js
@@ -19,9 +19,7 @@ MinMaxHeap = function (comparator, options) {
   }, options);
 };
 
-var F = function () {};
-F.prototype = MaxHeap.prototype;
-MinMaxHeap.prototype = new F;
+Meteor._inherits(MinMaxHeap, MaxHeap);
 
 _.extend(MinMaxHeap.prototype, {
   set: function (id, value) {
diff --git a/packages/minimongo/id_map.js b/packages/minimongo/id_map.js
index cf098dbee1..ba4880980b 100644
--- a/packages/minimongo/id_map.js
+++ b/packages/minimongo/id_map.js
@@ -3,5 +3,5 @@ LocalCollection._IdMap = function () {
   IdMap.call(self, LocalCollection._idStringify, LocalCollection._idParse);
 };
 
-LocalCollection._IdMap.prototype = IdMap.prototype;
+Meteor._inherits(LocalCollection._IdMap, IdMap);
 

From b8c8d87d5fec8a681ea8e73026551f242dbf0349 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Thu, 20 Feb 2014 16:37:35 -0800
Subject: [PATCH 064/123] Relax tests and remove a dead piece of code

Apparently the implementation of sin/cos changed in the recent Chrome as now the
results are off by 1e-6 (compared to a different C++ implementation we generated
the tests previously).

It is not worth tracking down the 1e-6 error and change implementation for it.
---
 packages/geojson-utils/geojson-utils.tests.js | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/packages/geojson-utils/geojson-utils.tests.js b/packages/geojson-utils/geojson-utils.tests.js
index 73cc5d1554..ea7ab39462 100644
--- a/packages/geojson-utils/geojson-utils.tests.js
+++ b/packages/geojson-utils/geojson-utils.tests.js
@@ -54,10 +54,6 @@ Tinytest.add("geojson-utils - point distance", function (test) {
 });
 
 Tinytest.add("geojson-utils - points distance generated tests", function (test) {
-  var floatEqual = function (a, b) {
-    test.isTrue(Math.abs(a - b) < 0.000001);
-  };
-
   // Pairs of points we will be looking a distance between
   var tests = [[[-19.416501816827804,-13.442164216190577], [8.694866622798145,-8.511979941977188]],
     [[151.2841189110186,-56.14564002258703], [167.77983197313733,0.05544793023727834]],
@@ -91,8 +87,8 @@ Tinytest.add("geojson-utils - points distance generated tests", function (test)
 
   _.each(tests, function (pair, testN) {
     var distance = GeoJSON.pointDistance.apply(this, _.map(pair, toGeoJSONPoint));
-    test.isTrue(Math.abs(distance - answers[testN]) < 0.00000001,
-      "Wrong distance between points " + JSON.stringify(pair) + ": " + distance);
+    test.isTrue(Math.abs(distance - answers[testN]) < 0.000001,
+      "Wrong distance between points " + JSON.stringify(pair) + ": " + distance + ", " + Math.abs(distance - answers[testN]) + " differenc");
   });
 
   function toGeoJSONPoint (coordinates) {

From db28492e6bd99aa138e0c7d4a1ce15fe4661f13b Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 00:45:22 -0800
Subject: [PATCH 065/123] Tweak the description of id-map

---
 packages/id-map/package.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/id-map/package.js b/packages/id-map/package.js
index d3ef1db346..876d3406a6 100644
--- a/packages/id-map/package.js
+++ b/packages/id-map/package.js
@@ -1,5 +1,5 @@
 Package.describe({
-  summary: "Dictionary data structure: a wrapper for a raw object",
+  summary: "Dictionary data structure allowing non-string keys",
   internal: true
 });
 

From 4657f8c1df75b388003a2aef42754d7a57acb17c Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 00:51:00 -0800
Subject: [PATCH 066/123] Use binary-heap package in mongo-livedata

---
 packages/mongo-livedata/dummy-structure.js    | 44 -------------------
 .../mongo-livedata/oplog_observe_driver.js    |  5 ++-
 packages/mongo-livedata/package.js            |  6 ++-
 3 files changed, 7 insertions(+), 48 deletions(-)
 delete mode 100644 packages/mongo-livedata/dummy-structure.js

diff --git a/packages/mongo-livedata/dummy-structure.js b/packages/mongo-livedata/dummy-structure.js
deleted file mode 100644
index 0b0708c210..0000000000
--- a/packages/mongo-livedata/dummy-structure.js
+++ /dev/null
@@ -1,44 +0,0 @@
-// Implements the interface of IdMap and knows how to find Min or Max element
-DummyStructure = function (comparator) {
-  var self = this;
-  self.comparator = comparator;
-  self.idMap = new LocalCollection._IdMap;
-};
-
-_.each(['get', 'set', 'remove', 'has', 'empty', 'clear', 'forEach', 'size', 'setDefault'], function (method) {
-  DummyStructure.prototype[method] = function (/* arguments */) {
-    var self = this;
-    return self.idMap[method].apply(self.idMap, arguments);
-  };
-});
-
-DummyStructure.prototype.clone = function () {
-  var self = this;
-  var clone = new DummyStructure;
-  clone.comparator = self.comparator;
-  clone.idMap = self.idMap.clone();
-  return clone;
-};
-
-DummyStructure.prototype.minElementId = function () {
-  var self = this;
-  var minElementId = null;
-  self.idMap.forEach(function (value, key) {
-    if (minElementId === null)
-      minElementId = key;
-    else if (self.comparator(value, self.idMap.get(minElementId)) < 0)
-      minElementId = key;
-  });
-
-  return minElementId;
-};
-
-DummyStructure.prototype.maxElementId = function () {
-  var self = this;
-  var comparator = self.comparator;
-  self.comparator = function (a, b) { return -comparator(a, b); };
-  var maxElementId = self.minElementId();
-  self.comparator = comparator;
-  return maxElementId;
-};
-
diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 16d5b7f04b..41f58efa3d 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -38,11 +38,12 @@ OplogObserveDriver = function (options) {
     // comparator only once in the constructor.
     var sorter = new Minimongo.Sorter(options.cursorDescription.options.sort);
     var comparator = sorter.getComparator();
+    var heapOptions = { IdMap: LocalCollection._IdMap };
     self._limit = self._cursorDescription.options.limit;
     self._comparator = comparator;
-    self._unpublishedBuffer = new DummyStructure(comparator);
+    self._unpublishedBuffer = new MinMaxHeap(comparator, heapOptions);
     // We need something that can find Max value in addition to IdMap interface
-    self._published = new DummyStructure(comparator);
+    self._published = new MaxHeap(comparator, heapOptions);
   } else {
     self._limit = 0;
     self._comparator = null;
diff --git a/packages/mongo-livedata/package.js b/packages/mongo-livedata/package.js
index 898258486a..4f1dfa8866 100644
--- a/packages/mongo-livedata/package.js
+++ b/packages/mongo-livedata/package.js
@@ -24,6 +24,10 @@ Package.on_use(function (api) {
           ['client', 'server']);
   api.use('check', ['client', 'server']);
 
+  // Binary Heap data structure is used to optimize oplog observe driver
+  // performance.
+  api.use('binary-heap', 'server');
+
   // Allow us to detect 'insecure'.
   api.use('insecure', {weak: true});
 
@@ -47,8 +51,6 @@ Package.on_use(function (api) {
   // For tests only.
   api.export('MongoTest', 'server', {testOnly: true});
 
-  // xcxc temporary
-  api.add_files('dummy-structure.js', 'server');
   api.add_files(['mongo_driver.js', 'oplog_tailing.js',
                  'observe_multiplex.js', 'doc_fetcher.js',
                  'polling_observe_driver.js','oplog_observe_driver.js'],

From be7038e1e8a22c77f22943acd1a571af4af28c5f Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 01:11:35 -0800
Subject: [PATCH 067/123] More white-box tests

one test fails so far
---
 .../mongo-livedata/mongo_livedata_tests.js     | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 832eb35416..0d36a112e7 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -800,7 +800,7 @@ if (Meteor.isServer) {
         bufferIds.push(id);
       });
 
-      test.isTrue(setsEqual(ids, bufferIds));
+      test.isTrue(setsEqual(ids, bufferIds), "expected: " + ids + "; got: " + bufferIds);
     };
 
     // Insert a doc and start observing.
@@ -885,8 +885,9 @@ if (Meteor.isServer) {
                           expectedRemoves));
     test.equal(_.filter(o.output, function (e){return e.added;}), expectedAdds);
     clearOutput(o);
+    usesOplog && testOplogBufferIds([docId4, docId7, docId8]);
 
-    // The new arrangement is [3 5 6] 7 17 18] 19
+    // The new arrangement is [3 5 6 | 7 17 18] 19
     // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
     // Remove first 4 docs (3, 1, 2, 4) forcing buffer to become empty and
     // schedule a repoll.
@@ -921,27 +922,33 @@ if (Meteor.isServer) {
     var docId11 = ins({ foo: 22, bar: 41 });
     var docId12 = ins({ foo: 22, bar: 51 });
 
+    // Becomes [17 18 19 | 21 31 41] 51
+    usesOplog && testOplogBufferIds([docId9, docId10, docId11]);
     test.length(o.output, 0);
     upd({ bar: { $lt: 20 } }, { $inc: { bar: 5 } }, { multi: true });
-    // Becomes [21 22 23] 24 31 41] 51
+    // Becomes [21 22 23 | 24 31 41] 51
     test.length(o.output, 4);
     test.isTrue(setsEqual(o.output, [{removed: docId6},
                                      {added: docId9},
                                      {changed: docId7},
                                      {changed: docId8}]));
     clearOutput(o);
+    usesOplog && testOplogBufferIds([docId6, docId10, docId11]);
 
     rem(docId9);
-    // Becomes [22 23 24] 31 41] 51
+    // Becomes [22 23 24 | 31 41] 51
     test.length(o.output, 2);
     test.isTrue(setsEqual(o.output, [{removed: docId9}, {added: docId6}]));
     clearOutput(o);
+    usesOplog && testOplogBufferIds([docId10, docId11]);
 
     upd({ bar: { $gt: 25 } }, { $inc: { bar: -7.5 } }, { multi: true });
-    // Becomes [22 23 23.5] 24 33.5] 43.5
+    // Becomes [22 23 23.5 | 24 33.5] 43.5
     test.length(o.output, 2);
     test.isTrue(setsEqual(o.output, [{removed: docId6}, {added: docId10}]));
     clearOutput(o);
+    // xcxc this test fails :(
+    usesOplog && testOplogBufferIds([docId6, docId11]);
 
     // Force buffer objects to be moved into published set so we can check them
     rem(docId7);
@@ -957,6 +964,7 @@ if (Meteor.isServer) {
     test.equal(o.state[docId11], { _id: docId11, foo: 22, bar: 33.5 });
     test.equal(o.state[docId12], { _id: docId12, foo: 22, bar: 43.5 });
     clearOutput(o);
+    usesOplog && testOplogBufferIds([]);
 
     o.handle.stop();
     onComplete();

From b082f1847a77e2a58aa1bbed1e7e9dc765674476 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 10:00:27 -0800
Subject: [PATCH 068/123] Another small test missing

---
 packages/mongo-livedata/mongo_livedata_tests.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 0d36a112e7..942d26ccb4 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -915,6 +915,7 @@ if (Meteor.isServer) {
                           expectedRemoves));
     test.equal(_.filter(o.output, function (e) {return e.added;}), expectedAdds);
     clearOutput(o);
+    usesOplog && testOplogBufferIds([]);
 
     // The new arrangement is [17 18 19] or [docId6 docId7 docId8]
     var docId9 = ins({ foo: 22, bar: 21 });

From 3355ba7e85db7f504bafb310e760c66a43e66c22 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 12:08:41 -0800
Subject: [PATCH 069/123] Fix a check when buffer can be null

---
 packages/mongo-livedata/oplog_observe_driver.js | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 41f58efa3d..cb125a6c78 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -481,9 +481,12 @@ _.extend(OplogObserveDriver.prototype, {
       var canDirectlyModifyDoc =
             !isReplace && modifierCanBeDirectlyApplied(op.o);
 
+      var publishedBefore = self._published.has(id);
+      var bufferedBefore = self._limit && self._unpublishedBuffer.has(id);
+
       if (isReplace) {
         self._handleDoc(id, _.extend({_id: id}, op.o));
-      } else if ((self._published.has(id) || self._unpublishedBuffer.has(id)) && canDirectlyModifyDoc) {
+      } else if ((publishedBefore || bufferedBefore) && canDirectlyModifyDoc) {
         // Oh great, we actually know what the document is, so we can apply
         // this directly.
         var newDoc = self._published.has(id) ?

From 6ad69c52ae6cfc032e8a4032dc872b2ca8162c15 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Mon, 24 Feb 2014 12:10:15 -0800
Subject: [PATCH 070/123] Explicit test for unsupported cursor with a single
 skip

---
 packages/mongo-livedata/oplog_tests.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/mongo-livedata/oplog_tests.js b/packages/mongo-livedata/oplog_tests.js
index 5d28c01a44..d9bc31709c 100644
--- a/packages/mongo-livedata/oplog_tests.js
+++ b/packages/mongo-livedata/oplog_tests.js
@@ -43,4 +43,5 @@ Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) {
   supported(true, {}, { sort: {x:1}, limit: 5 });
   supported(false, {}, { limit: 5 });
   supported(false, {}, { skip: 2, limit: 5 });
+  supported(false, {}, { skip: 2 });
 });

From 76ffc74e0efb96971499ed8c05a88e250b757478 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 11:26:38 -0800
Subject: [PATCH 071/123] Tests tests tests

---
 packages/mongo-livedata/mongo_livedata_tests.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 942d26ccb4..f747a771b6 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -860,6 +860,7 @@ if (Meteor.isServer) {
 
     test.equal(o.output, expected);
     clearOutput(o);
+    usesOplog && testOplogBufferIds([docId1, docId2, docId3]);
 
     // Now the state is [-3 -2 -1 | 3 5 6] 7
     // If we update first 3 docs (increment them by 20), it would be

From 370ee125a9252d747a7edd975c7c32f6de12ab87 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 11:37:01 -0800
Subject: [PATCH 072/123] Rationalize the test failure

---
 packages/mongo-livedata/mongo_livedata_tests.js | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index f747a771b6..d29114c3b3 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -945,12 +945,13 @@ if (Meteor.isServer) {
     usesOplog && testOplogBufferIds([docId10, docId11]);
 
     upd({ bar: { $gt: 25 } }, { $inc: { bar: -7.5 } }, { multi: true });
-    // Becomes [22 23 23.5 | 24 33.5] 43.5
+    // Becomes [22 23 23.5 | 24] 33.5 43.5 - 33.5 doesn't update in-place in
+    // buffer, because it the driver is not sure it can do it and there is no a
+    // different doc which is less than 33.5.
     test.length(o.output, 2);
     test.isTrue(setsEqual(o.output, [{removed: docId6}, {added: docId10}]));
     clearOutput(o);
-    // xcxc this test fails :(
-    usesOplog && testOplogBufferIds([docId6, docId11]);
+    usesOplog && testOplogBufferIds([docId6]);
 
     // Force buffer objects to be moved into published set so we can check them
     rem(docId7);

From b23a26fbbd1e5449e422661a11e5dfcf23f0b88c Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 11:49:54 -0800
Subject: [PATCH 073/123] More white-box style tests for _safeAppendToBuffer
 flag

---
 packages/mongo-livedata/mongo_livedata_tests.js | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index d29114c3b3..161ccbf36d 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -802,6 +802,12 @@ if (Meteor.isServer) {
 
       test.isTrue(setsEqual(ids, bufferIds), "expected: " + ids + "; got: " + bufferIds);
     };
+    var testSafeAppendToBufferFlag = function (expected) {
+      if (expected)
+        test.isTrue(o.handle._multiplexer._observeDriver._safeAppendToBuffer);
+      else
+        test.isFalse(o.handle._multiplexer._observeDriver._safeAppendToBuffer);
+    };
 
     // Insert a doc and start observing.
     var docId1 = ins({foo: 22, bar: 5});
@@ -845,6 +851,7 @@ if (Meteor.isServer) {
     test.isTrue(setsEqual(o.output, [{removed: docId5}, {added: docId2}]));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId4]);
+    usesOplog && testSafeAppendToBufferFlag(true);
 
     // Current state is [3 5 6 | 7]
     // Add some negative numbers overflowing the buffer.
@@ -861,6 +868,7 @@ if (Meteor.isServer) {
     test.equal(o.output, expected);
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId1, docId2, docId3]);
+    usesOplog && testSafeAppendToBufferFlag(false);
 
     // Now the state is [-3 -2 -1 | 3 5 6] 7
     // If we update first 3 docs (increment them by 20), it would be
@@ -887,6 +895,7 @@ if (Meteor.isServer) {
     test.equal(_.filter(o.output, function (e){return e.added;}), expectedAdds);
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId4, docId7, docId8]);
+    usesOplog && testSafeAppendToBufferFlag(false);
 
     // The new arrangement is [3 5 6 | 7 17 18] 19
     // By ids: [docId3, docId1, docId2] docId4] docId6 docId7 docId8
@@ -917,6 +926,7 @@ if (Meteor.isServer) {
     test.equal(_.filter(o.output, function (e) {return e.added;}), expectedAdds);
     clearOutput(o);
     usesOplog && testOplogBufferIds([]);
+    usesOplog && testSafeAppendToBufferFlag(true);
 
     // The new arrangement is [17 18 19] or [docId6 docId7 docId8]
     var docId9 = ins({ foo: 22, bar: 21 });
@@ -926,6 +936,7 @@ if (Meteor.isServer) {
 
     // Becomes [17 18 19 | 21 31 41] 51
     usesOplog && testOplogBufferIds([docId9, docId10, docId11]);
+    usesOplog && testSafeAppendToBufferFlag(false);
     test.length(o.output, 0);
     upd({ bar: { $lt: 20 } }, { $inc: { bar: 5 } }, { multi: true });
     // Becomes [21 22 23 | 24 31 41] 51
@@ -936,6 +947,7 @@ if (Meteor.isServer) {
                                      {changed: docId8}]));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId6, docId10, docId11]);
+    usesOplog && testSafeAppendToBufferFlag(false);
 
     rem(docId9);
     // Becomes [22 23 24 | 31 41] 51
@@ -943,6 +955,7 @@ if (Meteor.isServer) {
     test.isTrue(setsEqual(o.output, [{removed: docId9}, {added: docId6}]));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId10, docId11]);
+    usesOplog && testSafeAppendToBufferFlag(false);
 
     upd({ bar: { $gt: 25 } }, { $inc: { bar: -7.5 } }, { multi: true });
     // Becomes [22 23 23.5 | 24] 33.5 43.5 - 33.5 doesn't update in-place in
@@ -952,6 +965,7 @@ if (Meteor.isServer) {
     test.isTrue(setsEqual(o.output, [{removed: docId6}, {added: docId10}]));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId6]);
+    usesOplog && testSafeAppendToBufferFlag(false);
 
     // Force buffer objects to be moved into published set so we can check them
     rem(docId7);
@@ -968,6 +982,7 @@ if (Meteor.isServer) {
     test.equal(o.state[docId12], { _id: docId12, foo: 22, bar: 43.5 });
     clearOutput(o);
     usesOplog && testOplogBufferIds([]);
+    usesOplog && testSafeAppendToBufferFlag(true);
 
     o.handle.stop();
     onComplete();

From 4b955b5afc6407c3e05ab67eff3756f03e1c55dc Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 22:45:29 -0800
Subject: [PATCH 074/123] Relax events order in tests as mongo-livedata doesn't
 define the order in observe interface

---
 .../mongo-livedata/mongo_livedata_tests.js     | 18 ++++++------------
 1 file changed, 6 insertions(+), 12 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 161ccbf36d..01c5b54b52 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -841,9 +841,9 @@ if (Meteor.isServer) {
     var docId5 = ins({ foo: 22, bar: -1 });
     // We should get an added and a removed events
     test.length(o.output, 2);
-    test.equal(o.output.shift(), {added: docId5});
     // doc 2 was removed from the published set as it is too big to be in
-    test.equal(o.output.shift(), {removed: docId2});
+    test.isTrue(setsEqual(o.output, [{added: docId5}, {removed: docId2}]));
+    clearOutput(o);
 
     // Now remove something and that doc 2 should be right back
     rem(docId5);
@@ -865,7 +865,7 @@ if (Meteor.isServer) {
                     {added: docId7}, {removed: docId1},
                     {added: docId8}, {removed: docId3}];
 
-    test.equal(o.output, expected);
+    test.isTrue(setsEqual(o.output, expected));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId1, docId2, docId3]);
     usesOplog && testSafeAppendToBufferFlag(false);
@@ -887,12 +887,7 @@ if (Meteor.isServer) {
                         {added: docId1},
                         {added: docId2}];
 
-    // Note: since we are updating multiple things, the order of updates may
-    // differ from launch to launch. That's why we compare even positions
-    // (removes) w/o looking at ordering.
-    test.isTrue(setsEqual(_.filter(o.output, function (e) {return e.removed;}),
-                          expectedRemoves));
-    test.equal(_.filter(o.output, function (e){return e.added;}), expectedAdds);
+    test.isTrue(setsEqual(o.output, expectedAdds.concat(expectedRemoves)));
     clearOutput(o);
     usesOplog && testOplogBufferIds([docId4, docId7, docId8]);
     usesOplog && testSafeAppendToBufferFlag(false);
@@ -921,9 +916,7 @@ if (Meteor.isServer) {
       test.length(o.output, 6);
     }
 
-    test.isTrue(setsEqual(_.filter(o.output, function (e) {return e.removed;}),
-                          expectedRemoves));
-    test.equal(_.filter(o.output, function (e) {return e.added;}), expectedAdds);
+    test.isTrue(setsEqual(o.output, expectedAdds.concat(expectedRemoves)));
     clearOutput(o);
     usesOplog && testOplogBufferIds([]);
     usesOplog && testSafeAppendToBufferFlag(true);
@@ -977,6 +970,7 @@ if (Meteor.isServer) {
                                      {removed: docId10}, {added: docId6},
                                      {added: docId11}, {added: docId12}]));
 
+    test.length(_.keys(o.state), 3);
     test.equal(o.state[docId6], { _id: docId6, foo: 22, bar: 24 });
     test.equal(o.state[docId11], { _id: docId11, foo: 22, bar: 33.5 });
     test.equal(o.state[docId12], { _id: docId12, foo: 22, bar: 43.5 });

From afc58bcac6f43f3d1631c682e2eb21e308e759ab Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 23:15:02 -0800
Subject: [PATCH 075/123] fix typo

---
 packages/mongo-livedata/oplog_observe_driver.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index cb125a6c78..199a91146d 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -759,7 +759,7 @@ OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   // skip is not supported: to support it we would need to keep track of all
   // "skipped" documents or at least their ids.
   // limit w/o a sort specifier is not supported: current implementation needs a
-  // determent way to order documents.
+  // deterministic way to order documents.
   if (options.skip || (options.limit && !options.sort)) return false;
 
   // If a fields projection option is given check if it is supported by

From a0cc339b6d5171345493379aefd5d547848d5c8b Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 23:19:20 -0800
Subject: [PATCH 076/123] toPublish and toBuffer (or staysInBuffer) can't be
 true at once

---
 packages/mongo-livedata/oplog_observe_driver.js | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 199a91146d..199fba6265 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -258,12 +258,13 @@ _.extend(OplogObserveDriver.prototype, {
     // Otherwise we might need to buffer it (only in case of limited query).
     // Buffering is allowed if the buffer is not filled up yet and all matching
     // docs are either in the published set or in the buffer.
-    var canAppendToBuffer = self._safeAppendToBuffer &&
+    var canAppendToBuffer = !toPublish && self._safeAppendToBuffer &&
                             self._unpublishedBuffer.size() < limit;
 
     // Or if it is small enough to be safely inserted to the middle or the
     // beginning of the buffer.
-    var canInsertIntoBuffer = maxBuffered && comparator(maxBuffered, doc) > 0;
+    var canInsertIntoBuffer = !toPublish && maxBuffered &&
+                              comparator(maxBuffered, doc) > 0;
 
     var toBuffer = canAppendToBuffer || canInsertIntoBuffer;
 
@@ -358,8 +359,8 @@ _.extend(OplogObserveDriver.prototype, {
         var toPublish = comparator(newDoc, maxPublished) < 0;
 
         // or stays in buffer even after the change
-        var staysInBuffer = self._safeAppendToBuffer ||
-          (maxBuffered && comparator(newDoc, maxBuffered) < 0);
+        var staysInBuffer = (! toPublish && self._safeAppendToBuffer) ||
+          (!toPublish && maxBuffered && comparator(newDoc, maxBuffered) < 0);
 
         if (toPublish) {
           self._addPublished(id, newDoc);

From 2f4dac064e7895787a4f7cc0d6c9896cb8289a45 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 23:22:59 -0800
Subject: [PATCH 077/123] Keep _id in published and buffer

---
 packages/mongo-livedata/oplog_observe_driver.js | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 199fba6265..8d422ac196 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -142,8 +142,10 @@ OplogObserveDriver = function (options) {
 _.extend(OplogObserveDriver.prototype, {
   _addPublished: function (id, doc) {
     var self = this;
+    var fields = _.clone(doc);
+    delete fields._id;
     self._published.set(id, self._sharedProjectionFn(doc));
-    self._multiplexer.added(id, self._projectionFn(doc));
+    self._multiplexer.added(id, self._projectionFn(fields));
 
     // After adding this document, the published set might be overflowed
     // (exceeding capacity specified by limit). If so, push the maximum element
@@ -236,8 +238,6 @@ _.extend(OplogObserveDriver.prototype, {
   _addMatching: function (doc) {
     var self = this;
     var id = doc._id;
-    var doc = _.clone(doc);
-    delete doc._id;
     if (self._published.has(id))
       throw Error("tried to add something already published " + id);
     if (self._limit && self._unpublishedBuffer.has(id))
@@ -310,7 +310,6 @@ _.extend(OplogObserveDriver.prototype, {
     } else if (cachedBefore && !matchesNow) {
       self._removeMatching(id);
     } else if (cachedBefore && matchesNow) {
-      delete newDoc._id;
       var oldDoc = self._published.get(id);
       var comparator = self._comparator;
       var minBuffered = self._limit && self._unpublishedBuffer.size() &&
@@ -690,7 +689,6 @@ _.extend(OplogObserveDriver.prototype, {
 
     // Finally, replace the buffer
     newBuffer.forEach(function (doc, id) {
-      delete doc._id;
       self._addBuffered(id, doc);
     });
 

From a94bb6d88310008bbd1c0c31f5a8b36f444f8909 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Tue, 25 Feb 2014 23:26:29 -0800
Subject: [PATCH 078/123] Allow appending to buffer if new doc is equivalent to
 the greatest element

---
 packages/mongo-livedata/oplog_observe_driver.js | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 8d422ac196..8400373390 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -210,10 +210,6 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._unpublishedBuffer.size() > self._limit) {
       var maxBufferedId = self._unpublishedBuffer.maxElementId();
 
-      if (EJSON.equals(maxBufferedId, id)) {
-        throw new Error("The document just added to buffer is overflowing the buffer");
-      }
-
       self._unpublishedBuffer.remove(maxBufferedId);
 
       // Since something matching is removed from cache (both published set and
@@ -253,7 +249,7 @@ _.extend(OplogObserveDriver.prototype, {
     // document would fit into published set pushing the maximum element out,
     // then we need to publish the doc.
     var toPublish = ! limit || self._published.size() < limit ||
-                    comparator(maxPublished, doc) > 0;
+                    comparator(doc, maxPublished) < 0;
 
     // Otherwise we might need to buffer it (only in case of limited query).
     // Buffering is allowed if the buffer is not filled up yet and all matching
@@ -264,7 +260,7 @@ _.extend(OplogObserveDriver.prototype, {
     // Or if it is small enough to be safely inserted to the middle or the
     // beginning of the buffer.
     var canInsertIntoBuffer = !toPublish && maxBuffered &&
-                              comparator(maxBuffered, doc) > 0;
+                              comparator(doc, maxBuffered) <= 0;
 
     var toBuffer = canAppendToBuffer || canInsertIntoBuffer;
 
@@ -336,7 +332,7 @@ _.extend(OplogObserveDriver.prototype, {
           var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId());
 
           var toBuffer = self._safeAppendToBuffer ||
-                         (maxBuffered && comparator(newDoc, maxBuffered) < 0);
+                         (maxBuffered && comparator(newDoc, maxBuffered) <= 0);
 
           if (toBuffer) {
             self._addBuffered(id, newDoc);
@@ -359,7 +355,7 @@ _.extend(OplogObserveDriver.prototype, {
 
         // or stays in buffer even after the change
         var staysInBuffer = (! toPublish && self._safeAppendToBuffer) ||
-          (!toPublish && maxBuffered && comparator(newDoc, maxBuffered) < 0);
+          (!toPublish && maxBuffered && comparator(newDoc, maxBuffered) <= 0);
 
         if (toPublish) {
           self._addPublished(id, newDoc);

From 6897d515e1ccb5313ae08705ff39077c5a54533e Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 13:34:24 -0800
Subject: [PATCH 079/123] Only set oplog-reply flag on the oplog collection

Otherwise can crash mongod!

Wed Feb 26 11:09:32.829 [conn499]  logs.tmp-test-logs-10833_meteor_com Assertion failure str::startsWith(ns, "local.oplog.") src/mongo/db/repl_block.cpp 261
0x10b9245ab 0x10b8fb64c 0x10b816015 0x10b5b2d06 0x10b714ce6 0x10b6bf354 0x10b6c398b 0x10b4dd9e2 0x10b917699 0x10b957055 0x7fff948637a2 0x7fff948501e1
I20140226-11:09:32.831(-8) (satellite.js:296) process 85850 for job tmp-test-logs-10833.meteor.com null exited with 1
 0   mongod                              0x000000010b9245ab _ZN5mongo15printStackTraceERSo + 43
 1   mongod                              0x000000010b8fb64c _ZN5mongo12verifyFailedEPKcS1_j + 284
 2   mongod                              0x000000010b816015 _ZN5mongo19updateSlaveLocationERNS_5CurOpEPKcNS_6OpTimeE + 2405
 3   mongod                              0x000000010b5b2d06 _ZN5mongo12ClientCursor19updateSlaveLocationERNS_5CurOpE + 52
 4   mongod                              0x000000010b714ce6 _ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRbPb + 534
 5   mongod                              0x000000010b6bf354 _ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE + 1492
 6   mongod                              0x000000010b6c398b _ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE + 4939
 7   mongod                              0x000000010b4dd9e2 _ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE + 198
 8   mongod                              0x000000010b917699 _ZN5mongo17PortMessageServer17handleIncomingMsgEPv + 1657
 9   mongod                              0x000000010b957055 thread_proxy + 229
 10  libsystem_c.dylib                   0x00007fff948637a2 _pthread_start + 327
 11  libsystem_c.dylib                   0x00007fff948501e1 thread_start + 13
---
 packages/mongo-livedata/mongo_driver.js  | 12 ++++++++----
 packages/mongo-livedata/oplog_tailing.js |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js
index b3469a09be..caf1c927fb 100644
--- a/packages/mongo-livedata/mongo_driver.js
+++ b/packages/mongo-livedata/mongo_driver.js
@@ -754,11 +754,15 @@ MongoConnection.prototype._createSynchronousCursor = function(
     // ... and to keep querying the server indefinitely rather than just 5 times
     // if there's no more data.
     mongoOptions.numberOfRetries = -1;
-    // And if this cursor specifies a 'ts', then set the undocumented oplog
-    // replay flag, which does a special scan to find the first document
-    // (instead of creating an index on ts).
-    if (cursorDescription.selector.ts)
+    // And if this is on the oplog collection and the cursor specifies a 'ts',
+    // then set the undocumented oplog replay flag, which does a special scan to
+    // find the first document (instead of creating an index on ts). This is a
+    // very hard-coded Mongo flag which only works on the oplog collection and
+    // only works with the ts field.
+    if (cursorDescription.collectionName === OPLOG_COLLECTION &&
+        cursorDescription.selector.ts) {
       mongoOptions.oplogReplay = true;
+    }
   }
 
   var dbCursor = collection.find(
diff --git a/packages/mongo-livedata/oplog_tailing.js b/packages/mongo-livedata/oplog_tailing.js
index b8c044ce3e..b203bfa412 100644
--- a/packages/mongo-livedata/oplog_tailing.js
+++ b/packages/mongo-livedata/oplog_tailing.js
@@ -1,6 +1,6 @@
 var Future = Npm.require('fibers/future');
 
-var OPLOG_COLLECTION = 'oplog.rs';
+OPLOG_COLLECTION = 'oplog.rs';
 var REPLSET_COLLECTION = 'system.replset';
 
 // Like Perl's quotemeta: quotes all regexp metacharacters. See

From 729f4123f1b8967302c6aef824593fcfa78ab71a Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 13:34:24 -0800
Subject: [PATCH 080/123] Only set oplog-reply flag on the oplog collection

Otherwise can crash mongod!

Wed Feb 26 11:09:32.829 [conn499]  logs.tmp-test-logs-10833_meteor_com Assertion failure str::startsWith(ns, "local.oplog.") src/mongo/db/repl_block.cpp 261
0x10b9245ab 0x10b8fb64c 0x10b816015 0x10b5b2d06 0x10b714ce6 0x10b6bf354 0x10b6c398b 0x10b4dd9e2 0x10b917699 0x10b957055 0x7fff948637a2 0x7fff948501e1
I20140226-11:09:32.831(-8) (satellite.js:296) process 85850 for job tmp-test-logs-10833.meteor.com null exited with 1
 0   mongod                              0x000000010b9245ab _ZN5mongo15printStackTraceERSo + 43
 1   mongod                              0x000000010b8fb64c _ZN5mongo12verifyFailedEPKcS1_j + 284
 2   mongod                              0x000000010b816015 _ZN5mongo19updateSlaveLocationERNS_5CurOpEPKcNS_6OpTimeE + 2405
 3   mongod                              0x000000010b5b2d06 _ZN5mongo12ClientCursor19updateSlaveLocationERNS_5CurOpE + 52
 4   mongod                              0x000000010b714ce6 _ZN5mongo14processGetMoreEPKcixRNS_5CurOpEiRbPb + 534
 5   mongod                              0x000000010b6bf354 _ZN5mongo15receivedGetMoreERNS_10DbResponseERNS_7MessageERNS_5CurOpE + 1492
 6   mongod                              0x000000010b6c398b _ZN5mongo16assembleResponseERNS_7MessageERNS_10DbResponseERKNS_11HostAndPortE + 4939
 7   mongod                              0x000000010b4dd9e2 _ZN5mongo16MyMessageHandler7processERNS_7MessageEPNS_21AbstractMessagingPortEPNS_9LastErrorE + 198
 8   mongod                              0x000000010b917699 _ZN5mongo17PortMessageServer17handleIncomingMsgEPv + 1657
 9   mongod                              0x000000010b957055 thread_proxy + 229
 10  libsystem_c.dylib                   0x00007fff948637a2 _pthread_start + 327
 11  libsystem_c.dylib                   0x00007fff948501e1 thread_start + 13
---
 packages/mongo-livedata/mongo_driver.js  | 12 ++++++++----
 packages/mongo-livedata/oplog_tailing.js |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js
index b3469a09be..caf1c927fb 100644
--- a/packages/mongo-livedata/mongo_driver.js
+++ b/packages/mongo-livedata/mongo_driver.js
@@ -754,11 +754,15 @@ MongoConnection.prototype._createSynchronousCursor = function(
     // ... and to keep querying the server indefinitely rather than just 5 times
     // if there's no more data.
     mongoOptions.numberOfRetries = -1;
-    // And if this cursor specifies a 'ts', then set the undocumented oplog
-    // replay flag, which does a special scan to find the first document
-    // (instead of creating an index on ts).
-    if (cursorDescription.selector.ts)
+    // And if this is on the oplog collection and the cursor specifies a 'ts',
+    // then set the undocumented oplog replay flag, which does a special scan to
+    // find the first document (instead of creating an index on ts). This is a
+    // very hard-coded Mongo flag which only works on the oplog collection and
+    // only works with the ts field.
+    if (cursorDescription.collectionName === OPLOG_COLLECTION &&
+        cursorDescription.selector.ts) {
       mongoOptions.oplogReplay = true;
+    }
   }
 
   var dbCursor = collection.find(
diff --git a/packages/mongo-livedata/oplog_tailing.js b/packages/mongo-livedata/oplog_tailing.js
index b8c044ce3e..b203bfa412 100644
--- a/packages/mongo-livedata/oplog_tailing.js
+++ b/packages/mongo-livedata/oplog_tailing.js
@@ -1,6 +1,6 @@
 var Future = Npm.require('fibers/future');
 
-var OPLOG_COLLECTION = 'oplog.rs';
+OPLOG_COLLECTION = 'oplog.rs';
 var REPLSET_COLLECTION = 'system.replset';
 
 // Like Perl's quotemeta: quotes all regexp metacharacters. See

From 0eccd1f2f49f674ba6e9f31a378fa17bc205d3df Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 13:51:38 -0800
Subject: [PATCH 081/123] Add History entries for 0.7.1.2 bug fixes

---
 History.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/History.md b/History.md
index c1e0c626b7..699137d47c 100644
--- a/History.md
+++ b/History.md
@@ -1,6 +1,16 @@
 ## v.NEXT
 
-## v0.7.1
+
+## v0.7.1.2
+
+* Fix bug in tool error handling that caused `meteor` to crash on Mac
+  OSX when no computer name is set.
+
+* Work around a bug that caused MongoDB to fail an assertion when using
+  tailable cursors on non-oplog collections.
+
+
+## v0.7.1.1
 
 * Integrate with Meteor developer accounts, a new way of managing your
   meteor.com deployed sites. When you use `meteor deploy`, you will be

From 3caa3b9e9c4cb1741034e07e4d50f300c4dbefaf Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 13:51:58 -0800
Subject: [PATCH 082/123] Update banner and notices for 0.7.1.2

---
 scripts/admin/banner.txt   | 7 ++-----
 scripts/admin/notices.json | 3 +++
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt
index a2c290ebd1..e672e9ae21 100644
--- a/scripts/admin/banner.txt
+++ b/scripts/admin/banner.txt
@@ -1,7 +1,4 @@
-=> Meteor 0.7.1: Extend oplog tailing driver to support most common
-   MongoDB queries.  Introduce Meteor developer accounts, a new way of
-   managing your meteor.com deployed sites. When you use `meteor
-   deploy`, you will be prompted to create a developer account.
+=> Meteor 0.7.1.2: Fix crash on OSX machines with no hostname set.
 
    This release is being downloaded in the background. Update your
-   project to Meteor 0.7.1 by running 'meteor update'.
+   project to Meteor 0.7.1.2 by running 'meteor update'.
diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json
index a0c2026c00..9c26c504a6 100644
--- a/scripts/admin/notices.json
+++ b/scripts/admin/notices.json
@@ -85,6 +85,9 @@
                  "http://jquery.com/upgrade-guide/1.9/"]
     }
   },
+  {
+    "release": "0.7.1.2"
+  },
   {
     "release": "NEXT"
   }

From e981b0c6297a0e57d3a39a27371af7eda1e338b0 Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 14:26:13 -0800
Subject: [PATCH 083/123] Bump run test timeouts

---
 tools/tests/run.js | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tools/tests/run.js b/tools/tests/run.js
index db0ff667e4..45883021c1 100644
--- a/tools/tests/run.js
+++ b/tools/tests/run.js
@@ -55,6 +55,7 @@ selftest.define("run", function () {
   run.waitSecs(5);
   run.match("restarted (x2)"); // see that restart counter reset
   s.write("crash.js", "process.kill(process.pid, 'SIGKILL');");
+  run.waitSecs(5);
   run.match("from signal: SIGKILL");
   run.waitSecs(5);
   run.match("is crashing");
@@ -82,6 +83,7 @@ selftest.define("run", function () {
 "  fs.writeFileSync(crashmark);\n" +
 "  process.exit(137);\n" +
 "}\n");
+  run.waitSecs(5);
   run.match("with code: 137");
   run.match("restarted");
   run.stop();

From de9bb4bfc4e6051988099bcd2356e72cf407582a Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 15:53:48 -0800
Subject: [PATCH 084/123] Change how we sanity-check _publishNewResults

The old version was just checking that docs *matched*, but we want the
stronger result that docs are *published*.
---
 .../mongo-livedata/oplog_observe_driver.js    | 21 ++++++++++++-------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 8400373390..5fb94ec4b1 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -287,15 +287,11 @@ _.extend(OplogObserveDriver.prototype, {
       self._removeBuffered(id);
     }
   },
-  _handleDoc: function (id, newDoc, mustMatchNow) {
+  _handleDoc: function (id, newDoc) {
     var self = this;
     newDoc = _.clone(newDoc);
 
     var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result;
-    if (mustMatchNow && !matchesNow) {
-      throw Error("expected " + EJSON.stringify(newDoc) + " to match "
-                  + EJSON.stringify(self._cursorDescription));
-    }
 
     var publishedBefore = self._published.has(id);
     var bufferedBefore = self._limit && self._unpublishedBuffer.has(id);
@@ -678,9 +674,18 @@ _.extend(OplogObserveDriver.prototype, {
     // If self has a buffer and limit, the new fetched result will be
     // limited correctly as the query has sort specifier.
     newResults.forEach(function (doc, id) {
-      // "true" here means to throw if we think this doc doesn't match the
-      // selector.
-      self._handleDoc(id, doc, true);
+      self._handleDoc(id, doc);
+    });
+
+    // Sanity-check that everything we tried to put into _published ended up
+    // there.
+    // XXX if this is slow, remove it later
+    if (self._published.size() !== newResults.size()) {
+      throw Error("failed to copy newResults into _published!");
+    }
+    self._published.forEach(function (doc, id) {
+      if (!newResults.has(id))
+        throw Error("_published has a doc that newResults doesn't; " + id);
     });
 
     // Finally, replace the buffer

From 078c00563ab212e2454b7c64b45378d9636b4413 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 15:54:37 -0800
Subject: [PATCH 085/123] Refactor an assertion in _removePublished

---
 .../mongo-livedata/oplog_observe_driver.js    | 48 ++++++++++++++-----
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 5fb94ec4b1..e91689002a 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -175,24 +175,50 @@ _.extend(OplogObserveDriver.prototype, {
     var self = this;
     self._published.remove(id);
     self._multiplexer.removed(id);
-    if (! self._limit)
+    if (! self._limit || self._published.size() === self._limit)
       return;
-    if (self._published.size() < self._limit) {
-      // The unpublished buffer is empty iff published contains the whole
-      // matching set, i.e. number of matching documents is less or equal to the
-      // queries limit.
-      if (! self._unpublishedBuffer.size()) {
-        // Assertion of the statement above
-        if (! self._safeAppendToBuffer && self._phase !== PHASE.QUERYING)
-          throw new Error("At this phase, buffer can be empty only if published contains the whole matching set");
-        return;
-      }
 
+    if (self._published.size() > self._limit)
+      throw Error("self._published got too big");
+
+    // OK, we are publishing less than the limit. Maybe we should look in the
+    // buffer to find the next element past what we were publishing before.
+
+    if (!self._unpublishedBuffer.empty()) {
+      // There's something in the buffer; move the first thing in it to
+      // _published.
       var newDocId = self._unpublishedBuffer.minElementId();
       var newDoc = self._unpublishedBuffer.get(newDocId);
       self._removeBuffered(newDocId);
       self._addPublished(newDocId, newDoc);
+      return;
     }
+
+    // There's nothing in the buffer.  This could mean one of a few things.
+
+    // (a) We could be in the middle of re-running the query (specifically, we
+    // could be in _publishNewResults). In that case, _unpublishedBuffer is
+    // empty because we clear it at the beginning of _publishNewResults. In this
+    // case, our caller already knows the entire answer to the query and we
+    // don't need to do anything fancy here.  Just return.
+    if (self._phase === PHASE.QUERYING)
+      return;
+
+    // (b) We're pretty confident that the union of _published and
+    // _unpublishedBuffer contain all documents that match selector. Because
+    // _unpublishedBuffer is empty, that means we're confident that _published
+    // contains all documents that match selector. So we have nothing to do.
+    if (self._safeAppendToBuffer)
+      return;
+
+    // (c) Maybe there are other documents out there that should be in our
+    // buffer. But in that case, when we emptied _unpublishedBuffer in
+    // _removeBuffered, we should have called _needToPollQuery, which will
+    // either put something in _unpublishedBuffer or set _safeAppendToBuffer (or
+    // both), and it will put us in QUERYING for that whole time. So in fact, we
+    // shouldnt' be able to get here.
+
+    throw new Error("Buffer inexplicably empty");
   },
   _changePublished: function (id, oldDoc, newDoc) {
     var self = this;

From 15a4cac27c6bf41f24fecec0d4f764a56135009c Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 15:58:08 -0800
Subject: [PATCH 086/123] typo

---
 packages/mongo-livedata/oplog_observe_driver.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index e91689002a..b7f1c26db6 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -216,7 +216,7 @@ _.extend(OplogObserveDriver.prototype, {
     // _removeBuffered, we should have called _needToPollQuery, which will
     // either put something in _unpublishedBuffer or set _safeAppendToBuffer (or
     // both), and it will put us in QUERYING for that whole time. So in fact, we
-    // shouldnt' be able to get here.
+    // shouldn't be able to get here.
 
     throw new Error("Buffer inexplicably empty");
   },

From af08231656987d32699d9dfb1305930e373224c7 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Wed, 26 Feb 2014 16:32:19 -0800
Subject: [PATCH 087/123] When switching to QUERYING, stop what you're doing

We don't think there was a bug before but this should make things a
little more firm.
---
 .../mongo-livedata/oplog_observe_driver.js    | 80 ++++++++++++-------
 1 file changed, 53 insertions(+), 27 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index b7f1c26db6..701c9bde84 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -7,6 +7,20 @@ var PHASE = {
   STEADY: "STEADY"
 };
 
+// Exception thrown by _needToPollQuery which unrolls the stack up to the
+// enclosing call to finishIfNeedToPollQuery.
+var SwitchedToQuery = function () {};
+var finishIfNeedToPollQuery = function (f) {
+  return function () {
+    try {
+      f.apply(this, arguments);
+    } catch (e) {
+      if (!(e instanceof SwitchedToQuery))
+        throw e;
+    }
+  };
+};
+
 // OplogObserveDriver is an alternative to PollingObserveDriver which follows
 // the Mongo operation log instead of just re-polling the query. It obeys the
 // same simple interface: constructing it starts sending observeChanges
@@ -84,7 +98,7 @@ OplogObserveDriver = function (options) {
   forEachTrigger(self._cursorDescription, function (trigger) {
     self._stopHandles.push(self._mongoHandle._oplogHandle.onOplogEntry(
       trigger, function (notification) {
-        Meteor._noYieldsAllowed(function () {
+        Meteor._noYieldsAllowed(finishIfNeedToPollQuery(function () {
           var op = notification.op;
           if (notification.dropCollection) {
             // Note: this call is not allowed to block on anything (especially
@@ -98,7 +112,7 @@ OplogObserveDriver = function (options) {
             else
               self._handleOplogEntrySteadyOrFetching(op);
           }
-        });
+        }));
       }
     ));
   });
@@ -134,9 +148,9 @@ OplogObserveDriver = function (options) {
 
   // Give _observeChanges a chance to add the new ObserveHandle to our
   // multiplexer, so that the added calls get streamed.
-  Meteor.defer(function () {
+  Meteor.defer(finishIfNeedToPollQuery(function () {
     self._runInitialQuery();
-  });
+  }));
 };
 
 _.extend(OplogObserveDriver.prototype, {
@@ -398,7 +412,7 @@ _.extend(OplogObserveDriver.prototype, {
     self._registerPhaseChange(PHASE.FETCHING);
     // Defer, because nothing called from the oplog entry handler may yield, but
     // fetch() yields.
-    Meteor.defer(function () {
+    Meteor.defer(finishIfNeedToPollQuery(function () {
       while (!self._stopped && !self._needToFetch.empty()) {
         if (self._phase !== PHASE.FETCHING)
           throw new Error("phase in fetchModifiedDocuments: " + self._phase);
@@ -415,36 +429,40 @@ _.extend(OplogObserveDriver.prototype, {
           waiting++;
           self._mongoHandle._docFetcher.fetch(
             self._cursorDescription.collectionName, id, cacheKey,
-            function (err, doc) {
-              if (err) {
-                if (!anyError)
-                  anyError = err;
-              } else if (!self._stopped && self._phase === PHASE.FETCHING
-                         && self._fetchGeneration === thisGeneration) {
-                // We re-check the generation in case we've had an explicit
-                // _pollQuery call which should effectively cancel this round of
-                // fetches.  (_pollQuery increments the generation.)
-                self._handleDoc(id, doc);
+            finishIfNeedToPollQuery(function (err, doc) {
+              try {
+                if (err) {
+                  if (!anyError)
+                    anyError = err;
+                } else if (!self._stopped && self._phase === PHASE.FETCHING
+                           && self._fetchGeneration === thisGeneration) {
+                  // We re-check the generation in case we've had an explicit
+                  // _pollQuery call (eg, in another fiber) which should
+                  // effectively cancel this round of fetches.  (_pollQuery
+                  // increments the generation.)
+                  self._handleDoc(id, doc);
+                }
+              } finally {
+                waiting--;
+                // Because fetch() never calls its callback synchronously, this
+                // is safe (ie, we won't call fut.return() before the forEach is
+                // done).
+                if (waiting === 0)
+                  fut.return();
               }
-              waiting--;
-              // Because fetch() never calls its callback synchronously, this is
-              // safe (ie, we won't call fut.return() before the forEach is
-              // done).
-              if (waiting === 0)
-                fut.return();
-            });
+            }));
         });
         fut.wait();
         // XXX do this even if we've switched to PHASE.QUERYING?
         if (anyError)
           throw anyError;
-        // Exit now if we've had a _pollQuery call.
+        // Exit now if we've had a _pollQuery call (here or in another fiber).
         if (self._phase === PHASE.QUERYING)
           return;
         self._currentlyFetching = null;
       }
       self._beSteady();
-    });
+    }));
   },
   _beSteady: function () {
     var self = this;
@@ -579,7 +597,8 @@ _.extend(OplogObserveDriver.prototype, {
     ++self._fetchGeneration;  // ignore any in-flight fetches
     self._registerPhaseChange(PHASE.QUERYING);
 
-    // Defer so that we don't block.
+    // Defer so that we don't block.  We don't need finishIfNeedToPollQuery here
+    // because SwitchedToQuery is not called in QUERYING mode.
     Meteor.defer(function () {
       // subtle note: _published does not contain _id fields, but newResults
       // does
@@ -604,7 +623,14 @@ _.extend(OplogObserveDriver.prototype, {
   // ensures that we will query again later.
   //
   // This function may not block, because it is called from an oplog entry
-  // handler.
+  // handler. However, if we were not already in the QUERYING phase, it throws
+  // an exception that is caught by the closest surrounding
+  // finishIfNeedToPollQuery call; this ensures that we don't continue running
+  // close that was designed for another phase inside PHASE.QUERYING.
+  //
+  // (It's also necessary whenever logic in this file yields to check that other
+  // phases haven't put us into QUERYING mode, though; eg,
+  // _fetchModifiedDocuments does this.)
   _needToPollQuery: function () {
     var self = this;
     if (self._stopped)
@@ -614,7 +640,7 @@ _.extend(OplogObserveDriver.prototype, {
     // pausing FETCHING).
     if (self._phase !== PHASE.QUERYING) {
       self._pollQuery();
-      return;
+      throw new SwitchedToQuery;
     }
 
     // We're currently in QUERYING. Set a flag to ensure that we run another

From 7948c0f38b65cab02fae0f3369449d30d74e1b05 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 26 Feb 2014 17:09:16 -0800
Subject: [PATCH 088/123] Test for preserving fields needed for sort

---
 .../mongo-livedata/mongo_livedata_tests.js    | 70 +++++++++++++++++--
 1 file changed, 63 insertions(+), 7 deletions(-)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index 01c5b54b52..f3ee3dcac6 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -746,6 +746,13 @@ if (Meteor.isServer) {
     x++;
   });
 
+  // compares arrays a and b w/o looking at order
+  var setsEqual = function (a, b) {
+    a = _.map(a, EJSON.stringify);
+    b = _.map(b, EJSON.stringify);
+    return _.isEmpty(_.difference(a, b)) && _.isEmpty(_.difference(b, a));
+  };
+
   // This test mainly checks the correctness of oplog code dealing with limited
   // queries. Compitablity with poll-diff is added as well.
   Tinytest.addAsync("mongo-livedata - observe sorted, limited " + idGeneration, function (test, onComplete) {
@@ -786,13 +793,6 @@ if (Meteor.isServer) {
         coll.update(sel, mod, opt);
       });
     };
-    // compares arrays a and b w/o looking at order
-    var setsEqual = function (a, b) {
-      a = _.map(a, EJSON.stringify);
-      b = _.map(b, EJSON.stringify);
-      return _.isEmpty(_.difference(a, b)) && _.isEmpty(_.difference(b, a));
-    };
-
     // tests '_id' subfields for all documents in oplog buffer
     var testOplogBufferIds = function (ids) {
       var bufferIds = [];
@@ -981,6 +981,62 @@ if (Meteor.isServer) {
     o.handle.stop();
     onComplete();
   });
+
+  Tinytest.addAsync("mongo-livedata - observe sorted, limited, sort fields " + idGeneration, function (test, onComplete) {
+    var run = test.runId();
+    var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
+
+    var observer = function () {
+      var state = {};
+      var output = [];
+      var callbacks = {
+        changed: function (newDoc) {
+          output.push({changed: newDoc._id});
+          state[newDoc._id] = newDoc;
+        },
+        added: function (newDoc) {
+          output.push({added: newDoc._id});
+          state[newDoc._id] = newDoc;
+        },
+        removed: function (oldDoc) {
+          output.push({removed: oldDoc._id});
+          delete state[oldDoc._id];
+        }
+      };
+      var handle = coll.find({}, {sort: {x: 1},
+                                  limit: 2,
+                                  fields: {y: 1}}).observe(callbacks);
+
+      return {output: output, handle: handle, state: state};
+    };
+    var clearOutput = function (o) { o.output.splice(0, o.output.length); };
+    var ins = function (doc) {
+      var id; runInFence(function () { id = coll.insert(doc); });
+      return id;
+    };
+
+    var o = observer();
+
+    var docId1 = ins({ x: 1, y: 1222 });
+    var docId2 = ins({ x: 5, y: 5222 });
+
+    test.length(o.output, 2);
+    test.equal(o.output, [{added: docId1}, {added: docId2}]);
+    clearOutput(o);
+
+    var docId3 = ins({ x: 7, y: 7222 });
+    var docId4 = ins({ x: -1, y: -1222 });
+
+    // Becomes [docId4 docId1 | docId2 docId3]
+    test.length(o.output, 2);
+    test.isTrue(setsEqual(o.output, [{added: docId4}, {removed: docId2}]));
+
+    test.equal(_.size(o.state), 2);
+    test.equal(o.state[docId4], {_id: docId4, y: -1222});
+    test.equal(o.state[docId1], {_id: docId1, y: 1222});
+
+    onComplete();
+  });
 }
 
 

From b0d86e535c394d1725f7b9ba6692e5587375325f Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 17:48:17 -0800
Subject: [PATCH 089/123] Update docs and examples

---
 docs/.meteor/release                 | 2 +-
 docs/lib/release-override.js         | 2 +-
 examples/leaderboard/.meteor/release | 2 +-
 examples/parties/.meteor/release     | 2 +-
 examples/todos/.meteor/release       | 2 +-
 examples/wordplay/.meteor/release    | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/.meteor/release b/docs/.meteor/release
index da0ead2e23..5a848a1d77 100644
--- a/docs/.meteor/release
+++ b/docs/.meteor/release
@@ -1 +1 @@
-0.7.1-rc3
+0.7.1.2
diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js
index 2ea0eedc8c..523bf17b11 100644
--- a/docs/lib/release-override.js
+++ b/docs/lib/release-override.js
@@ -1,5 +1,5 @@
 // While galaxy apps are on their own special meteor releases, override
 // Meteor.release here.
 if (Meteor.isClient) {
-  Meteor.release = Meteor.release ? "0.7.1" : undefined;
+  Meteor.release = Meteor.release ? "0.7.1.2" : undefined;
 }
diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release
index da0ead2e23..5a848a1d77 100644
--- a/examples/leaderboard/.meteor/release
+++ b/examples/leaderboard/.meteor/release
@@ -1 +1 @@
-0.7.1-rc3
+0.7.1.2
diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release
index da0ead2e23..5a848a1d77 100644
--- a/examples/parties/.meteor/release
+++ b/examples/parties/.meteor/release
@@ -1 +1 @@
-0.7.1-rc3
+0.7.1.2
diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release
index da0ead2e23..5a848a1d77 100644
--- a/examples/todos/.meteor/release
+++ b/examples/todos/.meteor/release
@@ -1 +1 @@
-0.7.1-rc3
+0.7.1.2
diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release
index da0ead2e23..5a848a1d77 100644
--- a/examples/wordplay/.meteor/release
+++ b/examples/wordplay/.meteor/release
@@ -1 +1 @@
-0.7.1-rc3
+0.7.1.2

From b3548e54378433ff67e5dbf2b7cfc922290bda1b Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 26 Feb 2014 18:44:24 -0800
Subject: [PATCH 090/123] Combine sorter spec with projection

---
 packages/minimongo/package.js             |  3 ++-
 packages/minimongo/selector_projection.js | 20 ++++++++++++--------
 packages/minimongo/sort.js                |  2 +-
 packages/minimongo/sorter_projection.js   | 19 +++++++++++++++++++
 4 files changed, 34 insertions(+), 10 deletions(-)
 create mode 100644 packages/minimongo/sorter_projection.js

diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js
index 58783070fb..4dca1e8433 100644
--- a/packages/minimongo/package.js
+++ b/packages/minimongo/package.js
@@ -28,7 +28,8 @@ Package.on_use(function (api) {
   // Functionality used only by oplog tailing on the server side
   api.add_files([
     'selector_projection.js',
-    'selector_modifier.js'
+    'selector_modifier.js',
+    'sorter_projection.js'
   ], 'server');
 });
 
diff --git a/packages/minimongo/selector_projection.js b/packages/minimongo/selector_projection.js
index 73a2727a8e..e7d7135587 100644
--- a/packages/minimongo/selector_projection.js
+++ b/packages/minimongo/selector_projection.js
@@ -12,12 +12,23 @@ Minimongo.Matcher.prototype.combineIntoProjection = function (projection) {
   if (_.contains(selectorPaths, ''))
     return {};
 
+  return combineImportantPathsIntoProjection(selectorPaths, projection);
+};
+
+Minimongo.Matcher.prototype._getPathsElidingNumericKeys = function () {
+  var self = this;
+  return _.map(self._getPaths(), function (path) {
+    return _.reject(path.split('.'), isNumericKey).join('.');
+  });
+};
+
+combineImportantPathsIntoProjection = function (paths, projection) {
   var prjDetails = projectionDetails(projection);
   var tree = prjDetails.tree;
   var mergedProjection = {};
 
   // merge the paths to include
-  tree = pathsToTree(selectorPaths,
+  tree = pathsToTree(paths,
                      function (path) { return true; },
                      function (node, path, fullPath) { return true; },
                      tree);
@@ -40,13 +51,6 @@ Minimongo.Matcher.prototype.combineIntoProjection = function (projection) {
   }
 };
 
-Minimongo.Matcher.prototype._getPathsElidingNumericKeys = function () {
-  var self = this;
-  return _.map(self._getPaths(), function (path) {
-    return _.reject(path.split('.'), isNumericKey).join('.');
-  });
-};
-
 // Returns a set of key paths similar to
 // { 'foo.bar': 1, 'a.b.c': 1 }
 var treeToPaths = function (tree, prefix) {
diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js
index bc74bd327f..e55e2af86f 100644
--- a/packages/minimongo/sort.js
+++ b/packages/minimongo/sort.js
@@ -38,7 +38,7 @@ Sorter = function (spec) {
       });
     }
   } else {
-    throw Error("Bad sort specification: ", JSON.stringify(spec));
+    throw Error("Bad sort specification: " + JSON.stringify(spec));
   }
 
   // reduceValue takes in all the possible values for the sort key along various
diff --git a/packages/minimongo/sorter_projection.js b/packages/minimongo/sorter_projection.js
new file mode 100644
index 0000000000..c9cad84862
--- /dev/null
+++ b/packages/minimongo/sorter_projection.js
@@ -0,0 +1,19 @@
+Sorter.combineSpecIntoProjection = function (spec, projection) {
+  var self = this;
+  var specPaths = getSortSpecPaths(spec);
+
+  return combineImportantPathsIntoProjection(specPaths, projection);
+};
+
+var getSortSpecPaths = function (spec) {
+  if (_.isArray(spec))
+    return _.map(spec, function (fieldSpec) {
+      return _.isArray(fieldSpec) ? fieldSpec[0] : fieldSpec;
+    });
+
+  if (_.isObject(spec))
+    return _.keys(spec);
+
+  throw new Error("Bad sort specification: " + JSON.stringify(spec));
+};
+

From 85ba085c78c3f3b185af063642a459bcab9c9007 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 26 Feb 2014 18:45:25 -0800
Subject: [PATCH 091/123] Oplog code takes sorter fields into shared projection

---
 packages/mongo-livedata/oplog_observe_driver.js | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 701c9bde84..7834a4cbb8 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -38,6 +38,8 @@ OplogObserveDriver = function (options) {
     throw Error("OplogObserveDriver only supports unordered observeChanges");
   }
 
+  var sortSpec = options.cursorDescription.options.sort;
+
   if (options.cursorDescription.options.limit) {
     // There are several properties ordered driver implements:
     // - _limit is a positive number
@@ -50,7 +52,7 @@ OplogObserveDriver = function (options) {
 
     // We don't support $near and other geo-queries so it's OK to initialize the
     // comparator only once in the constructor.
-    var sorter = new Minimongo.Sorter(options.cursorDescription.options.sort);
+    var sorter = new Minimongo.Sorter(sortSpec);
     var comparator = sorter.getComparator();
     var heapOptions = { IdMap: LocalCollection._IdMap };
     self._limit = self._cursorDescription.options.limit;
@@ -85,6 +87,8 @@ OplogObserveDriver = function (options) {
   // Projection function, result of combining important fields for selector and
   // existing fields projection
   self._sharedProjection = self._matcher.combineIntoProjection(projection);
+  if (sortSpec)
+    self._sharedProjection = Minimongo.Sorter.combineSpecIntoProjection(sortSpec, self._sharedProjection);
   self._sharedProjectionFn = LocalCollection._compileProjection(
     self._sharedProjection);
 

From 825082b3a81051f4c6d45fd698c6fbd4d8711da8 Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 23:49:18 -0800
Subject: [PATCH 092/123] Add missing semicolon in deploy-galaxy

---
 tools/deploy-galaxy.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js
index 6095545a7b..c0aa0fc808 100644
--- a/tools/deploy-galaxy.js
+++ b/tools/deploy-galaxy.js
@@ -78,7 +78,7 @@ var ServiceConnection = function (galaxy, service) {
   // from the hostname of endpointUrl, and run the login command for
   // that galaxy.
   if (! authToken)
-    throw new Error("not logged in to galaxy?")
+    throw new Error("not logged in to galaxy?");
 
   self.connection = Package.livedata.DDP.connect(endpointUrl, {
     headers: {

From 35c1a5fc457c6f2dd747875498420c959b719d0d Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Wed, 26 Feb 2014 23:57:18 -0800
Subject: [PATCH 093/123] Replace `subscribeAndWait` with `subscribe` inside
 `onReconnect`

---
 tools/deploy-galaxy.js | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js
index c0aa0fc808..945eef38ef 100644
--- a/tools/deploy-galaxy.js
+++ b/tools/deploy-galaxy.js
@@ -462,9 +462,17 @@ exports.logs = function (options) {
       var opts = { streaming: options.streaming };
       if (lastLogId)
         opts.resumeAfterId = lastLogId;
+      // Don't use subscribeAndWait here; it'll deadlock. We can't
+      // process the sub messages until `onReconnect` returns, and
+      // `onReconnect` won't return unless the sub messages have been
+      // processed. There's no reason we need to wait for the sub to be
+      // ready here anyway.
       // XXX correctly handle errors on resubscribe
-      logsSubscription = logReader.subscribeAndWait("logsForApp",
-                                                    options.app, opts);
+      logsSubscription = logReader.connection.subscribe(
+        "logsForApp",
+        options.app,
+        opts
+      );
     };
 
     try {

From 9d5782b9a064255a32c5b91e5193813dc8a73c59 Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Thu, 27 Feb 2014 07:56:19 -0800
Subject: [PATCH 094/123] Return the sub from
 `ServiceConnection.subscribeAndWait`

---
 tools/deploy-galaxy.js | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js
index 945eef38ef..d79ef34a31 100644
--- a/tools/deploy-galaxy.js
+++ b/tools/deploy-galaxy.js
@@ -151,8 +151,9 @@ _.extend(ServiceConnection.prototype, {
       }
     });
 
-    self.connection.subscribe.apply(self.connection, args);
-    return fut.wait();
+    var sub = self.connection.subscribe.apply(self.connection, args);
+    fut.wait();
+    return sub;
   },
 
   close: function () {

From bc4524b54410379be0455c19d0bd1668a3cb3c3b Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Thu, 27 Feb 2014 07:56:36 -0800
Subject: [PATCH 095/123] Set up onReconnect after initial sub on the
 connection to log-reader.

If we set it up before `subscribeAndWait` returns, then we'll end up
with two subscriptions; we don't have the log-reader sub yet, so we
can't stop it when `onReconnect` runs the first time, so we end up with
a redundant subscription. This means that if a real reconnect happens
later, we'll stop the sub that we set up inside `onReconnect`, but not
the initial sub, so we've leaked a sub and end up with duplicate
messages after reconnect.
---
 tools/deploy-galaxy.js | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js
index d79ef34a31..d879bb6871 100644
--- a/tools/deploy-galaxy.js
+++ b/tools/deploy-galaxy.js
@@ -457,7 +457,21 @@ exports.logs = function (options) {
       throw new Error("Can't listen to messages on the logs collection");
 
     var logsSubscription = null;
-    // In case of reconnect recover the state so user sees only new logs
+    try {
+      logsSubscription =
+        logReader.subscribeAndWait("logsForApp", options.app,
+                                   { streaming: options.streaming });
+    } catch (e) {
+      return handleError(e, galaxy, {
+        "no-such-app": "No such app: " + options.app
+      });
+    }
+
+    // In case of reconnect recover the state so user sees only new logs.
+    // Only set up the onReconnect handler after the subscribe and wait
+    // has returned; if we set it up before, then we'll end up with two
+    // subscriptions, because the onReconnect handler will run for the
+    // first time before the subscribeAndWait returns.
     logReader.connection.onReconnect = function () {
       logsSubscription && logsSubscription.stop();
       var opts = { streaming: options.streaming };
@@ -476,16 +490,6 @@ exports.logs = function (options) {
       );
     };
 
-    try {
-      logsSubscription =
-        logReader.subscribeAndWait("logsForApp", options.app,
-                                   { streaming: options.streaming });
-    } catch (e) {
-      return handleError(e, galaxy, {
-        "no-such-app": "No such app: " + options.app
-      });
-    }
-
     return options.streaming ? null : 0;
   } finally {
     // If not streaming, close the connection to log-reader so that

From 9c1dc8782cbd2c535261216073f0bf9a178dd42f Mon Sep 17 00:00:00 2001
From: Emily Stark <emily@meteor.com>
Date: Thu, 27 Feb 2014 08:09:31 -0800
Subject: [PATCH 096/123] Clean up ServiceConnection timer when we receive a
 result.

Previously, we could make a connection, do some method calls, and then
10 seconds later the connection happens to be dropped and the connection
timer fires, which not only throws an unexpected error into the future,
but also resolves the future twice. I think ServiceConnection is just
supposed to time out if you don't hear anything from the server within
10 seconds, so it now no longer times out if you hear things from the
server but then happen to be not connected when 10 seconds has elapsed.
---
 tools/deploy-galaxy.js | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js
index d879bb6871..53319a1379 100644
--- a/tools/deploy-galaxy.js
+++ b/tools/deploy-galaxy.js
@@ -117,10 +117,12 @@ _.extend(ServiceConnection.prototype, {
     var args = _.toArray(arguments);
     var name = args.shift();
     self.connection.apply(name, args, function (err, result) {
-      if (err)
+      if (err) {
         fut['throw'](err);
-      else
+      } else {
+        self._cleanUpTimer();
         fut['return'](result);
+      }
     });
 
     return fut.wait();
@@ -141,6 +143,7 @@ _.extend(ServiceConnection.prototype, {
     args.push({
       onReady: function () {
         ready = true;
+        self._cleanUpTimer();
         fut['return']();
       },
       onError: function (e) {
@@ -156,6 +159,13 @@ _.extend(ServiceConnection.prototype, {
     return sub;
   },
 
+  _cleanUpTimer: function () {
+    var self = this;
+    var Package = getPackage();
+    Package.meteor.Meteor.clearTimeout(self.connectionTimer);
+    self.connectionTimer = null;
+  },
+
   close: function () {
     var self = this;
     if (self.connection) {
@@ -164,9 +174,7 @@ _.extend(ServiceConnection.prototype, {
     }
     if (self.connectionTimer) {
       // Clean up the timer so that Node can exit cleanly
-      var Package = getPackage();
-      Package.meteor.Meteor.clearTimeout(self.connectionTimer);
-      self.connectionTimer = null;
+      self._cleanUpTimer();
     }
   }
 });

From d049bf75063e952e02b7bd719f575072147071e8 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 24 Feb 2014 20:22:49 -0800
Subject: [PATCH 097/123] Use faye-websocket for server-to-server DDP

This matches what the SockJS server uses; now we only need to understand
and fix bugs in the implementation of one websocket npm module.

Some notes:

  - I actually trust that it's possible to close a connection before it
    successfully connects, which allows me to simplify the code a
    lot (since there shouldn't be multiple connections active per
    ClientStream). I put in some assertions to make sure this is the
    case, though.  (Note that this module also has a simpler model,
    where there's a single object representing the client connection,
    not a "client" object that spawns "connections".)

  - We now print connect errors as well as post-connect errors.  (This
    required adding a flag to keep tests quiet since it makes an
    expected-to-fail-to-connect connection.)  We need a better approach
    to stream error handling, though.

  - We used to have a test to make sure that a certain not-user-visible
    callback is called within a Fiber; structuring the code such that
    this test is still possible would lead to the code being less
    consistent and harder to read, so I dropped the test.

  - Fix a few bugs where we weren't using Meteor.setTimeout.
---
 History.md                                    |   4 +
 .../livedata/.npm/package/npm-shrinkwrap.json |   9 +-
 packages/livedata/livedata_connection.js      |   5 +-
 packages/livedata/livedata_server.js          |   2 +-
 packages/livedata/livedata_tests.js           |   3 +-
 packages/livedata/package.js                  |   6 +-
 packages/livedata/stream_client_nodejs.js     | 194 ++++++++----------
 packages/livedata/stream_client_tests.js      |  37 ++--
 packages/retry/retry.js                       |   2 +-
 9 files changed, 127 insertions(+), 135 deletions(-)

diff --git a/History.md b/History.md
index 513267f88c..15d33efebc 100644
--- a/History.md
+++ b/History.md
@@ -1,5 +1,9 @@
 ## v.NEXT
 
+* Use "faye-websocket" (0.7.2) npm module instead of "websocket" (1.0.8) for
+  server-to-server DDP.
+
+
 ## v0.7.1.2
 
 * Fix bug in tool error handling that caused `meteor` to crash on Mac
diff --git a/packages/livedata/.npm/package/npm-shrinkwrap.json b/packages/livedata/.npm/package/npm-shrinkwrap.json
index f48e03a46b..92d8fd541c 100644
--- a/packages/livedata/.npm/package/npm-shrinkwrap.json
+++ b/packages/livedata/.npm/package/npm-shrinkwrap.json
@@ -16,8 +16,13 @@
         }
       }
     },
-    "websocket": {
-      "version": "1.0.8"
+    "faye-websocket": {
+      "version": "0.7.2",
+      "dependencies": {
+        "websocket-driver": {
+          "version": "0.3.2"
+        }
+      }
     }
   }
 }
diff --git a/packages/livedata/livedata_connection.js b/packages/livedata/livedata_connection.js
index 30ff1ef377..be5b79e15c 100644
--- a/packages/livedata/livedata_connection.js
+++ b/packages/livedata/livedata_connection.js
@@ -49,7 +49,10 @@ var Connection = function (url, options) {
     self._stream = new LivedataTest.ClientStream(url, {
       retry: options.retry,
       headers: options.headers,
-      _sockjsOptions: options._sockjsOptions
+      _sockjsOptions: options._sockjsOptions,
+      // To keep some tests quiet (because we don't have a real API for handling
+      // client-stream-level errors).
+      _dontPrintErrors: options._dontPrintErrors
     });
   }
 
diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js
index 95ddab9e6f..d579180911 100644
--- a/packages/livedata/livedata_server.js
+++ b/packages/livedata/livedata_server.js
@@ -1160,7 +1160,7 @@ _.extend(Server.prototype, {
       // drop all future data coming over this connection on the
       // floor. We don't want to confuse things.
       socket.removeAllListeners('data');
-      setTimeout(function () {
+      Meteor.setTimeout(function () {
         socket.send(stringifyDDP({msg: 'failed', version: version}));
         socket.close();
       }, timeout);
diff --git a/packages/livedata/livedata_tests.js b/packages/livedata/livedata_tests.js
index 65736d76f8..07bd42e779 100644
--- a/packages/livedata/livedata_tests.js
+++ b/packages/livedata/livedata_tests.js
@@ -683,9 +683,10 @@ if (Meteor.isServer) {
   testAsyncMulti("livedata - connect fails to unknown place", [
     function (test, expect) {
       var self = this;
-      self.conn = DDP.connect("example.com");
+      self.conn = DDP.connect("example.com", {_dontPrintErrors: true});
       Meteor.setTimeout(expect(function () {
         test.isFalse(self.conn.status().connected, "Not connected");
+        self.conn.close();
       }), 500);
     }
   ]);
diff --git a/packages/livedata/package.js b/packages/livedata/package.js
index d2b407ae13..0e585b8467 100644
--- a/packages/livedata/package.js
+++ b/packages/livedata/package.js
@@ -3,7 +3,11 @@ Package.describe({
   internal: true
 });
 
-Npm.depends({sockjs: "0.3.8", websocket: "1.0.8"});
+// We use 'faye-websocket' for connections in server-to-server DDP, mostly
+// because it's the same library used as a server in sockjs, and it's easiest to
+// deal with a single websocket implementation.  (Plus, its maintainer is easy
+// to work with on pull requests.)
+Npm.depends({sockjs: "0.3.8", "faye-websocket": "0.7.2"});
 
 Package.on_use(function (api) {
   api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps',
diff --git a/packages/livedata/stream_client_nodejs.js b/packages/livedata/stream_client_nodejs.js
index 708189deea..a3bc996d27 100644
--- a/packages/livedata/stream_client_nodejs.js
+++ b/packages/livedata/stream_client_nodejs.js
@@ -11,43 +11,16 @@
 // ping frames or with DDP-level messages.)
 LivedataTest.ClientStream = function (endpoint, options) {
   var self = this;
+  options = options || {};
+
   self.options = _.extend({
     retry: true
   }, options);
 
-  // WebSocket-Node https://github.com/Worlize/WebSocket-Node
-  // Chosen because it can run without native components. It has a
-  // somewhat idiosyncratic API. We may want to use 'ws' instead in the
-  // future.
-  //
-  // Since server-to-server DDP is still an experimental feature, we only
-  // require the module if we actually create a server-to-server
-  // connection. This is a minor efficiency improvement, but moreover: while
-  // 'websocket' doesn't require native components, it tries to use some
-  // optional native components and prints a warning if it can't load
-  // them. Since native components in packages don't work when transferred to
-  // other architectures yet, this means that require('websocket') prints a
-  // spammy log message when deployed to another architecture. Delaying the
-  // require means you only get the log message if you're actually using the
-  // feature.
-  self.client = new (Npm.require('websocket').client)();
+  self.client = null;  // created in _launchConnection
   self.endpoint = endpoint;
-  self.currentConnection = null;
 
-  options = options || {};
-  self.headers = options.headers || {};
-
-  self.client.on('connect', Meteor.bindEnvironment(
-    function (connection) {
-      return self._onConnect(connection);
-    },
-    "stream connect callback"
-  ));
-
-  self.client.on('connectFailed', function (error) {
-    // XXX: Make this do something better than make the tests hang if it does not work.
-    return self._lostConnection();
-  });
+  self.headers = self.options.headers || {};
 
   self._initCommon();
 
@@ -63,7 +36,7 @@ _.extend(LivedataTest.ClientStream.prototype, {
   send: function (data) {
     var self = this;
     if (self.currentStatus.connected) {
-      self.currentConnection.send(data);
+      self.client.send(data);
     }
   },
 
@@ -73,80 +46,37 @@ _.extend(LivedataTest.ClientStream.prototype, {
     self.endpoint = url;
   },
 
-  _onConnect: function (connection) {
+  _onConnect: function (client) {
     var self = this;
 
+    if (client !== self.client) {
+      // This connection is not from the last call to _launchConnection.
+      // But _launchConnection calls _cleanup which closes previous connections.
+      // It's our belief that this stifles future 'open' events, but maybe
+      // we are wrong?
+      throw new Error("Got open from inactive client");
+    }
+
     if (self._forcedToDisconnect) {
       // We were asked to disconnect between trying to open the connection and
       // actually opening it. Let's just pretend this never happened.
-      connection.close();
+      self.client.close();
+      self.client = null;
       return;
     }
 
     if (self.currentStatus.connected) {
-      // We already have a connection. It must have been the case that
-      // we started two parallel connection attempts (because we
-      // wanted to 'reconnect now' on a hanging connection and we had
-      // no way to cancel the connection attempt.) Just ignore/close
-      // the latecomer.
-      connection.close();
-      return;
+      // We already have a connection. It must have been the case that we
+      // started two parallel connection attempts (because we wanted to
+      // 'reconnect now' on a hanging connection and we had no way to cancel the
+      // connection attempt.) But this shouldn't happen (similarly to the client
+      // !== self.client check above).
+      throw new Error("Two parallel connections?");
     }
 
-    if (self.connectionTimer) {
-      clearTimeout(self.connectionTimer);
-      self.connectionTimer = null;
-    }
-
-    var onError = Meteor.bindEnvironment(
-      function (_this, error) {
-        if (self.currentConnection !== _this)
-          return;
-
-        Meteor._debug("stream error", error.toString(),
-                      (new Date()).toDateString());
-        self._lostConnection();
-      },
-      "stream error callback"
-    );
-
-    connection.on('error', function (error) {
-      // We have to pass in `this` explicitly because bindEnvironment
-      // doesn't propagate it for us.
-      onError(this, error);
-    });
-
-    var onClose = Meteor.bindEnvironment(
-      function (_this) {
-        if (self.options._testOnClose)
-          self.options._testOnClose();
-
-        if (self.currentConnection !== _this)
-          return;
-
-        self._lostConnection();
-      },
-      "stream close callback"
-    );
-
-    connection.on('close', function () {
-      // We have to pass in `this` explicitly because bindEnvironment
-      // doesn't propagate it for us.
-      onClose(this);
-    });
-
-    connection.on('message', function (message) {
-      if (self.currentConnection !== this)
-        return; // old connection still emitting messages
-
-      if (message.type === "utf8") // ignore binary frames
-        _.each(self.eventCallbacks.message, function (callback) {
-          callback(message.utf8Data);
-        });
-    });
+    self._clearConnectionTimer();
 
     // update status
-    self.currentConnection = connection;
     self.currentStatus.status = "connected";
     self.currentStatus.connected = true;
     self.currentStatus.retryCount = 0;
@@ -161,10 +91,10 @@ _.extend(LivedataTest.ClientStream.prototype, {
     var self = this;
 
     self._clearConnectionTimer();
-    if (self.currentConnection) {
-      var conn = self.currentConnection;
-      self.currentConnection = null;
-      conn.close();
+    if (self.client) {
+      var client = self.client;
+      self.client = null;
+      client.close();
     }
   },
 
@@ -181,25 +111,63 @@ _.extend(LivedataTest.ClientStream.prototype, {
     var self = this;
     self._cleanup(); // cleanup the old socket, if there was one.
 
-    // launch a connect attempt. we have no way to track it. we either
-    // get an _onConnect event, or we don't.
+    // Since server-to-server DDP is still an experimental feature, we only
+    // require the module if we actually create a server-to-server
+    // connection.
+    var FayeWebSocket = Npm.require('faye-websocket');
 
-    // XXX: set up a timeout on this.
+    // We would like to specify 'ddp' as the subprotocol here. The npm module we
+    // used to use as a client would fail the handshake if we ask for a
+    // subprotocol and the server doesn't send one back (and sockjs doesn't).
+    // Faye doesn't have that behavior; it's unclear from reading RFC 6455 if
+    // Faye is erroneous or not.  So for now, we don't specify protocols.
+    var client = self.client = new FayeWebSocket.Client(
+      toWebsocketUrl(self.endpoint),
+      [/*no subprotocols*/],
+      {headers: self.headers}
+    );
 
-    // we would like to specify 'ddp' as the protocol here, but
-    // unfortunately WebSocket-Node fails the handshake if we ask for
-    // a protocol and the server doesn't send one back (and sockjs
-    // doesn't). also, related: I guess we have to accept that
-    // 'stream' is ddp-specific
-    self.client.connect(toWebsocketUrl(self.endpoint),
-                        undefined, // protocols
-                        undefined, // origin
-                        self.headers);
-
-    if (self.connectionTimer)
-      clearTimeout(self.connectionTimer);
-    self.connectionTimer = setTimeout(
+    self._clearConnectionTimer();
+    self.connectionTimer = Meteor.setTimeout(
       _.bind(self._lostConnection, self),
       self.CONNECT_TIMEOUT);
+
+    self.client.on('open', Meteor.bindEnvironment(function () {
+      return self._onConnect(client);
+    }, "stream connect callback"));
+
+    var clientOnIfCurrent = function (event, description, f) {
+      self.client.on(event, Meteor.bindEnvironment(function () {
+        // Ignore events from any connection we've already cleaned up.
+        if (client !== self.client)
+          return;
+        f.apply(this, arguments);
+      }, description));
+    };
+
+    clientOnIfCurrent('error', 'stream error callback', function (error) {
+      if (!self.options._dontPrintErrors)
+        Meteor._debug("stream error", error.message);
+
+      // XXX: Make this do something better than make the tests hang if it does
+      // not work.
+      self._lostConnection();
+    });
+
+
+    clientOnIfCurrent('close', 'stream close callback', function () {
+      self._lostConnection();
+    });
+
+
+    clientOnIfCurrent('message', 'stream message callback', function (message) {
+      // Ignore binary frames, where message.data is a Buffer
+      if (typeof message.data !== "string")
+        return;
+
+      _.each(self.eventCallbacks.message, function (callback) {
+        callback(message.data);
+      });
+    });
   }
 });
diff --git a/packages/livedata/stream_client_tests.js b/packages/livedata/stream_client_tests.js
index dbb675852d..d9402a2157 100644
--- a/packages/livedata/stream_client_tests.js
+++ b/packages/livedata/stream_client_tests.js
@@ -1,17 +1,24 @@
 var Fiber = Npm.require('fibers');
 
-Tinytest.addAsync("stream client - callbacks run in a fiber", function (test, onComplete) {
-  stream = new LivedataTest.ClientStream(
-    Meteor.absoluteUrl(),
-    {
-      _testOnClose: function () {
-        test.isTrue(Fiber.current);
-        onComplete();
-      }
-    }
-  );
-  stream.on('reset', function () {
-    test.isTrue(Fiber.current);
-    stream.disconnect();
-  });
-});
+testAsyncMulti("stream client - callbacks run in a fiber", [
+  function (test, expect) {
+    var stream = new LivedataTest.ClientStream(Meteor.absoluteUrl());
+
+    var messageFired = false;
+    var resetFired = false;
+
+    stream.on('message', expect(function () {
+      test.isTrue(Fiber.current);
+      if (resetFired)
+        stream.disconnect();
+      messageFired = true;
+    }));
+
+    stream.on('reset', expect(function () {
+      test.isTrue(Fiber.current);
+      if (messageFired)
+        stream.disconnect();
+      resetFired = true;
+    }));
+  }
+]);
diff --git a/packages/retry/retry.js b/packages/retry/retry.js
index a4407b5bdb..4a377bca4f 100644
--- a/packages/retry/retry.js
+++ b/packages/retry/retry.js
@@ -57,7 +57,7 @@ _.extend(Retry.prototype, {
     var timeout = self._timeout(count);
     if (self.retryTimer)
       clearTimeout(self.retryTimer);
-    self.retryTimer = setTimeout(fn, timeout);
+    self.retryTimer = Meteor.setTimeout(fn, timeout);
     return timeout;
   }
 

From dcf806f852dcc6e9a6041b1d9a8f028abbf7e581 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Fri, 28 Feb 2014 11:52:44 -0800
Subject: [PATCH 098/123] typo in history

---
 History.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/History.md b/History.md
index 15d33efebc..f96b92f149 100644
--- a/History.md
+++ b/History.md
@@ -42,7 +42,7 @@
 * Add and improve support for minimongo operators.
   - Support `$comment`.
   - Support `obj` name in `$where`.
-  - `$regexp` matches actual regexps properly.
+  - `$regex` matches actual regexps properly.
   - Improve support for `$nin`, `$ne`, `$not`.
   - Support using `{ $in: [/foo/, /bar/] }`. #1707
   - Support `{$exists: false}`.

From 45db64d4bf8f994fa53f6d50a4f1f480c8eeaf93 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Fri, 28 Feb 2014 11:58:29 -0800
Subject: [PATCH 099/123] Support {a: {$regex: '', $options: 'i'}}

Fixes #1874.
---
 packages/minimongo/minimongo_tests.js | 3 +++
 packages/minimongo/selector.js        | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js
index ef2fed60b1..dd8dc6de94 100644
--- a/packages/minimongo/minimongo_tests.js
+++ b/packages/minimongo/minimongo_tests.js
@@ -678,6 +678,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$regex: 'a'}}, {a: 'cut'});
   nomatch({a: {$regex: 'a'}}, {a: 'CAT'});
   match({a: {$regex: 'a', $options: 'i'}}, {a: 'CAT'});
+  match({a: {$regex: '', $options: 'i'}}, {a: 'foo'});
+  nomatch({a: {$regex: '', $options: 'i'}}, {});
+  nomatch({a: {$regex: '', $options: 'i'}}, {a: 5});
   nomatch({a: /undefined/}, {});
   nomatch({a: {$regex: 'undefined'}}, {});
   nomatch({a: /xxx/}, {});
diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js
index 87abd1ca27..3b68a320f9 100644
--- a/packages/minimongo/selector.js
+++ b/packages/minimongo/selector.js
@@ -384,7 +384,7 @@ var VALUE_OPERATORS = {
   },
   // $options just provides options for $regex; its logic is inside $regex
   $options: function (operand, valueSelector) {
-    if (!valueSelector.$regex)
+    if (!_.has(valueSelector, '$regex'))
       throw Error("$options needs a $regex");
     return everythingMatcher;
   },

From 3ebc9aba60fea68a095aec6b4f2adb51ce46e6a4 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Fri, 28 Feb 2014 11:59:03 -0800
Subject: [PATCH 100/123] arguably support empty regexs in file watcher

---
 tools/watch.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/watch.js b/tools/watch.js
index f71cb7805c..a66d265be9 100644
--- a/tools/watch.js
+++ b/tools/watch.js
@@ -191,7 +191,7 @@ WatchSet.fromJSON = function (json) {
   set.files = _.clone(json.files);
 
   var reFromJSON = function (j) {
-    if (j.$regex)
+    if (_.has(j, '$regex'))
       return new RegExp(j.$regex, j.$options);
     return new RegExp(j);
   };

From a96243e746682559d8b03597108ebdaf4e3c43f1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 14:01:19 -0800
Subject: [PATCH 101/123] Sorter.combineIntoProjection is an instance method

Reuse existing sort spec parsing, store sort spec explicitly with key-paths. Elide numeric key in Sorter._getPaths
---
 packages/minimongo/selector_projection.js |  6 +++---
 packages/minimongo/sort.js                | 10 +++++++++-
 packages/minimongo/sorter_projection.js   | 17 ++---------------
 3 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/packages/minimongo/selector_projection.js b/packages/minimongo/selector_projection.js
index e7d7135587..5f6e101b5b 100644
--- a/packages/minimongo/selector_projection.js
+++ b/packages/minimongo/selector_projection.js
@@ -3,7 +3,7 @@
 // @returns Object - projection object (same as fields option of mongo cursor)
 Minimongo.Matcher.prototype.combineIntoProjection = function (projection) {
   var self = this;
-  var selectorPaths = self._getPathsElidingNumericKeys();
+  var selectorPaths = Minimongo._pathsElidingNumericKeys(self._getPaths());
 
   // Special case for $where operator in the selector - projection should depend
   // on all fields of the document. getSelectorPaths returns a list of paths
@@ -15,9 +15,9 @@ Minimongo.Matcher.prototype.combineIntoProjection = function (projection) {
   return combineImportantPathsIntoProjection(selectorPaths, projection);
 };
 
-Minimongo.Matcher.prototype._getPathsElidingNumericKeys = function () {
+Minimongo._pathsElidingNumericKeys = function (paths) {
   var self = this;
-  return _.map(self._getPaths(), function (path) {
+  return _.map(paths, function (path) {
     return _.reject(path.split('.'), isNumericKey).join('.');
   });
 };
diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js
index e55e2af86f..879d0e48c4 100644
--- a/packages/minimongo/sort.js
+++ b/packages/minimongo/sort.js
@@ -14,17 +14,19 @@
 Sorter = function (spec) {
   var self = this;
 
-  var sortSpecParts = [];
+  var sortSpecParts = self._sortSpecParts = [];
 
   if (spec instanceof Array) {
     for (var i = 0; i < spec.length; i++) {
       if (typeof spec[i] === "string") {
         sortSpecParts.push({
+          path: spec[i],
           lookup: makeLookupFunction(spec[i]),
           ascending: true
         });
       } else {
         sortSpecParts.push({
+          path: spec[i][0],
           lookup: makeLookupFunction(spec[i][0]),
           ascending: spec[i][1] !== "desc"
         });
@@ -33,6 +35,7 @@ Sorter = function (spec) {
   } else if (typeof spec === "object") {
     for (var key in spec) {
       sortSpecParts.push({
+        path: key,
         lookup: makeLookupFunction(key),
         ascending: spec[key] >= 0
       });
@@ -118,6 +121,11 @@ Sorter.prototype.getComparator = function (options) {
   }]);
 };
 
+Sorter.prototype._getPaths = function () {
+  var self = this;
+  return _.pluck(self._sortSpecParts, 'path');
+};
+
 Minimongo.Sorter = Sorter;
 
 // Given an array of comparators
diff --git a/packages/minimongo/sorter_projection.js b/packages/minimongo/sorter_projection.js
index c9cad84862..f02f388f7e 100644
--- a/packages/minimongo/sorter_projection.js
+++ b/packages/minimongo/sorter_projection.js
@@ -1,19 +1,6 @@
-Sorter.combineSpecIntoProjection = function (spec, projection) {
+Sorter.prototype.combineIntoProjection = function (projection) {
   var self = this;
-  var specPaths = getSortSpecPaths(spec);
-
+  var specPaths = Minimongo._pathsElidingNumericKeys(self._getPaths());
   return combineImportantPathsIntoProjection(specPaths, projection);
 };
 
-var getSortSpecPaths = function (spec) {
-  if (_.isArray(spec))
-    return _.map(spec, function (fieldSpec) {
-      return _.isArray(fieldSpec) ? fieldSpec[0] : fieldSpec;
-    });
-
-  if (_.isObject(spec))
-    return _.keys(spec);
-
-  throw new Error("Bad sort specification: " + JSON.stringify(spec));
-};
-

From c14e41a0d7440fd4f41f3b2201d8b5b6900d56b1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 14:22:30 -0800
Subject: [PATCH 102/123] Simple tests on Sorter.combineIntoProjection

---
 packages/minimongo/minimongo_server_tests.js | 25 ++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/packages/minimongo/minimongo_server_tests.js b/packages/minimongo/minimongo_server_tests.js
index efd7808b4c..127e30a8e6 100644
--- a/packages/minimongo/minimongo_server_tests.js
+++ b/packages/minimongo/minimongo_server_tests.js
@@ -332,6 +332,31 @@ Tinytest.add("minimongo - selector and projection combination", function (test)
 
 });
 
+Tinytest.add("minimongo - sorter and projection combination", function (test) {
+  function testSorterProjectionComb (sortSpec, proj, expected, desc) {
+    var sorter = new Minimongo.Sorter(sortSpec);
+    test.equal(sorter.combineIntoProjection(proj), expected, desc);
+  }
+
+  // Test with inclusive projection
+  testSorterProjectionComb({ a: 1, b: 1 }, { b: 1, c: 1, d: 1 }, { a: true, b: true, c: true, d: true }, "simplest incl");
+  testSorterProjectionComb({ a: 1, b: -1 }, { b: 1, c: 1, d: 1 }, { a: true, b: true, c: true, d: true }, "simplest incl");
+  testSorterProjectionComb({ 'a.c': 1 }, { b: 1 }, { 'a.c': true, b: true }, "dot path incl");
+  testSorterProjectionComb({ 'a.1.c': 1 }, { b: 1 }, { 'a.c': true, b: true }, "dot num path incl");
+  testSorterProjectionComb({ 'a.1.c': 1 }, { b: 1, a: 1 }, { a: true, b: true }, "dot num path incl overlap");
+  testSorterProjectionComb({ 'a.1.c': 1, 'a.2.b': -1 }, { b: 1 }, { 'a.c': true, 'a.b': true, b: true }, "dot num path incl");
+  testSorterProjectionComb({ 'a.1.c': 1, 'a.2.b': -1 }, {}, {}, "dot num path with empty incl");
+
+  // Test with exclusive projection
+  testSorterProjectionComb({ a: 1, b: 1 }, { b: 0, c: 0, d: 0 }, { c: false, d: false }, "simplest excl");
+  testSorterProjectionComb({ a: 1, b: -1 }, { b: 0, c: 0, d: 0 }, { c: false, d: false }, "simplest excl");
+  testSorterProjectionComb({ 'a.c': 1 }, { b: 0 }, { b: false }, "dot path excl");
+  testSorterProjectionComb({ 'a.1.c': 1 }, { b: 0 }, { b: false }, "dot num path excl");
+  testSorterProjectionComb({ 'a.1.c': 1 }, { b: 0, a: 0 }, { b: false }, "dot num path excl overlap");
+  testSorterProjectionComb({ 'a.1.c': 1, 'a.2.b': -1 }, { b: 0 }, { b: false }, "dot num path excl");
+});
+
+
 (function () {
   // TODO: Tests for "can selector become true by modifier" are incomplete,
   // absent or test the functionality of "not ideal" implementation (test checks

From 571ba01f16cb554e04bd54c538b591ad374e34b9 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 14:25:33 -0800
Subject: [PATCH 103/123] Use new sorter + projection api in oplog code

---
 packages/mongo-livedata/oplog_observe_driver.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 7834a4cbb8..3a12d5a5e1 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -39,6 +39,8 @@ OplogObserveDriver = function (options) {
   }
 
   var sortSpec = options.cursorDescription.options.sort;
+  var sorter = sortSpec && new Minimongo.Sorter(sortSpec);
+  var comparator = sorter && sorter.getComparator();
 
   if (options.cursorDescription.options.limit) {
     // There are several properties ordered driver implements:
@@ -52,8 +54,6 @@ OplogObserveDriver = function (options) {
 
     // We don't support $near and other geo-queries so it's OK to initialize the
     // comparator only once in the constructor.
-    var sorter = new Minimongo.Sorter(sortSpec);
-    var comparator = sorter.getComparator();
     var heapOptions = { IdMap: LocalCollection._IdMap };
     self._limit = self._cursorDescription.options.limit;
     self._comparator = comparator;
@@ -87,8 +87,8 @@ OplogObserveDriver = function (options) {
   // Projection function, result of combining important fields for selector and
   // existing fields projection
   self._sharedProjection = self._matcher.combineIntoProjection(projection);
-  if (sortSpec)
-    self._sharedProjection = Minimongo.Sorter.combineSpecIntoProjection(sortSpec, self._sharedProjection);
+  if (sorter)
+    self._sharedProjection = sorter.combineIntoProjection(self._sharedProjection);
   self._sharedProjectionFn = LocalCollection._compileProjection(
     self._sharedProjection);
 

From 29ebdbfeae0fef98ceb0e563da11b66384eaced1 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 14:34:44 -0800
Subject: [PATCH 104/123] Sophisticate the tests for oplog + limits + sorter

---
 .../mongo-livedata/mongo_livedata_tests.js    | 20 +++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index f3ee3dcac6..ea0349b1d8 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -1014,6 +1014,9 @@ if (Meteor.isServer) {
       var id; runInFence(function () { id = coll.insert(doc); });
       return id;
     };
+    var rem = function (id) {
+      runInFence(function () { coll.remove(id); });
+    };
 
     var o = observer();
 
@@ -1025,6 +1028,8 @@ if (Meteor.isServer) {
     clearOutput(o);
 
     var docId3 = ins({ x: 7, y: 7222 });
+    test.length(o.output, 0);
+
     var docId4 = ins({ x: -1, y: -1222 });
 
     // Becomes [docId4 docId1 | docId2 docId3]
@@ -1034,6 +1039,21 @@ if (Meteor.isServer) {
     test.equal(_.size(o.state), 2);
     test.equal(o.state[docId4], {_id: docId4, y: -1222});
     test.equal(o.state[docId1], {_id: docId1, y: 1222});
+    clearOutput(o);
+
+    rem(docId2);
+    // Becomes [docId4 docId1 | docId3]
+    test.length(o.output, 0);
+
+    rem(docId4);
+    // Becomes [docId1 docId3]
+    test.length(o.output, 2);
+    test.isTrue(setsEqual(o.output, [{added: docId3}, {removed: docId4}]));
+
+    test.equal(_.size(o.state), 2);
+    test.equal(o.state[docId3], {_id: docId3, y: 7222});
+    test.equal(o.state[docId1], {_id: docId1, y: 1222});
+    clearOutput(o);
 
     onComplete();
   });

From c615ec11efef3dd78c29b31339284fa48fc989fb Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 16:51:09 -0800
Subject: [PATCH 105/123] Move the comment

---
 packages/mongo-livedata/oplog_observe_driver.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 3a12d5a5e1..1743902156 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -40,6 +40,8 @@ OplogObserveDriver = function (options) {
 
   var sortSpec = options.cursorDescription.options.sort;
   var sorter = sortSpec && new Minimongo.Sorter(sortSpec);
+  // We don't support $near and other geo-queries so it's OK to initialize the
+  // comparator only once in the constructor.
   var comparator = sorter && sorter.getComparator();
 
   if (options.cursorDescription.options.limit) {
@@ -52,8 +54,6 @@ OplogObserveDriver = function (options) {
     //                      into published set.
     // - _published - Min Heap (also implements IdMap methods)
 
-    // We don't support $near and other geo-queries so it's OK to initialize the
-    // comparator only once in the constructor.
     var heapOptions = { IdMap: LocalCollection._IdMap };
     self._limit = self._cursorDescription.options.limit;
     self._comparator = comparator;

From 1fef6e5662c556bf6a294465b6583aa231060817 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Fri, 28 Feb 2014 18:18:09 -0800
Subject: [PATCH 106/123] Factor out published/buffer renewal into _runQuery

Fixes the bug when the buffer is empty on the first run if the size of query is more than limit*2
---
 .../mongo-livedata/oplog_observe_driver.js    | 52 +++++++++----------
 1 file changed, 24 insertions(+), 28 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 1743902156..7512fe74f4 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -552,19 +552,7 @@ _.extend(OplogObserveDriver.prototype, {
     if (self._stopped)
       throw new Error("oplog stopped surprisingly early");
 
-    // Query 2x documents as the half excluded from the original query will go
-    // into unpublished buffer to reduce additional Mongo lookups in cases when
-    // documents are removed from the published set and need a replacement.
-    // XXX needs more thought on non-zero skip
-    // XXX "2" here is a "magic number"
-    var initialCursor = self._cursorForQuery({ limit: self._limit * 2 });
-    var fetchedDocsCount = 0;
-    initialCursor.forEach(function (initialDoc) {
-      self._addMatching(initialDoc);
-      fetchedDocsCount++;
-    });
-
-    self._safeAppendToBuffer = fetchedDocsCount < self._limit * 2;
+    self._runQuery();
 
     if (self._stopped)
       throw new Error("oplog stopped quite early");
@@ -604,25 +592,33 @@ _.extend(OplogObserveDriver.prototype, {
     // Defer so that we don't block.  We don't need finishIfNeedToPollQuery here
     // because SwitchedToQuery is not called in QUERYING mode.
     Meteor.defer(function () {
-      // subtle note: _published does not contain _id fields, but newResults
-      // does
-      var newResults = new LocalCollection._IdMap;
-      var newBuffer = new LocalCollection._IdMap;
-      // XXX 2 is a "magic number" meaning there is an extra chunk of docs for
-      // buffer if such is needed.
-      var cursor = self._cursorForQuery({ limit: self._limit * 2 });
-      cursor.forEach(function (doc, i) {
-        if (!self._limit || i < self._limit)
-          newResults.set(doc._id, doc);
-        else
-          newBuffer.set(doc._id, doc);
-      });
-
-      self._publishNewResults(newResults, newBuffer);
+      self._runQuery();
       self._doneQuerying();
     });
   },
 
+  _runQuery: function () {
+    var self = this;
+    var newResults = new LocalCollection._IdMap;
+    var newBuffer = new LocalCollection._IdMap;
+
+    // Query 2x documents as the half excluded from the original query will go
+    // into unpublished buffer to reduce additional Mongo lookups in cases when
+    // documents are removed from the published set and need a replacement.
+    // XXX needs more thought on non-zero skip
+    // XXX 2 is a "magic number" meaning there is an extra chunk of docs for
+    // buffer if such is needed.
+    var cursor = self._cursorForQuery({ limit: self._limit * 2 });
+    cursor.forEach(function (doc, i) {
+      if (!self._limit || i < self._limit)
+        newResults.set(doc._id, doc);
+      else
+        newBuffer.set(doc._id, doc);
+    });
+
+    self._publishNewResults(newResults, newBuffer);
+  },
+
   // Transitions to QUERYING and runs another query, or (if already in QUERYING)
   // ensures that we will query again later.
   //

From 6c595e7b8d206bad7b4b3f482be5f1023d43904c Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Wed, 26 Feb 2014 21:53:07 -0800
Subject: [PATCH 107/123] A remove a redundant clone

---
 packages/mongo-livedata/oplog_observe_driver.js | 2 --
 1 file changed, 2 deletions(-)

diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js
index 7512fe74f4..5c83afdd6e 100644
--- a/packages/mongo-livedata/oplog_observe_driver.js
+++ b/packages/mongo-livedata/oplog_observe_driver.js
@@ -333,8 +333,6 @@ _.extend(OplogObserveDriver.prototype, {
   },
   _handleDoc: function (id, newDoc) {
     var self = this;
-    newDoc = _.clone(newDoc);
-
     var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result;
 
     var publishedBefore = self._published.has(id);

From bd05fdb2c5dba32529311369b263d4bbfacf1931 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Fri, 28 Feb 2014 23:12:36 -0800
Subject: [PATCH 108/123] minimongo: support semi-logical $elemMatch

ie, a document with some logical operator and some equality in an
elemMatch.

eg: {a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}}

Fixes #1875.
---
 History.md                            |  2 ++
 packages/minimongo/helpers.js         | 10 ++++++++--
 packages/minimongo/minimongo_tests.js | 16 ++++++++++++++++
 packages/minimongo/selector.js        |  2 +-
 4 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/History.md b/History.md
index f96b92f149..34de130d71 100644
--- a/History.md
+++ b/History.md
@@ -3,6 +3,8 @@
 * Use "faye-websocket" (0.7.2) npm module instead of "websocket" (1.0.8) for
   server-to-server DDP.
 
+* minimongo: Support {a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}}  #1875
+
 
 ## v0.7.1.2
 
diff --git a/packages/minimongo/helpers.js b/packages/minimongo/helpers.js
index d4046124f2..ab8cc78b6f 100644
--- a/packages/minimongo/helpers.js
+++ b/packages/minimongo/helpers.js
@@ -16,7 +16,10 @@ isIndexable = function (x) {
   return isArray(x) || isPlainObject(x);
 };
 
-isOperatorObject = function (valueSelector) {
+// Returns true if this is an object with at least one key and all keys begin
+// with $.  Unless inconsistentOK is set, throws if some keys begin with $ and
+// others don't.
+isOperatorObject = function (valueSelector, inconsistentOK) {
   if (!isPlainObject(valueSelector))
     return false;
 
@@ -26,7 +29,10 @@ isOperatorObject = function (valueSelector) {
     if (theseAreOperators === undefined) {
       theseAreOperators = thisIsOperator;
     } else if (theseAreOperators !== thisIsOperator) {
-      throw new Error("Inconsistent operator: " + valueSelector);
+      if (!inconsistentOK)
+        throw new Error("Inconsistent operator: " +
+                        JSON.stringify(valueSelector));
+      theseAreOperators = false;
     }
   });
   return !!theseAreOperators;  // {} has no operators
diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js
index dd8dc6de94..6c9359761d 100644
--- a/packages/minimongo/minimongo_tests.js
+++ b/packages/minimongo/minimongo_tests.js
@@ -820,6 +820,12 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({$or: [{a: 2}, {a: 3}], b: 2}, {a: 1, b: 2});
   nomatch({$or: [{a: 1}, {a: 2}], b: 3}, {a: 1, b: 2});
 
+  // Combining $or with equality
+  match({x: 1, $or: [{a: 1}, {b: 1}]}, {x: 1, b: 1});
+  match({$or: [{a: 1}, {b: 1}], x: 1}, {x: 1, b: 1});
+  nomatch({x: 1, $or: [{a: 1}, {b: 1}]}, {b: 1});
+  nomatch({x: 1, $or: [{a: 1}, {b: 1}]}, {x: 1});
+
   // $or and $lt, $lte, $gt, $gte
   match({$or: [{a: {$lte: 1}}, {a: 2}]}, {a: 1});
   nomatch({$or: [{a: {$lt: 1}}, {a: 2}]}, {a: 1});
@@ -1103,6 +1109,16 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$elemMatch: {x: 5}}}, {a: {x: 5}});
   match({a: {$elemMatch: {0: {$gt: 5, $lt: 9}}}}, {a: [[6]]});
   match({a: {$elemMatch: {'0.b': {$gt: 5, $lt: 9}}}}, {a: [[{b:6}]]});
+  match({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+        {a: [{x: 1, b: 1}]});
+  match({a: {$elemMatch: {$or: [{a: 1}, {b: 1}], x: 1}}},
+        {a: [{x: 1, b: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{b: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{x: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{x: 1}, {b: 1}]});
 
   // $comment
   match({a: 5, $comment: "asdf"}, {a: 5});
diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js
index 3b68a320f9..80636bb12d 100644
--- a/packages/minimongo/selector.js
+++ b/packages/minimongo/selector.js
@@ -653,7 +653,7 @@ var ELEMENT_OPERATORS = {
         throw Error("$elemMatch need an object");
 
       var subMatcher, isDocMatcher;
-      if (isOperatorObject(operand)) {
+      if (isOperatorObject(operand, true)) {
         subMatcher = compileValueSelector(operand, matcher);
         isDocMatcher = false;
       } else {

From 89a96084b864b2bdc64f16a95baa03f4796412bc Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Fri, 28 Feb 2014 23:15:33 -0800
Subject: [PATCH 109/123] history update

---
 History.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/History.md b/History.md
index 34de130d71..8d0ac759db 100644
--- a/History.md
+++ b/History.md
@@ -5,6 +5,7 @@
 
 * minimongo: Support {a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}}  #1875
 
+* minimongo: Support {a: {$regex: '', $options: 'i'}}  #1874
 
 ## v0.7.1.2
 

From e4c5f6ac97580fbb269c63549efd23a8e9b12447 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sat, 1 Mar 2014 01:13:00 -0800
Subject: [PATCH 110/123] Oplog limits test: big initial set, repolls,
 safeToAppend flag

---
 .../mongo-livedata/mongo_livedata_tests.js    | 107 ++++++++++++++++++
 1 file changed, 107 insertions(+)

diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js
index ea0349b1d8..f155b85fc5 100644
--- a/packages/mongo-livedata/mongo_livedata_tests.js
+++ b/packages/mongo-livedata/mongo_livedata_tests.js
@@ -1055,6 +1055,113 @@ if (Meteor.isServer) {
     test.equal(o.state[docId1], {_id: docId1, y: 1222});
     clearOutput(o);
 
+    onComplete();
+  });
+
+  Tinytest.addAsync("mongo-livedata - observe sorted, limited, big initial set" + idGeneration, function (test, onComplete) {
+    var run = test.runId();
+    var coll = new Meteor.Collection("observeLimit-"+run, collectionOptions);
+
+    var observer = function () {
+      var state = {};
+      var output = [];
+      var callbacks = {
+        changed: function (newDoc) {
+          output.push({changed: newDoc._id});
+          state[newDoc._id] = newDoc;
+        },
+        added: function (newDoc) {
+          output.push({added: newDoc._id});
+          state[newDoc._id] = newDoc;
+        },
+        removed: function (oldDoc) {
+          output.push({removed: oldDoc._id});
+          delete state[oldDoc._id];
+        }
+      };
+      var handle = coll.find({}, {sort: {x: 1, y: 1}, limit: 3})
+                    .observe(callbacks);
+
+      return {output: output, handle: handle, state: state};
+    };
+    var clearOutput = function (o) { o.output.splice(0, o.output.length); };
+    var ins = function (doc) {
+      var id; runInFence(function () { id = coll.insert(doc); });
+      return id;
+    };
+    var rem = function (id) {
+      runInFence(function () { coll.remove(id); });
+    };
+    // tests '_id' subfields for all documents in oplog buffer
+    var testOplogBufferIds = function (ids) {
+      var bufferIds = [];
+      o.handle._multiplexer._observeDriver._unpublishedBuffer.forEach(function (x, id) {
+        bufferIds.push(id);
+      });
+
+      test.isTrue(setsEqual(ids, bufferIds), "expected: " + ids + "; got: " + bufferIds);
+    };
+    var testSafeAppendToBufferFlag = function (expected) {
+      if (expected)
+        test.isTrue(o.handle._multiplexer._observeDriver._safeAppendToBuffer);
+      else
+        test.isFalse(o.handle._multiplexer._observeDriver._safeAppendToBuffer);
+    };
+
+    var ids = {};
+    _.each([2, 4, 1, 3, 5, 5, 9, 1, 3, 2, 5], function (x, i) {
+      ids[i] = ins({ x: x, y: i });
+    });
+
+    var o = observer();
+    var usesOplog = o.handle._multiplexer._observeDriver._usesOplog;
+    //  x: [1 1 2 | 2 3 3] 4 5 5 5  9
+    // id: [2 7 0 | 9 3 8] 1 4 5 10 6
+
+    test.length(o.output, 3);
+    test.isTrue(setsEqual([{added: ids[2]}, {added: ids[7]}, {added: ids[0]}], o.output));
+    usesOplog && testOplogBufferIds([ids[9], ids[3], ids[8]]);
+    usesOplog && testSafeAppendToBufferFlag(false);
+    clearOutput(o);
+
+    rem(ids[0]);
+    //  x: [1 1 2 | 3 3] 4 5 5 5  9
+    // id: [2 7 9 | 3 8] 1 4 5 10 6
+    test.length(o.output, 2);
+    test.isTrue(setsEqual([{removed: ids[0]}, {added: ids[9]}], o.output));
+    usesOplog && testOplogBufferIds([ids[3], ids[8]]);
+    usesOplog && testSafeAppendToBufferFlag(false);
+    clearOutput(o);
+
+    rem(ids[7]);
+    //  x: [1 2 3 | 3] 4 5 5 5  9
+    // id: [2 9 3 | 8] 1 4 5 10 6
+    test.length(o.output, 2);
+    test.isTrue(setsEqual([{removed: ids[7]}, {added: ids[3]}], o.output));
+    usesOplog && testOplogBufferIds([ids[8]]);
+    usesOplog && testSafeAppendToBufferFlag(false);
+    clearOutput(o);
+
+    rem(ids[3]);
+    //  x: [1 2 3 | 4 5 5] 5  9
+    // id: [2 9 8 | 1 4 5] 10 6
+    test.length(o.output, 2);
+    test.isTrue(setsEqual([{removed: ids[3]}, {added: ids[8]}], o.output));
+    usesOplog && testOplogBufferIds([ids[1], ids[4], ids[5]]);
+    usesOplog && testSafeAppendToBufferFlag(false);
+    clearOutput(o);
+
+    rem({ x: {$lt: 4} });
+    //  x: [4 5 5 | 5  9]
+    // id: [1 4 5 | 10 6]
+    test.length(o.output, 6);
+    test.isTrue(setsEqual([{removed: ids[2]}, {removed: ids[9]}, {removed: ids[8]},
+                           {added: ids[5]}, {added: ids[4]}, {added: ids[1]}], o.output));
+    usesOplog && testOplogBufferIds([ids[10], ids[6]]);
+    usesOplog && testSafeAppendToBufferFlag(true);
+    clearOutput(o);
+
+
     onComplete();
   });
 }

From f933b7f97f3b8ae36e75a368dbfc22de28d8e5c8 Mon Sep 17 00:00:00 2001
From: Slava Kim <slava@meteor.com>
Date: Sun, 2 Mar 2014 18:30:26 -0800
Subject: [PATCH 111/123] Make paths relative in .gitignore

Some tools like `ag` don't work well with absolute paths but work well with
relative ones. Considering that this project-wise '.gitignore' stays in root, it
will be OK to change the paths to relative.
---
 .gitignore | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index 245f622e04..eb2b69b25f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
 .DS_Store
-/.meteor
+./.meteor
 *~
-/dev_bundle
-/dev_bundle*.tar.gz
-/dist
+./dev_bundle
+./dev_bundle*.tar.gz
+./dist
 \#*\#
 .\#*
 .idea

From a2d73ca9bfcd319dec905f35f157d74a73328093 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 3 Mar 2014 15:35:43 -0800
Subject: [PATCH 112/123] Revert "Make paths relative in .gitignore"

This reverts commit f933b7f97f3b8ae36e75a368dbfc22de28d8e5c8.

This is sadly not how gitignore works.
---
 .gitignore | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.gitignore b/.gitignore
index eb2b69b25f..245f622e04 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,9 +1,9 @@
 .DS_Store
-./.meteor
+/.meteor
 *~
-./dev_bundle
-./dev_bundle*.tar.gz
-./dist
+/dev_bundle
+/dev_bundle*.tar.gz
+/dist
 \#*\#
 .\#*
 .idea

From 06267daa93f95b1be99c1e5685d9a3d7d9350974 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 3 Mar 2014 20:55:37 -0800
Subject: [PATCH 113/123] Comment: we only partially do AJAX crawling spec

Fixes #487
---
 packages/spiderable/spiderable.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/spiderable/spiderable.js b/packages/spiderable/spiderable.js
index 0a899f6efc..3f8f40bc2c 100644
--- a/packages/spiderable/spiderable.js
+++ b/packages/spiderable/spiderable.js
@@ -24,6 +24,13 @@ var REQUEST_TIMEOUT = 15*1000;
 var MAX_BUFFER = 5*1024*1024; // 5MB
 
 WebApp.connectHandlers.use(function (req, res, next) {
+  // _escaped_fragment_ comes from Google's AJAX crawling spec:
+  // https://developers.google.com/webmasters/ajax-crawling/docs/specification
+  // This spec was designed during the brief era where using "#!" URLs was
+  // common, so it mostly describes how to translate "#!" URLs into
+  // _escaped_fragment_ URLs. Since then, "#!" URLs have gone out of style, but
+  // the <meta name="fragment" content="!"> (see spiderable.html) approach also
+  // described in the spec is still common and used by several crawlers.
   if (/\?.*_escaped_fragment_=/.test(req.url) ||
       _.any(Spiderable.userAgentRegExps, function (re) {
         return re.test(req.headers['user-agent']); })) {

From e356c23fdbc114e3662e1938c4d91494bf8ad368 Mon Sep 17 00:00:00 2001
From: Tim Haines <tmhaines@gmail.com>
Date: Mon, 24 Feb 2014 12:08:30 -0800
Subject: [PATCH 114/123] Make factsByPackage data available to the server
 environment

---
 packages/facts/facts.js | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/packages/facts/facts.js b/packages/facts/facts.js
index 543023b271..d2f8d3f979 100644
--- a/packages/facts/facts.js
+++ b/packages/facts/facts.js
@@ -19,6 +19,9 @@ if (Meteor.isServer) {
   var factsByPackage = {};
   var activeSubscriptions = [];
 
+  // Make factsByPackage data available to the server environment
+  Facts._factsByPackage = factsByPackage;
+
   Facts.incrementServerFact = function (pkg, fact, increment) {
     if (!_.has(factsByPackage, pkg)) {
       factsByPackage[pkg] = {};

From f97b00f7ee7d2cb6592b4d69ead9cd68173c4b28 Mon Sep 17 00:00:00 2001
From: Nick Martin <nim@meteor.com>
Date: Mon, 3 Mar 2014 21:13:51 -0800
Subject: [PATCH 115/123] clean up example in docs. #1854

---
 docs/client/api.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/client/api.html b/docs/client/api.html
index 284f73fa90..ee5a806e5f 100644
--- a/docs/client/api.html
+++ b/docs/client/api.html
@@ -1585,7 +1585,7 @@ By default, the current user's `username`, `emails` and `profile` are
 published to the client. You can publish additional fields for the
 current user with:
 
-    Meteor.publish("userData", function () {
+    Meteor.publish(null, function () {
       return Meteor.users.find({_id: this.userId},
                                {fields: {'other': 1, 'things': 1}});
     });

From 51148c84df8285b0e44ed7bbcd53063fe61f9b18 Mon Sep 17 00:00:00 2001
From: rgould <rwgould@gmail.com>
Date: Mon, 24 Feb 2014 13:55:36 -0800
Subject: [PATCH 116/123] Update AmplifyJS from 1.1.0 to 1.1.2

---
 History.md                  |   4 +
 packages/amplify/amplify.js | 220 +++++++++++++++++++++---------------
 2 files changed, 135 insertions(+), 89 deletions(-)

diff --git a/History.md b/History.md
index 8d0ac759db..2e8be40749 100644
--- a/History.md
+++ b/History.md
@@ -7,6 +7,10 @@
 
 * minimongo: Support {a: {$regex: '', $options: 'i'}}  #1874
 
+* Upgraded dependencies
+  - amplify: 1.1.2 (from 1.1.0)
+
+
 ## v0.7.1.2
 
 * Fix bug in tool error handling that caused `meteor` to crash on Mac
diff --git a/packages/amplify/amplify.js b/packages/amplify/amplify.js
index 00252de12c..a7b584f122 100644
--- a/packages/amplify/amplify.js
+++ b/packages/amplify/amplify.js
@@ -1,19 +1,10 @@
 /*!
- * AmplifyJS 1.1.0 - Core, Store, Request
- * 
- * Copyright 2011 appendTo LLC. (http://appendto.com/team)
+ * Amplify 1.1.2
+ *
+ * Copyright 2011 - 2013 appendTo LLC. (http://appendto.com/team)
  * Dual licensed under the MIT or GPL licenses.
  * http://appendto.com/open-source-licenses
- * 
- * http://amplifyjs.com
- */
-/*!
- * Amplify Core 1.1.0
- * 
- * Copyright 2011 appendTo LLC. (http://appendto.com/team)
- * Dual licensed under the MIT or GPL licenses.
- * http://appendto.com/open-source-licenses
- * 
+ *
  * http://amplifyjs.com
  */
 (function( global, undefined ) {
@@ -23,6 +14,10 @@ var slice = [].slice,
 
 var amplify = global.amplify = {
 	publish: function( topic ) {
+		if ( typeof topic !== "string" ) {
+			throw new Error( "You must provide a valid topic to publish." );
+		}
+
 		var args = slice.call( arguments, 1 ),
 			topicSubscriptions,
 			subscription,
@@ -46,6 +41,10 @@ var amplify = global.amplify = {
 	},
 
 	subscribe: function( topic, context, callback, priority ) {
+		if ( typeof topic !== "string" ) {
+			throw new Error( "You must provide a valid topic to create a subscription." );
+		}
+
 		if ( arguments.length === 3 && typeof callback === "number" ) {
 			priority = callback;
 			callback = context;
@@ -67,14 +66,14 @@ var amplify = global.amplify = {
 			if ( !subscriptions[ topic ] ) {
 				subscriptions[ topic ] = [];
 			}
-	
+
 			var i = subscriptions[ topic ].length - 1,
 				subscriptionInfo = {
 					callback: callback,
 					context: context,
 					priority: priority
 				};
-	
+
 			for ( ; i >= 0; i-- ) {
 				if ( subscriptions[ topic ][ i ].priority <= priority ) {
 					subscriptions[ topic ].splice( i + 1, 0, subscriptionInfo );
@@ -91,7 +90,16 @@ var amplify = global.amplify = {
 		return callback;
 	},
 
-	unsubscribe: function( topic, callback ) {
+	unsubscribe: function( topic, context, callback ) {
+		if ( typeof topic !== "string" ) {
+			throw new Error( "You must provide a valid topic to remove a subscription." );
+		}
+
+		if ( arguments.length === 2 ) {
+			callback = context;
+			context = null;
+		}
+
 		if ( !subscriptions[ topic ] ) {
 			return;
 		}
@@ -101,26 +109,23 @@ var amplify = global.amplify = {
 
 		for ( ; i < length; i++ ) {
 			if ( subscriptions[ topic ][ i ].callback === callback ) {
-				subscriptions[ topic ].splice( i, 1 );
-				break;
+				if ( !context || subscriptions[ topic ][ i ].context === context ) {
+					subscriptions[ topic ].splice( i, 1 );
+					
+					// Adjust counter and length for removed item
+					i--;
+					length--;
+				}
 			}
 		}
 	}
 };
 
 }( this ) );
-/*!
- * Amplify Store - Persistent Client-Side Storage 1.1.0
- * 
- * Copyright 2011 appendTo LLC. (http://appendto.com/team)
- * Dual licensed under the MIT or GPL licenses.
- * http://appendto.com/open-source-licenses
- * 
- * http://amplifyjs.com
- */
+
 (function( amplify, undefined ) {
 
-var store = amplify.store = function( key, value, options, type ) {
+var store = amplify.store = function( key, value, options ) {
 	var type = store.type;
 	if ( options && options.type && options.type in store.types ) {
 		type = options.type;
@@ -141,9 +146,9 @@ store.addType = function( type, storage ) {
 		options.type = type;
 		return store( key, value, options );
 	};
-}
+};
 store.error = function() {
-	return "amplify.store quota exceeded"; 
+	return "amplify.store quota exceeded";
 };
 
 var rprefix = /^__amplify__/;
@@ -223,18 +228,21 @@ function createFromStorageInterface( storageType, storage ) {
 // localStorage + sessionStorage
 // IE 8+, Firefox 3.5+, Safari 4+, Chrome 4+, Opera 10.5+, iPhone 2+, Android 2+
 for ( var webStorageType in { localStorage: 1, sessionStorage: 1 } ) {
-	// try/catch for file protocol in Firefox
+	// try/catch for file protocol in Firefox and Private Browsing in Safari 5
 	try {
-		if ( window[ webStorageType ].getItem ) {
-			createFromStorageInterface( webStorageType, window[ webStorageType ] );
-		}
+		// Safari 5 in Private Browsing mode exposes localStorage
+		// but doesn't allow storing data, so we attempt to store and remove an item.
+		// This will unfortunately give us a false negative if we're at the limit.
+		window[ webStorageType ].setItem( "__amplify__", "x" );
+		window[ webStorageType ].removeItem( "__amplify__" );
+		createFromStorageInterface( webStorageType, window[ webStorageType ] );
 	} catch( e ) {}
 }
 
 // globalStorage
 // non-standard: Firefox 2+
 // https://developer.mozilla.org/en/dom/storage#globalStorage
-if ( window.globalStorage ) {
+if ( !store.types.localStorage && window.globalStorage ) {
 	// try/catch for file protocol in Firefox
 	try {
 		createFromStorageInterface( "globalStorage",
@@ -306,7 +314,9 @@ if ( window.globalStorage ) {
 		// http://www.w3.org/TR/REC-xml/#NT-Name
 		// simplified to assume the starting character is valid
 		// also removed colon as it is invalid in HTML attribute names
-		key = key.replace( /[^-._0-9A-Za-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u37f-\u1fff\u200c-\u200d\u203f\u2040\u2070-\u218f]/g, "-" );
+		key = key.replace( /[^\-._0-9A-Za-z\xb7\xc0-\xd6\xd8-\xf6\xf8-\u037d\u037f-\u1fff\u200c-\u200d\u203f\u2040\u2070-\u218f]/g, "-" );
+		// adjust invalid starting character to deal with our simplified sanitization
+		key = key.replace( /^-/, "_-" );
 
 		if ( value === undefined ) {
 			attr = div.getAttribute( key );
@@ -402,16 +412,11 @@ if ( window.globalStorage ) {
 }() );
 
 }( this.amplify = this.amplify || {} ) );
-/*!
- * Amplify Request 1.1.0
- * 
- * Copyright 2011 appendTo LLC. (http://appendto.com/team)
- * Dual licensed under the MIT or GPL licenses.
- * http://appendto.com/open-source-licenses
- * 
- * http://amplifyjs.com
- */
+
+/*global amplify*/
+
 (function( amplify, undefined ) {
+'use strict';
 
 function noop() {}
 function isFunction( obj ) {
@@ -457,12 +462,14 @@ amplify.request = function( resourceId, data, callback ) {
 		resource = amplify.request.resources[ settings.resourceId ],
 		success = settings.success || noop,
 		error = settings.error || noop;
+
 	settings.success = async( function( data, status ) {
 		status = status || "success";
 		amplify.publish( "request.success", settings, data, status );
 		amplify.publish( "request.complete", settings, data, status );
 		success( data, status );
 	});
+
 	settings.error = async( function( data, status ) {
 		status = status || "error";
 		amplify.publish( "request.error", settings, data, status );
@@ -506,13 +513,11 @@ amplify.request.define = function( resourceId, type, settings ) {
 }( amplify ) );
 
 
-
-
-
 (function( amplify, $, undefined ) {
+'use strict';
 
 var xhrProps = [ "status", "statusText", "responseText", "responseXML", "readyState" ],
-    rurlData = /\{([^\}]+)\}/g;
+		rurlData = /\{([^\}]+)\}/g;
 
 amplify.request.types.ajax = function( defnSettings ) {
 	defnSettings = $.extend({
@@ -520,7 +525,7 @@ amplify.request.types.ajax = function( defnSettings ) {
 	}, defnSettings );
 
 	return function( settings, request ) {
-		var xhr,
+		var xhr, handleResponse,
 			url = defnSettings.url,
 			abort = request.abort,
 			ajaxSettings = $.extend( true, {}, defnSettings, { data: settings.data } ),
@@ -537,7 +542,7 @@ amplify.request.types.ajax = function( defnSettings ) {
 					return xhr.getResponseHeader( key );
 				},
 				overrideMimeType: function( type ) {
-					return xhr.overrideMideType( type );
+					return xhr.overrideMimeType( type );
 				},
 				abort: function() {
 					aborted = true;
@@ -555,28 +560,7 @@ amplify.request.types.ajax = function( defnSettings ) {
 				}
 			};
 
-		amplify.publish( "request.ajax.preprocess",
-			defnSettings, settings, ajaxSettings, ampXHR );
-
-		$.extend( ajaxSettings, {
-			success: function( data, status ) {
-				handleResponse( data, status );
-			},
-			error: function( _xhr, status ) {
-				handleResponse( null, status );
-			},
-			beforeSend: function( _xhr, _ajaxSettings ) {
-				xhr = _xhr;
-				ajaxSettings = _ajaxSettings;
-				var ret = defnSettings.beforeSend ?
-					defnSettings.beforeSend.call( this, ampXHR, ajaxSettings ) : true;
-				return ret && amplify.publish( "request.before.ajax",
-					defnSettings, settings, ajaxSettings, ampXHR );
-			}
-		});
-		$.ajax( ajaxSettings );
-
-		function handleResponse( data, status ) {
+		handleResponse = function( data, status ) {
 			$.each( xhrProps, function( i, key ) {
 				try {
 					ampXHR[ key ] = xhr[ key ];
@@ -603,8 +587,62 @@ amplify.request.types.ajax = function( defnSettings ) {
 			// this can happen if a request is aborted
 			// TODO: figure out if this breaks polling or multi-part responses
 			handleResponse = $.noop;
+		};
+
+		amplify.publish( "request.ajax.preprocess",
+			defnSettings, settings, ajaxSettings, ampXHR );
+
+		$.extend( ajaxSettings, {
+			isJSONP: function () {
+				return (/jsonp/gi).test(this.dataType);
+			},
+			cacheURL: function () {
+				if (!this.isJSONP()) {
+					return this.url;
+				}
+
+				var callbackName = 'callback';
+
+				// possible for the callback function name to be overridden
+				if (this.hasOwnProperty('jsonp')) {
+					if (this.jsonp !== false) {
+						callbackName = this.jsonp;
+					} else {
+						if (this.hasOwnProperty('jsonpCallback')) {
+							callbackName = this.jsonpCallback;
+						}
+					}
+				}
+
+				// search and replace callback parameter in query string with empty string
+				var callbackRegex = new RegExp('&?' + callbackName + '=[^&]*&?', 'gi');
+				return this.url.replace(callbackRegex, '');
+			},
+			success: function( data, status ) {
+				handleResponse( data, status );
+			},
+			error: function( _xhr, status ) {
+				handleResponse( null, status );
+			},
+			beforeSend: function( _xhr, _ajaxSettings ) {
+				xhr = _xhr;
+				ajaxSettings = _ajaxSettings;
+				var ret = defnSettings.beforeSend ?
+					defnSettings.beforeSend.call( this, ampXHR, ajaxSettings ) : true;
+				return ret && amplify.publish( "request.before.ajax",
+					defnSettings, settings, ajaxSettings, ampXHR );
+			}
+		});
+
+		// cache all JSONP requests
+		if (ajaxSettings.cache && ajaxSettings.isJSONP()) {
+			$.extend(ajaxSettings, {
+				cache: true
+			});
 		}
 
+		$.ajax( ajaxSettings );
+
 		request.abort = function() {
 			ampXHR.abort();
 			abort.call( this );
@@ -626,8 +664,8 @@ amplify.subscribe( "request.ajax.preprocess", function( defnSettings, settings,
 
 	ajaxSettings.url = ajaxSettings.url.replace( rurlData, function ( m, key ) {
 		if ( key in data ) {
-		    mappedKeys.push( key );
-		    return data[ key ];
+			mappedKeys.push( key );
+			return data[ key ];
 		}
 	});
 
@@ -668,13 +706,9 @@ var cache = amplify.request.cache = {
 	_key: function( resourceId, url, data ) {
 		data = url + data;
 		var length = data.length,
-			i = 0,
-			checksum = chunk();
-
-		while ( i < length ) {
-			checksum ^= chunk();
-		}
+			i = 0;
 
+		/*jshint bitwise:false*/
 		function chunk() {
 			return data.charCodeAt( i++ ) << 24 |
 				data.charCodeAt( i++ ) << 16 |
@@ -682,6 +716,12 @@ var cache = amplify.request.cache = {
 				data.charCodeAt( i++ ) << 0;
 		}
 
+		var checksum = chunk();
+		while ( i < length ) {
+			checksum ^= chunk();
+		}
+		/*jshint bitwise:true*/
+
 		return "request-" + resourceId + "-" + checksum;
 	},
 
@@ -690,7 +730,7 @@ var cache = amplify.request.cache = {
 		return function( resource, settings, ajaxSettings, ampXHR ) {
 			// data is already converted to a string by the time we get here
 			var cacheKey = cache._key( settings.resourceId,
-					ajaxSettings.url, ajaxSettings.data ),
+					ajaxSettings.cacheURL(), ajaxSettings.data ),
 				duration = resource.cache;
 
 			if ( cacheKey in memoryStore ) {
@@ -715,7 +755,7 @@ if ( amplify.store ) {
 	$.each( amplify.store.types, function( type ) {
 		cache[ type ] = function( resource, settings, ajaxSettings, ampXHR ) {
 			var cacheKey = cache._key( settings.resourceId,
-					ajaxSettings.url, ajaxSettings.data ),
+					ajaxSettings.cacheURL(), ajaxSettings.data ),
 				cached = amplify.store[ type ]( cacheKey );
 
 			if ( cached ) {
@@ -723,7 +763,7 @@ if ( amplify.store ) {
 				return false;
 			}
 			var success = ampXHR.success;
-			ampXHR.success = function( data ) {	
+			ampXHR.success = function( data ) {
 				amplify.store[ type ]( cacheKey, data, { expires: resource.cache.expires } );
 				success.apply( this, arguments );
 			};
@@ -754,6 +794,8 @@ amplify.request.decoders = {
 		} else if ( data.status === "error" ) {
 			delete data.status;
 			error( data, "error" );
+		} else {
+			error( null, "error" );
 		}
 	}
 };
@@ -761,11 +803,11 @@ amplify.request.decoders = {
 amplify.subscribe( "request.before.ajax", function( resource, settings, ajaxSettings, ampXHR ) {
 	var _success = ampXHR.success,
 		_error = ampXHR.error,
-		decoder = $.isFunction( resource.decoder )
-			? resource.decoder
-			: resource.decoder in amplify.request.decoders
-				? amplify.request.decoders[ resource.decoder ]
-				: amplify.request.decoders._default;
+		decoder = $.isFunction( resource.decoder ) ?
+			resource.decoder :
+			resource.decoder in amplify.request.decoders ?
+				amplify.request.decoders[ resource.decoder ] :
+				amplify.request.decoders._default;
 
 	if ( !decoder ) {
 		return;

From c0447737ec725e316e5d1d543f797456b4db7524 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 3 Mar 2014 21:55:58 -0800
Subject: [PATCH 117/123] Fix less error handling during ast.toCSS

Fixes #1877
---
 packages/less/plugin/compile-less.js | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/packages/less/plugin/compile-less.js b/packages/less/plugin/compile-less.js
index fb8265abe7..a1e15445b5 100644
--- a/packages/less/plugin/compile-less.js
+++ b/packages/less/plugin/compile-less.js
@@ -25,10 +25,17 @@ Plugin.registerSourceHandler("less", function (compileStep) {
 
   var parser = new less.Parser(options);
   var astFuture = new Future;
-  var ast;
+  var sourceMap = null;
   try {
     parser.parse(source, astFuture.resolver());
-    ast = astFuture.wait();
+    var ast = astFuture.wait();
+
+    var css = ast.toCSS({
+      sourceMap: true,
+      writeSourceMap: function (sm) {
+        sourceMap = JSON.parse(sm);
+      }
+    });
   } catch (e) {
     // less.Parser.parse is supposed to report any errors via its
     // callback. But sometimes, it throws them instead. This is
@@ -42,14 +49,6 @@ Plugin.registerSourceHandler("less", function (compileStep) {
     return;
   }
 
-  var sourceMap = null;
-  var css = ast.toCSS({
-    sourceMap: true,
-    writeSourceMap: function (sm) {
-      sourceMap = JSON.parse(sm);
-    }
-  });
-
 
   if (sourceMap) {
     sourceMap.sources = [compileStep.inputPath];

From f079efe48b285bd5b75732e3bbd8b0347331df3d Mon Sep 17 00:00:00 2001
From: Tim Haines <tmhaines@gmail.com>
Date: Mon, 3 Mar 2014 08:49:38 -0800
Subject: [PATCH 118/123] Update the instructions for creating a Google Client
 ID

---
 packages/google/google_configure.html | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/packages/google/google_configure.html b/packages/google/google_configure.html
index bc03e7471f..11a91073f5 100644
--- a/packages/google/google_configure.html
+++ b/packages/google/google_configure.html
@@ -9,28 +9,22 @@
     <li>
       If necessary, "Create Project"
     <li>
-      Click "APIs &amp; auth" and "Registered apps" on the left
+      Click "APIs &amp; auth" and "Credentials" on the left
     </li>
     <li>
-      Click the "Register App" button
+      Click the "Create New Client ID" button
     </li>
     <li>
       Choose "Web application" as the type
     </li>
     <li>
-      Click "Register"
+      Set Authorized Javascript Origins to: <span class="url">{{siteUrl}}</span>
     </li>
     <li>
-      Expand the "OAuth 2.0 Client ID section"
+      Set Authorized Redirect URI to: <span class="url">{{siteUrl}}_oauth/google?close</span>
     </li>
     <li>
-      Set Web Origin to: <span class="url">{{siteUrl}}</span>
-    </li>
-    <li>
-      Set Redirect URI to: <span class="url">{{siteUrl}}_oauth/google?close</span>
-    </li>
-    <li>
-      Click "Generate"
+      Click "Create Client ID"
     </li>
   </ol>
 </template>

From f9c947a7497de122288f2db2f5bd429a5c5f47e9 Mon Sep 17 00:00:00 2001
From: Nick Martin <nim@meteor.com>
Date: Mon, 3 Mar 2014 22:17:32 -0800
Subject: [PATCH 119/123] Print when we're listening, for use in automated
 tests. Fixes #1884.

---
 packages/test-in-console/reporter.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/test-in-console/reporter.js b/packages/test-in-console/reporter.js
index ef7e83734a..5f12d912f9 100644
--- a/packages/test-in-console/reporter.js
+++ b/packages/test-in-console/reporter.js
@@ -26,3 +26,10 @@ Meteor.methods({
     return null;
   }
 });
+
+// provide some notification we're started. This is to allow use
+// in automated scripts with `meteor run --once` which does not
+// print when the proxy is listening.
+Meteor.startup(function () {
+  Meteor._debug("test-in-console listening");
+});

From 8924417e23efb986984c5b77f5e1b6b8ea706591 Mon Sep 17 00:00:00 2001
From: Nick Martin <nim@meteor.com>
Date: Mon, 3 Mar 2014 22:41:33 -0800
Subject: [PATCH 120/123] More correct pattern for publishing user.

Long, but probably better to show the pedantically correct thing so users don't
get hurt if they copy, paste and modify.

Fixes #1885
---
 docs/client/api.html | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/docs/client/api.html b/docs/client/api.html
index ee5a806e5f..fc22971c38 100644
--- a/docs/client/api.html
+++ b/docs/client/api.html
@@ -1585,11 +1585,19 @@ By default, the current user's `username`, `emails` and `profile` are
 published to the client. You can publish additional fields for the
 current user with:
 
-    Meteor.publish(null, function () {
-      return Meteor.users.find({_id: this.userId},
-                               {fields: {'other': 1, 'things': 1}});
+    // server
+    Meteor.publish("userData", function () {
+      if (this.userId) {
+        return Meteor.users.find({_id: this.userId},
+                                 {fields: {'other': 1, 'things': 1}});
+      } else {
+        this.ready();
+      }
     });
 
+    // client
+    Meteor.subscribe("userData");
+
 If the autopublish package is installed, information about all users
 on the system is published to all clients. This includes `username`,
 `profile`, and any fields in `services` that are meant to be public

From a1fed0caaa8d0dcae70754130648611cb8d28f8d Mon Sep 17 00:00:00 2001
From: Nick Martin <nim@meteor.com>
Date: Mon, 3 Mar 2014 22:49:49 -0800
Subject: [PATCH 121/123] Comment only. Record idea about unblocking subs.

---
 packages/livedata/livedata_server.js | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js
index d579180911..84dc2bb68f 100644
--- a/packages/livedata/livedata_server.js
+++ b/packages/livedata/livedata_server.js
@@ -839,6 +839,13 @@ var Subscription = function (
 
 _.extend(Subscription.prototype, {
   _runHandler: function () {
+    // XXX should we unblock() here? Either before running the publish
+    // function, or before running _publishCursor.
+    //
+    // Right now, each publish function blocks all future publishes and
+    // methods waiting on data from Mongo (or whatever else the function
+    // blocks on). This probably slows page load in common cases.
+
     var self = this;
     try {
       var res = maybeAuditArgumentChecks(

From 6069da6427824d529759fbe733b972dbbb0e4ff5 Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 3 Mar 2014 23:15:19 -0800
Subject: [PATCH 122/123] Update facebook config instructions.

---
 packages/facebook/facebook_configure.html | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/packages/facebook/facebook_configure.html b/packages/facebook/facebook_configure.html
index 223b8adf2b..57285081e0 100644
--- a/packages/facebook/facebook_configure.html
+++ b/packages/facebook/facebook_configure.html
@@ -7,16 +7,21 @@
       Visit <a href="https://developers.facebook.com/apps" target="_blank">https://developers.facebook.com/apps</a>
     </li>
     <li>
-      Create New App (Only a name is required.)
+      Select "Apps", then "Create a New App". (You don't need to enter a namespace.)
     </li>
     <li>
-      Set "Sandbox Mode" to "Disabled"
-    </li>
-    <li>
-      Under "Select how your app integrates with Facebook", expand "Website with Facebook Login".
+      Select "Settings" and enter a "Contact Email".  Then select "Add Platform"
+      and choose "Website".
     </li>
     <li>
       Set Site URL to: <span class="url">{{siteUrl}}</span>
     </li>
+    <li>
+      Select "Status" and make the app and all its live features available to
+      the general public.
+    </li>
+    <li>
+      Select "Dashboard".
+    </li>
   </ol>
 </template>

From 0791b31b3015122a113f403db56af5b9545f330e Mon Sep 17 00:00:00 2001
From: David Glasser <glasser@meteor.com>
Date: Mon, 3 Mar 2014 21:38:37 -0800
Subject: [PATCH 123/123] dev bundle: fork node-http-proxy to fix #1871

---
 meteor                         | 2 +-
 scripts/generate-dev-bundle.sh | 4 +++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/meteor b/meteor
index 94e8ff0660..29fe3a330c 100755
--- a/meteor
+++ b/meteor
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BUNDLE_VERSION=0.3.32
+BUNDLE_VERSION=0.3.33
 
 # OS Check. Put here because here is where we download the precompiled
 # bundles that are arch specific.
diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh
index b1b16f1d45..942a092caf 100755
--- a/scripts/generate-dev-bundle.sh
+++ b/scripts/generate-dev-bundle.sh
@@ -107,9 +107,11 @@ npm install kexec@0.2.0
 npm install source-map@0.1.32
 npm install source-map-support@0.2.5
 npm install bcrypt@0.7.7
-npm install http-proxy@1.0.2
 npm install heapdump@0.2.5
 
+# Fork of 1.0.2 with https://github.com/nodejitsu/node-http-proxy/pull/592
+npm install https://github.com/meteor/node-http-proxy/tarball/99f757251b42aeb5d26535a7363c96804ee057f0
+
 # Using the unreleased 1.1 branch. We can probably switch to a built NPM version
 # when it gets released.
 npm install https://github.com/ariya/esprima/tarball/5044b87f94fb802d9609f1426c838874ec2007b3