Merge branch 'sso' into tool-refactoring

Conflicts:
	tools/auth.js
	tools/mongo_runner.js
	tools/tests/test-bundler-npm.js

History.md

@@ -1,3 +1,32 @@
+## v.NEXT
+
+* Hash login tokens before storing them in the database.
+
+* Cursors with a field specifier containing `{_id: 0}` can no longer be used
+  with `observeChanges` or `observe`. This includes the implicit calls to these
+  functions that are done when returning a cursor from a publish function or
+  using `{{#each}}`.
+
+* Patch Underscore to not treat plain objects (`x.constructor === Object`)
+  with numeric `length` fields as arrays.  Among other things, this allows you
+  to use documents with numeric `length` fields with Mongo.  #594 #1737
+
+* Fix races when calling login and/or logoutOtherClients from multiple
+  tabs. #1616
+
+* Upgrade `jquery-waypoints` package from 1.1.7 to 2.0.3. (Contains
+  backward-incompatible changes).
+
+## v0.7.0.1
+
+* Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the
+  message "Initializing mongo database... this may take a moment.".  #1696
+
+* Apply the Node patch to 0.10.24 as well (see the 0.7.0 section for details).
+
+* Fix gratuitous IE7 incompatibility.  #1690
+
+
 ## v0.7.0
 
 This version of Meteor contains a patch for a bug in Node 0.10 which
@@ -96,6 +125,8 @@ apply the patch and will instead disable websockets.
 * Increase the maximum size spiderable will return for a page from 200kB
   to 5MB.
 
+* New 'facts' package publishes internal statistics about Meteor.
+
 * Upgraded dependencies:
   * SockJS server from 0.3.7 to 0.3.8, including new faye-websocket module.
   * Node from 0.10.21 to 0.10.22

docs/client/api.html

@@ -586,23 +586,30 @@ access the same collection using the same API.
 
 Specifically, when you pass a `name`, here's what happens:
 
-* On the server, a collection with that name is created on a backend
-Mongo server. When you call methods on that collection on the server,
-they translate directly into normal Mongo operations (after checking that
-they match your [access control rules](#allow)).
+* On the server (if you do not specify a `connection`), a collection with that
+name is created on a backend Mongo server. When you call methods on that
+collection on the server, they translate directly into normal Mongo operations
+(after checking that they match your [access control rules](#allow)).
 
-* On the client, a Minimongo instance is
-created. Minimongo is essentially an in-memory, non-persistent
-implementation of Mongo in pure JavaScript. It serves as a local cache
-that stores just the subset of the database that this client is working
-with. Queries on the client ([`find`](#find)) are served directly out of
-this cache, without talking to the server.
+* On the client (and on the server if you specify a `connection`), a Minimongo
+instance is created. Minimongo is essentially an in-memory, non-persistent
+implementation of Mongo in pure JavaScript. It serves as a local cache that
+stores just the subset of the database that this client is working with. Queries
+([`find`](#find)) on these collections are served directly out of this cache,
+without talking to the server.
 
 * When you write to the database on the client ([`insert`](#insert),
-[`update`](#update), [`remove`](#remove)), the command is executed
-immediately on the client, and, simultaneously, it's shipped up to the
-server and executed there too.  The `livedata` package is
-responsible for this.
+[`update`](#update), [`remove`](#remove)), the command is executed locally
+immediately, and, simultaneously, it's sent to the server and executed
+there too. This happens via [stubs](#meteor_methods), because writes are
+implemented as methods.
+
+{{#note}}
+When, on the server, you write to a collection which has a specified
+`connection` to another server, it sends the corresponding method to the other
+server and receives the changed values back from it over DDP. Unlike on the
+client, it does not execute the write locally first.
+{{/note}}
 
 If you pass `null` as the `name`, then you're creating a local
 collection. It's not synchronized anywhere; it's just a local scratchpad
@@ -678,8 +685,6 @@ In this release, Minimongo has some limitations:
 
 * `$pull` in modifiers can only accept certain kinds
 of selectors.
-* `$` to denote the matched array position is not
-supported in modifier.
 * `findAndModify`, aggregate functions, and
 map/reduce aren't supported.
 
@@ -1319,21 +1324,32 @@ it's up to you to be sure.
 Queries can specify a particular set of fields to include or exclude from the
 result object.
 
-To exclude certain fields from the result objects, the field specifier
-is a dictionary whose keys are field names and whose values are `0`.
+To exclude specific fields from the result objects, the field specifier is a
+dictionary whose keys are field names and whose values are `0`. All unspecified
+fields are included.
 
     // Users.find({}, {fields: {password: 0, hash: 0}})
 
-To return an object that only includes the specified field, use `1` as
+To include only specific fields in the result documents, use `1` as
 the value. The `_id` field is still included in the result.
 
     // Users.find({}, {fields: {firstname: 1, lastname: 1}})
 
-It is not possible to mix inclusion and exclusion styles (except for the cases
-when `_id` is included by default or explicitly excluded). Field operators such
-as `$` and `$elemMatch` are not available on the client side yet.
+With one exception, it is not possible to mix inclusion and exclusion styles:
+the keys must either be all 1 or all 0.  The exception is that you may specify
+`_id: 0` in an inclusion specifier, which will leave `_id` out of the result
+object as well.  However, such field specifiers can not be used with
+[`observeChanges`](#observe_changes), [`observe`](#observe), cursors returned
+from a [publish function](#meteor_publish), or cursors used in
+`{{dstache}}#each}}` in a template. They may be used with [`fetch`](#fetch),
+[`findOne`](#findone), [`forEach`](#foreach), and [`map`](#map).
 
-More advanced example:
+
+<a href="http://docs.mongodb.org/manual/reference/operator/projection/">Field
+operators</a> such as `$` and `$elemMatch` are not available on the client side
+yet.
+
+A more advanced example:
 
     Users.insert({ alterEgos: [{ name: "Kira", alliance: "murderer" },
                                { name: "L", alliance: "police" }],
@@ -2632,9 +2648,10 @@ computation.
 
 {{> api_box deps_nonreactive }}
 
-Calls `func()` with `Deps.currentComputation` temporarily set to
-`null`.  If `func` accesses reactive data sources, these data sources
-will never cause a rerun of the enclosing computation.
+Calls `func` with `Deps.currentComputation` temporarily set to `null`
+and returns `func`'s own return value.  If `func` accesses reactive data
+sources, these data sources will never cause a rerun of the enclosing
+computation.
 
 {{> api_box deps_active }}
 

docs/client/concepts.html

@@ -558,8 +558,9 @@ database cursor is passed to `#each`, it will wire up all of the
 machinery to efficiently add and move DOM nodes as new results enter
 the query.
 
-Helpers can take arguments, and they receive the current template data
-in `this`:
+Helpers can take arguments, and they receive the current template context data
+in `this`. Note that some block helpers change the current context (notably
+`each` and `with`):
 
     // in a JavaScript file
     Template.players.leagueIs = function (league) {
@@ -825,10 +826,10 @@ To get started, run
 This command will generate a fully-contained Node.js application in the form of
 a tarball.  To run this application, you need to provide Node.js 0.10 and a
 MongoDB server.  (The current release of Meteor has been tested with Node
-0.10.21.) You can then run the application by invoking node, specifying the HTTP
-port for the application to listen on, and the MongoDB endpoint.  If you don't
-already have a MongoDB server, we can recommend our friends at
-[MongoHQ](http://mongohq.com).
+0.10.22, and is recommended for use with 0.10.22 through 0.10.24 only.) You can
+then run the application by invoking node, specifying the HTTP port for the
+application to listen on, and the MongoDB endpoint.  If you don't already have a
+MongoDB server, we can recommend our friends at [MongoHQ](http://mongohq.com).
 
     $ PORT=3000 MONGO_URL=mongodb://localhost:27017/myapp node bundle/main.js
 

docs/client/docs.js

@@ -47,10 +47,17 @@ Meteor.startup(function () {
   }
 
   var ignore_waypoints = false;
-  $('body').delegate('h1, h2, h3', 'waypoint.reached', function (evt, dir) {
+  var lastTimeout = null;
+  $('h1, h2, h3').waypoint(function (evt, dir) {
     if (!ignore_waypoints) {
       var active = (dir === "up") ? this.prev : this;
-      Session.set("section", active.id);
+      if (active.id) {
+        if (lastTimeout)
+          Meteor.clearTimeout(lastTimeout);
+        lastTimeout = Meteor.setTimeout(function () {
+          Session.set("section", active.id);
+        }, 200);
+      }
     }
   });
 

docs/client/packages/coffeescript.html

@@ -8,8 +8,8 @@ braces and parentheses.  The code compiles one-to-one into the
 equivalent JS, and there is no interpretation at runtime.
 
 CoffeeScript is supported on both the client and the server. Files
-ending with `.coffee` or `.litcoffee` are automatically compiled to
-JavaScript.
+ending with `.coffee`, `.litcoffee`, or `.coffee.md` are automatically
+compiled to JavaScript.
 
 ### Namespacing and CoffeeScript
 

docs/client/packages/underscore.html

@@ -18,6 +18,17 @@ if you do use underscore in your application, you should still add the
 package as we will remove the default underscore in the future.
 {{/warning}}
 
+{{#warning}}
+We have slightly modified the way Underscore differentiates between
+objects and arrays in [collection functions](http://underscorejs.org/#each).
+The original Underscore logic is to treat any object with a numeric `length`
+property as an array (which helps it work properly on
+[`NodeList`s](https://developer.mozilla.org/en-US/docs/Web/API/NodeList)).
+In Meteor's version of Underscore, objects with a numeric `length` property
+are treated as objects if they have no prototype (specifically, if
+`x.constructor === Object`.
+{{/warning}}
+
 
 {{/better_markdown}}
 </template>

docs/lib/release-override.js

@@ -1,5 +1,5 @@
 // While galaxy apps are on their own special meteor releases, override
 // Meteor.release here.
 if (Meteor.isClient) {
-  Meteor.release = Meteor.release ? "0.7.0" : undefined;
+  Meteor.release = Meteor.release ? "0.7.0.1" : undefined;
 }

examples/leaderboard/.meteor/release

@@ -1 +1 @@
-0.7.0
+0.7.0.1

examples/parties/.meteor/release

@@ -1 +1 @@
-0.7.0
+0.7.0.1

examples/todos/.meteor/release

@@ -1 +1 @@
-0.7.0
+0.7.0.1

examples/wordplay/.meteor/release

@@ -1 +1 @@
-0.7.0
+0.7.0.1

packages/accounts-base/accounts_client.js

@@ -116,8 +116,28 @@ Accounts.callLoginMethod = function (options) {
             // need to show a logging-in animation.
             _suppressLoggingIn: true,
             userCallback: function (error) {
+              var storedTokenNow = storedLoginToken();
               if (error) {
-                makeClientLoggedOut();
+                // If we had a login error AND the current stored token is the
+                // one that we tried to log in with, then declare ourselves
+                // logged out. If there's a token in storage but it's not the
+                // token that we tried to log in with, we don't know anything
+                // about whether that token is valid or not, so do nothing. The
+                // periodic localStorage poll will decide if we are logged in or
+                // out with this token, if it hasn't already. Of course, even
+                // with this check, another tab could insert a new valid token
+                // immediately before we clear localStorage here, which would
+                // lead to both tabs being logged out, but by checking the token
+                // in storage right now we hope to make that unlikely to happen.
+                //
+                // If there is no token in storage right now, we don't have to
+                // do anything; whatever code removed the token from storage was
+                // responsible for calling `makeClientLoggedOut()`, or the
+                // periodic localStorage poll will call `makeClientLoggedOut`
+                // eventually if another tab wiped the token from storage.
+                if (storedTokenNow && storedTokenNow === result.token) {
+                  makeClientLoggedOut();
+                }
               }
               // Possibly a weird callback to call, but better than nothing if
               // there is a reconnect between "login result received" and "data
@@ -195,25 +215,41 @@ Meteor.logout = function (callback) {
 };
 
 Meteor.logoutOtherClients = function (callback) {
-  // Our connection is going to be closed, but we don't want to call the
-  // onReconnect handler until the result comes back for this method, because
-  // the token will have been deleted on the server. Instead, wait until we get
-  // a new token and call the reconnect handler with that.
-  // XXX this is messy.
-  // XXX what if login gets called before the callback runs?
-  var origOnReconnect = Accounts.connection.onReconnect;
-  var userId = Meteor.userId();
-  Accounts.connection.onReconnect = null;
+  // Call the `logoutOtherClients` method. Store the login token that we get
+  // back and use it to log in again. The server is not supposed to close
+  // connections on the old token for 10 seconds, so we should have time to
+  // store our new token and log in with it before being disconnected. If we get
+  // disconnected, then we'll immediately reconnect with the new token. If for
+  // some reason we get disconnected before storing the new token, then the
+  // worst that will happen is that we'll have a flicker from trying to log in
+  // with the old token before storing and logging in with the new one.
   Accounts.connection.apply('logoutOtherClients', [], { wait: true },
                function (error, result) {
-                 Accounts.connection.onReconnect = origOnReconnect;
-                 if (! error)
+                 if (error) {
+                   callback && callback(error);
+                 } else {
+                   var userId = Meteor.userId();
                    storeLoginToken(userId, result.token, result.tokenExpires);
-                 Accounts.connection.onReconnect();
-                 callback && callback(error);
+                   // If the server hasn't disconnected us yet by deleting our
+                   // old token, then logging in now with the new valid token
+                   // will prevent us from getting disconnected. If the server
+                   // has already disconnected us due to our old invalid token,
+                   // then we would have already tried and failed to login with
+                   // the old token on reconnect, and we have to make sure a
+                   // login method gets sent here with the new token.
+                   Meteor.loginWithToken(result.token, function (err) {
+                     if (err &&
+                         storedLoginToken() &&
+                         storedLoginToken().token === result.token) {
+                       makeClientLoggedOut();
+                     }
+                     callback && callback(err);
+                   });
+                 }
                });
 };
 
+
 ///
 /// LOGIN SERVICES
 ///

packages/accounts-base/accounts_server.js

@@ -1,3 +1,5 @@
+var crypto = Npm.require('crypto');
+
 ///
 /// CURRENT USER
 ///
@@ -60,7 +62,7 @@ loginHandlers = [];
 //   { user: { username: <username> }, password: <password> }, or
 //   { user: { email: <email> }, password: <password> }.
 Accounts.createToken = function (options) {
-  // Try all of the registered login handlers until one of them doesn' return
+  // Try all of the registered login handlers until one of them doesn't return
   // `undefined`, meaning it handled this call to `login`. Return that return
   // value, which ought to be a {id/token} pair.
   for (var i = 0; i < loginHandlers.length; ++i) {
@@ -72,12 +74,23 @@ Accounts.createToken = function (options) {
   throw new Meteor.Error(400, "Unrecognized options for login request");
 };
 
-// Deletes the given loginToken from the database. This will cause all
-// connections associated with the token to be closed.
+// Deletes the given loginToken from the database.
+//
+// For new-style hashed token, this will cause all connections
+// associated with the token to be closed.
+//
+// Any connections associated with old-style unhashed tokens will be
+// in the process of becoming associated with hashed tokens and then
+// they'll get closed.
 Accounts.destroyToken = function (userId, loginToken) {
   Meteor.users.update(userId, {
     $pull: {
-      "services.resume.loginTokens": { "token": loginToken }
+      "services.resume.loginTokens": {
+        $or: [
+          { hashedToken: loginToken },
+          { token: loginToken }
+        ]
+      }
     }
   });
 };
@@ -104,7 +117,11 @@ Meteor.methods({
       // currently a public API for reading the login token on a
       // connection).
       Meteor._noYieldsAllowed(function () {
-        Accounts._setLoginToken(self.connection.id, result.token);
+        Accounts._setLoginToken(
+          result.id,
+          self.connection,
+          Accounts._hashLoginToken(result.token)
+        );
       });
       self.setUserId(result.id);
     }
@@ -113,7 +130,7 @@ Meteor.methods({
 
   logout: function() {
     var token = Accounts._getLoginToken(this.connection.id);
-    Accounts._setLoginToken(this.connection.id, null);
+    Accounts._setLoginToken(this.userId, this.connection, null);
     if (token && this.userId)
       Accounts.destroyToken(this.userId, token);
     this.setUserId(null);
@@ -146,7 +163,7 @@ Meteor.methods({
           "services.resume.loginTokensToDelete": tokens,
           "services.resume.haveLoginTokensToDelete": true
         },
-        $push: { "services.resume.loginTokens": newToken }
+        $push: { "services.resume.loginTokens": Accounts._hashStampedToken(newToken) }
       });
       Meteor.setTimeout(function () {
         // The observe on Meteor.users will take care of closing the connections
@@ -209,7 +226,23 @@ Meteor.server.onConnection(function (connection) {
 ///
 /// support reconnecting using a meteor login token
 
-// token -> list of connection ids
+Accounts._hashLoginToken = function (loginToken) {
+  var hash = crypto.createHash('sha256');
+  hash.update(loginToken);
+  return hash.digest('base64');
+};
+
+
+// {token, when} => {hashedToken, when}
+Accounts._hashStampedToken = function (stampedToken) {
+  return _.extend(
+    _.omit(stampedToken, 'token'),
+    {hashedToken: Accounts._hashLoginToken(stampedToken.token)}
+  );
+};
+
+
+// hashed token -> list of connection ids
 var connectionsByLoginToken = {};
 
 // test hook
@@ -217,7 +250,8 @@ Accounts._getTokenConnections = function (token) {
   return connectionsByLoginToken[token];
 };
 
-// Remove the connection from the list of open connections for the token.
+// Remove the connection from the list of open connections for the connection's
+// token.
 var removeConnectionFromToken = function (connectionId) {
   var token = Accounts._getLoginToken(connectionId);
   if (token) {
@@ -234,15 +268,43 @@ Accounts._getLoginToken = function (connectionId) {
   return Accounts._getAccountData(connectionId, 'loginToken');
 };
 
-Accounts._setLoginToken = function (connectionId, newToken) {
-  removeConnectionFromToken(connectionId);
-
-  Accounts._setAccountData(connectionId, 'loginToken', newToken);
+// newToken is a hashed token.
+Accounts._setLoginToken = function (userId, connection, newToken) {
+  removeConnectionFromToken(connection.id);
+  Accounts._setAccountData(connection.id, 'loginToken', newToken);
 
   if (newToken) {
     if (! _.has(connectionsByLoginToken, newToken))
       connectionsByLoginToken[newToken] = [];
-    connectionsByLoginToken[newToken].push(connectionId);
+    connectionsByLoginToken[newToken].push(connection.id);
+
+    // Now that we've added the connection to the
+    // connectionsByLoginToken map for the token, the connection will
+    // be closed if the token is removed from the database.  However
+    // at this point the token might have already been deleted, which
+    // wouldn't have closed the connection because it wasn't in the
+    // map yet.
+    //
+    // We also did need to first add the connection to the map above
+    // (and now remove it here if the token was deleted), because we
+    // could be getting a response from the database that the token
+    // still exists, but then it could be deleted in another fiber
+    // before our `findOne` call returns... and then that other fiber
+    // would need for the connection to be in the map for it to close
+    // the connection.
+    //
+    // We defer this check because there's no need for it to be on the critical
+    // path for login; we just need to ensure that the connection will get
+    // closed at some point if the token has been deleted.
+    Meteor.defer(function () {
+      if (! Meteor.users.findOne({
+        _id: userId,
+        "services.resume.loginTokens.hashedToken": newToken
+      })) {
+        removeConnectionFromToken(connection.id);
+        connection.close();
+      }
+    });
   }
 };
 
@@ -270,23 +332,84 @@ Accounts.registerLoginHandler(function(options) {
     return undefined;
 
   check(options.resume, String);
-  var user = Meteor.users.findOne({
-    "services.resume.loginTokens.token": ""+options.resume
-  });
 
-  if (!user) {
+  var hashedToken = Accounts._hashLoginToken(options.resume);
+
+  // First look for just the new-style hashed login token, to avoid
+  // sending the unhashed token to the database in a query if we don't
+  // need to.
+  var user = Meteor.users.findOne(
+    {"services.resume.loginTokens.hashedToken": hashedToken});
+
+  if (! user) {
+    // If we didn't find the hashed login token, try also looking for
+    // the old-style unhashed token.  But we need to look for either
+    // the old-style token OR the new-style token, because another
+    // client connection logging in simultaneously might have already
+    // converted the token.
+    user = Meteor.users.findOne({
+      $or: [
+        {"services.resume.loginTokens.hashedToken": hashedToken},
+        {"services.resume.loginTokens.token": options.resume}
+      ]
+    });
+  }
+
+  if (! user) {
     throw new Meteor.Error(403, "You've been logged out by the server. " +
     "Please login again.");
   }
 
+  // Find the token, which will either be an object with fields
+  // {hashedToken, when} for a hashed token or {token, when} for an
+  // unhashed token.
+  var oldUnhashedStyleToken;
   var token = _.find(user.services.resume.loginTokens, function (token) {
-    return token.token === options.resume;
+    return token.hashedToken === hashedToken;
   });
+  if (token) {
+    oldUnhashedStyleToken = false;
+  } else {
+    token = _.find(user.services.resume.loginTokens, function (token) {
+      return token.token === options.resume;
+    });
+    oldUnhashedStyleToken = true;
+  }
 
   var tokenExpires = Accounts._tokenExpiration(token.when);
   if (new Date() >= tokenExpires)
     throw new Meteor.Error(403, "Your session has expired. Please login again.");
 
+  // Update to a hashed token when an unhashed token is encountered.
+  if (oldUnhashedStyleToken) {
+    // Only add the new hashed token if the old unhashed token still
+    // exists (this avoids resurrecting the token if it was deleted
+    // after we read it).  Using $addToSet avoids getting an index
+    // error if another client logging in simultaneously has already
+    // inserted the new hashed token.
+    Meteor.users.update(
+      {
+        _id: user._id,
+        "services.resume.loginTokens.token": options.resume
+      },
+      {$addToSet: {
+        "services.resume.loginTokens": {
+          "hashedToken": hashedToken,
+          "when": token.when
+        }
+      }}
+    );
+
+    // Remove the old token *after* adding the new, since otherwise
+    // another client trying to login between our removing the old and
+    // adding the new wouldn't find a token to login with.
+    Meteor.users.update(user._id, {
+      $pull: {
+        "services.resume.loginTokens": { "token": options.resume }
+      },
+    });
+  }
+
   return {
     token: options.resume,
     tokenExpires: tokenExpires,
@@ -301,7 +424,6 @@ Accounts._generateStampedLoginToken = function () {
   return {token: Random.id(), when: (new Date)};
 };
 
-
 ///
 /// TOKEN EXPIRATION
 ///
@@ -402,11 +524,12 @@ Accounts.insertUserDoc = function (options, user) {
     var stampedToken = Accounts._generateStampedLoginToken();
     result.token = stampedToken.token;
     result.tokenExpires = Accounts._tokenExpiration(stampedToken.when);
+    var token = Accounts._hashStampedToken(stampedToken);
     Meteor._ensure(user, 'services', 'resume');
     if (_.has(user.services.resume, 'loginTokens'))
-      user.services.resume.loginTokens.push(stampedToken);
+      user.services.resume.loginTokens.push(token);
     else
-      user.services.resume.loginTokens = [stampedToken];
+      user.services.resume.loginTokens = [token];
   }
 
   var fullUser;
@@ -560,7 +683,7 @@ Accounts.updateOrCreateUserFromExternalService = function(
     Meteor.users.update(
       user._id,
       {$set: setAttrs,
-       $push: {'services.resume.loginTokens': stampedToken}});
+       $push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedToken)}});
     return {
       token: stampedToken.token,
       id: user._id,
@@ -714,6 +837,8 @@ Meteor.users.allow({
 /// DEFAULT INDEXES ON USERS
 Meteor.users._ensureIndex('username', {unique: 1, sparse: 1});
 Meteor.users._ensureIndex('emails.address', {unique: 1, sparse: 1});
+Meteor.users._ensureIndex('services.resume.loginTokens.hashedToken',
+                          {unique: 1, sparse: 1});
 Meteor.users._ensureIndex('services.resume.loginTokens.token',
                           {unique: 1, sparse: 1});
 // For taking care of logoutOtherClients calls that crashed before the tokens
@@ -762,8 +887,14 @@ Meteor.startup(function () {
 /// LOGGING OUT DELETED USERS
 ///
 
+// When login tokens are removed from the database, close any sessions
+// logged in with those tokens.
+//
+// Because we upgrade unhashed login tokens to hashed tokens at login
+// time, sessions will only be logged in with a hashed token.  Thus we
+// only need to pull out hashed tokens here.
 var closeTokensForUser = function (userTokens) {
-  closeConnectionsForTokens(_.pluck(userTokens, "token"));
+  closeConnectionsForTokens(_.compact(_.pluck(userTokens, "hashedToken")));
 };
 
 // Like _.difference, but uses EJSON.equals to compute which values to return.

packages/accounts-base/accounts_tests.js

@@ -189,10 +189,10 @@ Tinytest.addAsync('accounts - expire numeric token', function (test, onComplete)
   Meteor.users.update(result.id, {
     $set: {
       "services.resume.loginTokens": [{
-        token: Random.id(),
+        hashedToken: Random.id(),
         when: date
       }, {
-        token: Random.id(),
+        hashedToken: Random.id(),
         when: +date
       }]
     }
@@ -210,6 +210,78 @@ Tinytest.addAsync('accounts - expire numeric token', function (test, onComplete)
 });
 
 
+// Login tokens used to be stored unhashed in the database.  We want
+// to make sure users can still login after upgrading.
+
+var userWithUnhashedLoginToken = function () {
+  var result = Accounts.insertUserDoc({}, {username: Random.id()});
+
+  // Construct an old-style unhashed login token.
+  var stampedToken = Accounts._generateStampedLoginToken();
+
+  Meteor.users.update(
+    result.id,
+    {$push: {'services.resume.loginTokens': stampedToken}}
+  );
+
+  // Return result, as if from Accounts.insertUserDoc.
+  result.token = stampedToken.token;
+  result.tokenExpires = Accounts._tokenExpiration(stampedToken.when);
+  return result;
+};
+
+Tinytest.addAsync('accounts - login token', function (test, onComplete) {
+  // Test that we can login when the database contains a leftover
+  // old style unhashed login token.
+  var user1 = userWithUnhashedLoginToken();
+  var connection = DDP.connect(Meteor.absoluteUrl());
+  connection.call('login', {resume: user1.token});
+  connection.disconnect();
+
+  // Steal the unhashed token from the database and use it to login.
+  // This is a sanity check so that when we *can't* login with a
+  // stolen *hashed* token, we know it's not a problem with the test.
+  var user2 = userWithUnhashedLoginToken();
+  var stolenToken = Meteor.users.findOne(user2.id).services.resume.loginTokens[0].token;
+  test.isTrue(stolenToken);
+  connection = DDP.connect(Meteor.absoluteUrl());
+  connection.call('login', {resume: stolenToken});
+  connection.disconnect();
+
+  // Now do the same thing, this time with a stolen hashed token.
+  var user3 = Accounts.insertUserDoc({generateLoginToken: true}, {username: Random.id()});
+  stolenToken = Meteor.users.findOne(user3.id).services.resume.loginTokens[0].hashedToken;
+  test.isTrue(stolenToken);
+  connection = DDP.connect(Meteor.absoluteUrl());
+  // evil plan foiled
+  test.throws(
+    function () {
+      connection.call('login', {resume: stolenToken});
+    },
+    /You\'ve been logged out by the server/
+  );
+  connection.disconnect();
+
+  // Old style unhashed tokens are replaced by hashed tokens when
+  // encountered.  This means that after someone logins once, the
+  // old unhashed token is no longer available to be stolen.
+  var user4 = userWithUnhashedLoginToken();
+  connection = DDP.connect(Meteor.absoluteUrl());
+  connection.call('login', {resume: user4.token});
+  connection.disconnect();
+
+  // The token is no longer available to be stolen.
+  stolenToken = Meteor.users.findOne(user4.id).services.resume.loginTokens[0].token;
+  test.isFalse(stolenToken);
+
+  // After the upgrade, the client can still login with their login token.
+  connection = DDP.connect(Meteor.absoluteUrl());
+  connection.call('login', {resume: user4.token});
+  connection.disconnect();
+
+  onComplete();
+});
+
 Tinytest.addAsync(
   'accounts - connection data cleaned up',
   function (test, onComplete) {

packages/accounts-base/localstorage_token.js

@@ -115,10 +115,14 @@ var pollStoredLoginToken = function() {
 
   // != instead of !== just to make sure undefined and null are treated the same
   if (lastLoginTokenWhenPolled != currentLoginToken) {
-    if (currentLoginToken)
-      Meteor.loginWithToken(currentLoginToken); // XXX should we pass a callback here?
-    else
+    if (currentLoginToken) {
+      Meteor.loginWithToken(currentLoginToken, function (err) {
+        if (err)
+          makeClientLoggedOut();
+      });
+    } else {
       Meteor.logout();
+    }
   }
   lastLoginTokenWhenPolled = currentLoginToken;
 };

packages/accounts-password/password_server.js

@@ -95,7 +95,7 @@ Accounts.registerLoginHandler(function (options) {
     throw new Meteor.Error(403, "User not found");
   var stampedLoginToken = Accounts._generateStampedLoginToken();
   Meteor.users.update(
-    userId, {$push: {'services.resume.loginTokens': stampedLoginToken}});
+    userId, {$push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedLoginToken)}});
 
   return {
     token: stampedLoginToken.token,
@@ -139,7 +139,7 @@ Accounts.registerLoginHandler(function (options) {
 
   var stampedLoginToken = Accounts._generateStampedLoginToken();
   Meteor.users.update(
-    user._id, {$push: {'services.resume.loginTokens': stampedLoginToken}});
+    user._id, {$push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedLoginToken)}});
 
   return {
     token: stampedLoginToken.token,
@@ -314,13 +314,14 @@ Meteor.methods({resetPassword: function (token, newVerifier) {
     throw new Meteor.Error(403, "Token has invalid email address");
 
   var stampedLoginToken = Accounts._generateStampedLoginToken();
+  var newHashedToken = Accounts._hashStampedToken(stampedLoginToken);
 
   // NOTE: We're about to invalidate tokens on the user, who we might be
   // logged in as. Make sure to avoid logging ourselves out if this
   // happens. But also make sure not to leave the connection in a state
   // of having a bad token set if things fail.
   var oldToken = Accounts._getLoginToken(this.connection.id);
-  Accounts._setLoginToken(this.connection.id, null);
+  Accounts._setLoginToken(user._id, this.connection, null);
 
   try {
     // Update the user record by:
@@ -331,17 +332,17 @@ Meteor.methods({resetPassword: function (token, newVerifier) {
     // - Verifying their email, since they got the password reset via email.
     Meteor.users.update({_id: user._id, 'emails.address': email}, {
       $set: {'services.password.srp': newVerifier,
-             'services.resume.loginTokens': [stampedLoginToken],
+             'services.resume.loginTokens': [newHashedToken],
              'emails.$.verified': true},
       $unset: {'services.password.reset': 1}
     });
   } catch (err) {
     // update failed somehow. reset to old token.
-    Accounts._setLoginToken(this.connection.id, oldToken);
+    Accounts._setLoginToken(user._id, this.connection, oldToken);
     throw err;
   }
 
-  Accounts._setLoginToken(this.connection.id, stampedLoginToken.token);
+  Accounts._setLoginToken(user._id, this.connection, newHashedToken.hashedToken);
   this.setUserId(user._id);
 
   return {
@@ -421,6 +422,7 @@ Meteor.methods({verifyEmail: function (token) {
 
   // Log the user in with a new login token.
   var stampedLoginToken = Accounts._generateStampedLoginToken();
+  var hashedToken = Accounts._hashStampedToken(stampedLoginToken);
 
   // By including the address in the query, we can use 'emails.$' in the
   // modifier to get a reference to the specific object in the emails
@@ -432,10 +434,10 @@ Meteor.methods({verifyEmail: function (token) {
      'emails.address': tokenRecord.address},
     {$set: {'emails.$.verified': true},
      $pull: {'services.email.verificationTokens': {token: token}},
-     $push: {'services.resume.loginTokens': stampedLoginToken}});
+     $push: {'services.resume.loginTokens': hashedToken}});
 
   this.setUserId(user._id);
-  Accounts._setLoginToken(this.connection.id, stampedLoginToken.token);
+  Accounts._setLoginToken(user._id, this.connection, hashedToken.hashedToken);
   return {
     token: stampedLoginToken.token,
     tokenExpires: Accounts._tokenExpiration(stampedLoginToken.when),
@@ -514,7 +516,11 @@ Meteor.methods({createUser: function (options) {
 
   // client gets logged in as the new user afterwards.
   this.setUserId(result.id);
-  Accounts._setLoginToken(this.connection.id, result.token);
+  Accounts._setLoginToken(
+    result.id,
+    this.connection,
+    Accounts._hashLoginToken(result.token)
+  );
   return result;
 }});
 

packages/accounts-password/password_tests.js

@@ -1,5 +1,13 @@
 Accounts._noConnectionCloseDelayForTest = true;
 
+if (Meteor.isServer) {
+  Meteor.methods({
+    getUserId: function () {
+      return this.userId;
+    }
+  });
+}
+
 if (Meteor.isClient) (function () {
 
   // XXX note, only one test can do login/logout things at once! for
@@ -344,7 +352,23 @@ if (Meteor.isClient) (function () {
         loggedInAs(this.username, test, expect));
     },
 
-    function(test, expect) {
+    function (test, expect) {
+      // we can't login with an invalid token
+      var expectLoginError = expect(function (err) {
+        test.isTrue(err);
+      });
+      Meteor.loginWithToken('invalid', expectLoginError);
+    },
+
+    function (test, expect) {
+      // we can login with a valid token
+      var expectLoginOK = expect(function (err) {
+        test.isFalse(err);
+      });
+      Meteor.loginWithToken(Accounts._storedLoginToken(), expectLoginOK);
+    },
+
+    function (test, expect) {
       // test logging out invalidates our token
       var expectLoginError = expect(function (err) {
         test.isTrue(err);
@@ -356,7 +380,7 @@ if (Meteor.isClient) (function () {
       });
     },
 
-    function(test, expect) {
+    function (test, expect) {
       var self = this;
       // Test that login tokens get expired. We should get logged out when a
       // token expires, and not be able to log in again with the same token.
@@ -388,10 +412,10 @@ if (Meteor.isClient) (function () {
       var self = this;
 
       // copied from livedata/client_convenience.js
-      var ddpUrl = '/';
+      self.ddpUrl = '/';
       if (typeof __meteor_runtime_config__ !== "undefined") {
         if (__meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL)
-          ddpUrl = __meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL;
+          self.ddpUrl = __meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL;
       }
       // XXX can we get the url from the existing connection somehow
       // instead?
@@ -400,7 +424,7 @@ if (Meteor.isClient) (function () {
       // connection while leaving Accounts.connection logged in.
       var token;
       var userId;
-      self.secondConn = DDP.connect(ddpUrl);
+      self.secondConn = DDP.connect(self.ddpUrl);
 
       var expectLoginError = expect(function (err) {
         test.isTrue(err);
@@ -461,6 +485,41 @@ if (Meteor.isClient) (function () {
       );
     },
     logoutStep,
+    function (test, expect) {
+      var self = this;
+      // Test that, when we call logoutOtherClients, if the server disconnects
+      // us before the logoutOtherClients callback runs, then we still end up
+      // logged in.
+      var expectServerLoggedIn = expect(function (err, result) {
+        test.isFalse(err);
+        test.isTrue(Meteor.userId());
+        test.equal(result, Meteor.userId());
+      });
+
+      Meteor.loginWithPassword(
+        self.username,
+        self.password,
+        expect(function (err) {
+          test.isFalse(err);
+          test.isTrue(Meteor.userId());
+
+          // The test is only useful if things interleave in the following order:
+          // - logoutOtherClients runs on the server
+          // - onReconnect fires and sends a login method with the old token,
+          //   which results in an error
+          // - logoutOtherClients callback runs and stores the new token and
+          //   logs in with it
+          // In practice they seem to interleave this way, but I'm not sure how
+          // to make sure that they do.
+
+          Meteor.logoutOtherClients(function (err) {
+            test.isFalse(err);
+            Meteor.call("getUserId", expectServerLoggedIn);
+          });
+        })
+      );
+    },
+    logoutStep,
     function (test, expect) {
       var self = this;
       // Test that deleting a user logs out that user's connections.
@@ -568,7 +627,6 @@ if (Meteor.isServer) (function () {
           test.isTrue(result);
           var token = Accounts._getAccountData(serverConn.id, 'loginToken');
           test.isTrue(token);
-          test.equal(result.token, token);
           test.isTrue(_.contains(
             Accounts._getTokenConnections(token), serverConn.id));
           clientConn.disconnect();

packages/appcache/appcache-client.js

@@ -58,11 +58,10 @@ window.applicationCache.addEventListener('noupdate', cacheIsNowUpToDate, false);
 window.applicationCache.addEventListener('obsolete', (function() {
   if (reloadRetry) {
     cacheIsNowUpToDate();
-  }
-  else {
+  } else {
     appcacheUpdated = true;
     Reload._reload();
   }
 }), false);
 
-} // if window.applicationCache
+} // if window.applicationCache

packages/appcache/appcache-server.js

@@ -44,8 +44,7 @@ Meteor.AppCache = {
         _.each(value, function (urlPrefix) {
           RoutePolicy.declare(urlPrefix, 'static-online');
         });
-      }
-      else {
+      } else {
         throw new Error('Invalid AppCache config option: ' + option);
       }
     });

packages/application-configuration/config.js

@@ -149,15 +149,50 @@ AppConfig.configurePackage = function (packageName, configure) {
   };
 };
 
-AppConfig.configureService = function (serviceName, configure) {
+AppConfig.configureService = function (serviceName, version, configure) {
+
+  // Collect all the endpoints for this service, from both old- and new-format
+  // documents, and call the `configure` callback with all the service endpoints
+  // that we know about.
+  var callConfigure = function (doc) {
+    var serviceDocs = Services.find({
+      name: serviceName,
+      version: version
+    });
+    var endpoints = [];
+    serviceDocs.forEach(function (serviceDoc) {
+      if (serviceDoc.providers) {
+        _.each(serviceDoc.providers, function (endpoint, app) {
+          endpoints.push(endpoint);
+        });
+      } else {
+        endpoints.push(serviceDoc.endpoint);
+      }
+    });
+    configure(endpoints);
+  };
+
   if (ultra) {
     // there's a Meteor.startup() that produces the various collections, make
     // sure it runs first before we continue.
     collectionFuture.wait();
-    ultra.subscribe('servicesByName', serviceName);
-    return Services.find({name: serviceName}).observe({
-      added: configure,
-      changed: configure
+    // First try to subscribe to the new format service registrations; if that
+    // sub doesn't exist, then ultraworld hasn't updated to the new format yet,
+    // so try the old format `servicesByName` sub instead.
+    ultra.subscribe('services', serviceName, version, {
+      onError: function (err) {
+        if (err.error === 404) {
+          ultra.subscribe('servicesByName', serviceName);
+        }
+      }
+    });
+    return Services.find({
+      name: serviceName,
+      version: version
+    }).observe({
+      added: callConfigure,
+      changed: callConfigure,
+      removed: callConfigure
     });
   }
 

packages/autoupdate/autoupdate_client.js

@@ -44,9 +44,7 @@ Autoupdate.newClientAvailable = function () {
 
 
 
-// XXX Livedata exporting this via DDP is a hack. See
-// packages/livedata/livedata_common.js
-var retry = new DDP._Retry({
+var retry = new Retry({
   // Unlike the stream reconnect use of Retry, which we want to be instant
   // in normal operation, this is a wacky failure. We don't want to retry
   // right away, we can start slowly.

packages/autoupdate/package.js

@@ -5,7 +5,7 @@ Package.describe({
 
 Package.on_use(function (api) {
   api.use('webapp', 'server');
-  api.use('deps', 'client');
+  api.use(['deps', 'retry'], 'client');
   api.use(['livedata', 'mongo-livedata'], ['client', 'server']);
   api.use('deps', 'client');
   api.use('reload', 'client', {weak: true});

packages/ctl-helper/ctl-helper.js

@@ -45,6 +45,7 @@ _.extend(Ctl, {
       Ctl.findGalaxy(), 'getAppConfiguration', [Ctl.myAppName()]);
     if (typeof admin == 'undefined')
       admin = appConfig.admin;
+    admin = !!admin;
 
     var jobId = null;
     var rootUrl = Ctl.rootUrl;
@@ -65,13 +66,15 @@ _.extend(Ctl, {
     });
 
     // XXX args? env?
+    var env = {
+      ROOT_URL: rootUrl,
+      METEOR_SETTINGS: appConfig.settings || appConfig.METEOR_SETTINGS
+    };
+    if (admin)
+      env.ADMIN_APP = 'true';
     jobId = Ctl.prettyCall(Ctl.findGalaxy(), 'run', [Ctl.myAppName(), program, {
       exitPolicy: 'restart',
-      env: {
-        ROOT_URL: rootUrl,
-        METEOR_SETTINGS: appConfig.settings || appConfig.METEOR_SETTINGS,
-        ADMIN_APP: admin
-      },
+      env: env,
       ports: {
         "main": {
           bindEnv: "PORT",
@@ -139,27 +142,30 @@ _.extend(Ctl, {
   updateProxyActiveTags: function (tags) {
     var proxy;
     var proxyTagSwitchFuture = new Future;
-    AppConfig.configureService('proxy', function (proxyService) {
-      try {
-        proxy = Follower.connect(proxyService.providers.proxy, {
-        group: "proxy"
-        });
-        proxy.call('updateTags', Ctl.myAppName(), tags);
-        proxy.disconnect();
-        if (!proxyTagSwitchFuture.isResolved())
-          proxyTagSwitchFuture['return']();
-      } catch (e) {
-        if (!proxyTagSwitchFuture.isResolved())
-          proxyTagSwitchFuture['throw'](e);
+    AppConfig.configureService('proxy', 'pre0', function (proxyService) {
+      if (proxyService && ! _.isEmpty(proxyService)) {
+        try {
+          proxy = Follower.connect(proxyService, {
+            group: "proxy"
+          });
+          proxy.call('updateTags', Ctl.myAppName(), tags);
+          proxy.disconnect();
+          if (!proxyTagSwitchFuture.isResolved())
+            proxyTagSwitchFuture['return']();
+        } catch (e) {
+          if (!proxyTagSwitchFuture.isResolved())
+            proxyTagSwitchFuture['throw'](e);
+        }
       }
     });
 
     var proxyTimeout = Meteor.setTimeout(function () {
       if (!proxyTagSwitchFuture.isResolved())
         proxyTagSwitchFuture['throw'](
-          new Error("timed out looking for a proxy " +
-                    "or trying to change tags on it " +
-                    proxy.status().status));
+          new Error("Timed out looking for a proxy " +
+                    "or trying to change tags on it. Status: " +
+                    (proxy ? proxy.status().status : "no connection"))
+        );
     }, 10*1000);
     proxyTagSwitchFuture.wait();
     Meteor.clearTimeout(proxyTimeout);

packages/deps/deps.js

@@ -98,7 +98,9 @@ _.extend(Deps.Computation.prototype, {
 
     var g = function () {
       Deps.nonreactive(function () {
-        f(self);
+        return Meteor._noYieldsAllowed(function () {
+          f(self);
+        });
       });
     };
 
@@ -284,7 +286,9 @@ _.extend(Deps, {
       throw new Error('Deps.autorun requires a function argument');
 
     constructingComputation = true;
-    var c = new Deps.Computation(f, Deps.currentComputation);
+    var c = new Deps.Computation(function (c) {
+      Meteor._noYieldsAllowed(_.bind(f, this, c));
+    }, Deps.currentComputation);
 
     if (Deps.active)
       Deps.onInvalidate(function () {

packages/facebook/facebook_client.js

@@ -34,6 +34,6 @@ Facebook.requestCredential = function (options, credentialRequestCompleteCallbac
 
   Oauth.showPopup(
     loginUrl,
-    _.bind(credentialRequestCompleteCallack, null, credentialToken)
+    _.bind(credentialRequestCompleteCallback, null, credentialToken)
   );
 };

packages/follower-livedata/follower.js

@@ -104,7 +104,7 @@ Follower = {
           url: url
         }]);
       } else {
-        conn = DDP.connect(url);
+        conn = DDP.connect(url, options);
         prevReconnect = conn.reconnect;
         prevDisconnect = conn.disconnect;
         prevApply = conn.apply;

packages/livedata/.npm/package/npm-shrinkwrap.json

@@ -10,7 +10,7 @@
           "version": "0.7.0",
           "dependencies": {
             "websocket-driver": {
-              "version": "0.3.1"
+              "version": "0.3.2"
             }
           }
         }

packages/livedata/livedata_common.js

@@ -115,8 +115,3 @@ stringifyDDP = function (msg) {
 // state in the DDP session. Meteor.setTimeout and friends clear
 // it. We can probably find a better way to factor this.
 DDP._CurrentInvocation = new Meteor.EnvironmentVariable;
-
-
-// This is private and a hack. It is used by autoupdate_client. We
-// should refactor. Maybe a separate 'exponential-backoff' package?
-DDP._Retry = Retry;

packages/livedata/livedata_connection.js

@@ -545,6 +545,7 @@ _.extend(Connection.prototype, {
     var self = this;
     var f = new Future();
     var ready = false;
+    var handle;
     args = args || [];
     args.push({
       onReady: function () {
@@ -559,8 +560,9 @@ _.extend(Connection.prototype, {
       }
     });
 
-    self.subscribe.apply(self, [name].concat(args));
+    handle = self.subscribe.apply(self, [name].concat(args));
     f.wait();
+    return handle;
   },
 
   methods: function (methods) {

packages/livedata/livedata_server_tests.js

@@ -24,7 +24,6 @@ Tinytest.addAsync(
   }
 );
 
-
 Tinytest.addAsync(
   "livedata server - connectionHandle.close()",
   function (test, onComplete) {

packages/livedata/package.js

@@ -6,7 +6,8 @@ Package.describe({
 Npm.depends({sockjs: "0.3.8", websocket: "1.0.8"});
 
 Package.on_use(function (api) {
-  api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps', 'logging'],
+  api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps',
+           'logging', 'retry'],
           ['client', 'server']);
 
   // It is OK to use this package on a server architecture without making a
@@ -33,7 +34,6 @@ Package.on_use(function (api) {
   // Transport
   api.use('reload', 'client', {weak: true});
   api.add_files('common.js');
-  api.add_files('retry.js', ['client', 'server']);
   api.add_files(['sockjs-0.3.4.js', 'stream_client_sockjs.js'], 'client');
   api.add_files('stream_client_nodejs.js', 'server');
   api.add_files('stream_client_common.js', ['client', 'server']);

packages/liverange/liverange.js

@@ -331,8 +331,7 @@ LiveRange.prototype.visit = function(visitRange, visitNode) {
           recurse(rangeStart, rangeEnd, startIndex+1);
         visitRange(false, range);
         n = rangeEnd;
-      }
-      else {
+      } else {
         // bare node
         if (visitNode(true, n) !== false && n.firstChild)
           recurse(n.firstChild, n.lastChild);

packages/localstorage/localstorage.js

@@ -1,20 +1,37 @@
-// This is not an ideal name, but we can change it later.
+// Meteor._localStorage is not an ideal name, but we can change it later.
 
 if (window.localStorage) {
-  Meteor._localStorage = {
-    getItem: function (key) {
-      return window.localStorage.getItem(key);
-    },
-    setItem: function (key, value) {
-      window.localStorage.setItem(key, value);
-    },
-    removeItem: function (key) {
-      window.localStorage.removeItem(key);
-    }
-  };
+  // Let's test to make sure that localStorage actually works. For example, in
+  // Safari with private browsing on, window.localStorage exists but actually
+  // trying to use it throws.
+
+  var key = '_localstorage_test_' + Random.id();
+  var retrieved;
+  try {
+    window.localStorage.setItem(key, key);
+    retrieved = window.localStorage.getItem(key);
+    window.localStorage.removeItem(key);
+  } catch (e) {
+    // ... ignore
+  }
+  if (key === retrieved) {
+    Meteor._localStorage = {
+      getItem: function (key) {
+        return window.localStorage.getItem(key);
+      },
+      setItem: function (key, value) {
+        window.localStorage.setItem(key, value);
+      },
+      removeItem: function (key) {
+        window.localStorage.removeItem(key);
+      }
+    };
+  }
 }
+
 // XXX eliminate dependency on jQuery, detect browsers ourselves
-else if ($.browser.msie) { // If we are on IE, which support userData
+// Else, if we are on IE, which support userData
+if (!Meteor._localStorage && $.browser.msie) {
   var userdata = document.createElement('span'); // could be anything
   userdata.style.behavior = 'url("#default#userData")';
   userdata.id = 'localstorage-helper';
@@ -40,7 +57,9 @@ else if ($.browser.msie) { // If we are on IE, which support userData
       return userdata.getAttribute(key);
     }
   };
-} else {
+}
+
+if (!Meteor._localStorage) {
   Meteor._debug(
     "You are running a browser with no localStorage or userData "
       + "support. Logging in from one tab will not cause another "

packages/localstorage/package.js

@@ -5,6 +5,7 @@ Package.describe({
 
 Package.on_use(function (api) {
   api.use('jquery', 'client'); // XXX only used for browser detection. remove.
+  api.use('random', 'client');
 
   api.add_files('localstorage.js', 'client');
 });

packages/meteor/node-issue-6506-workaround.js

@@ -1,13 +1,16 @@
 // Temporary workaround for https://github.com/joyent/node/issues/6506
-// Our fix involves replicating a bunch of files in order to 
-//
-if (process.version !== 'v0.10.22' && process.version !== 'v0.10.23') {
+// Our fix involves replicating a bunch of functions in order to change
+// a single line.
+
+var PATCH_VERSIONS = ['v0.10.22', 'v0.10.23', 'v0.10.24'];
+
+if (!_.contains(PATCH_VERSIONS, process.version)) {
   if (!process.env.DISABLE_WEBSOCKETS) {
     console.error("This version of Meteor contains a patch for a bug in Node v0.10.");
-    console.error("The patch is against only versions 0.10.22 and 0.10.23.");
+    console.error("The patch is against only versions 0.10.22 through 0.10.24.");
     console.error("You are using version " + process.version + " instead, so we cannot apply the patch.");
     console.error("To mitigate the most common effect of the bug, websockets will be disabled.");
-    console.error("To enable websockets, use Node v0.10.22 or .23, or upgrade to a later version of Meteor (if available).");
+    console.error("To enable websockets, use Node v0.10.22 through v0.10.24, or upgrade to a later version of Meteor (if available).");
     process.env.DISABLE_WEBSOCKETS = 't';
   }
 } else {

packages/meteor/server_environment.js

@@ -4,21 +4,34 @@ Meteor = {
 };
 
 Meteor.settings = {};
-if (process.env.METEOR_SETTINGS) {
+
+if (process.env.APP_CONFIG) {
+  // put settings from the app configuration in the settings.  Don't depend on
+  // the Galaxy package for now, to avoid silly loops.
+ try {
+   var appConfig = JSON.parse(process.env.APP_CONFIG);
+   if (!appConfig.settings) {
+     Meteor.settings = {};
+   } else if (typeof appConfig.settings === "string") {
+     Meteor.settings = JSON.parse(appConfig.settings);
+   } else {
+     // Old versions of Galaxy may store settings in MongoDB as objects. Newer
+     // versions store it as strings (so that we aren't restricted to
+     // MongoDB-compatible objects). This line makes it work on older Galaxies.
+     // XXX delete this eventually
+     Meteor.settings = appConfig.settings;
+   }
+  } catch (e) {
+    throw new Error("Settings from app config are not valid JSON");
+  }
+} else if (process.env.METEOR_SETTINGS) {
   try {
     Meteor.settings = JSON.parse(process.env.METEOR_SETTINGS);
   } catch (e) {
     throw new Error("Settings are not valid JSON");
   }
-} else if ( process.env.APP_CONFIG) {
-  // put settings from the app configuration in the settings.  Don't depend on
-  // the Galaxy package for now, to avoid silly loops.
- try {
-    Meteor.settings = JSON.parse(process.env.APP_CONFIG).settings || {};
-  } catch (e) {
-    throw new Error("Settings are not valid JSON");
-  }
 }
+
 // Push a subset of settings to the client.
 if (Meteor.settings && Meteor.settings.public &&
     typeof __meteor_runtime_config__ === "object") {

packages/meteor/url_common.js

@@ -20,7 +20,7 @@ Meteor.absoluteUrl = function (path, options) {
   if (path)
     url += path;
 
-  // turn http to http if secure option is set, and we're not talking
+  // turn http to https if secure option is set, and we're not talking
   // to localhost.
   if (options.secure &&
       /^http:/.test(url) && // url starts with 'http:'

packages/meteorid/meteorid_client.js

@@ -26,6 +26,9 @@ MeteorId.requestCredential = function (credentialRequestCompleteCallback) {
   Oauth.showPopup(
     loginUrl,
     _.bind(credentialRequestCompleteCallback, null, credentialToken),
-    { height: 406 }
+    {
+      width: 430,
+      height: 406
+    }
   );
 };

packages/meteorid/meteorid_common.js

@@ -1,2 +1,2 @@
 // XXX fill me in!
-METEORID_URL = "";
+METEORID_URL = "http://10.0.2.2:3000";

packages/meteorid/meteorid_configure.html

@@ -3,7 +3,7 @@
     First, you'll need to get a MeteorId Client ID.
     Set Authorized Redirect URIs to:
     <span class="url">
-      {{siteUrl}}_oauth/meteor?close
+      {{siteUrl}}_oauth/meteorId?close
     </span>
   </p>
 </template>

packages/minimongo/NOTES

@@ -5,10 +5,22 @@ In update, $pull can't take a selector like {$gt: 3} (but it can take
 can be used, but selectors that are intended to match non-document
 values won't work.)
 
-In update, we don't support '$' to indicate the matched array object
-as in {$set: {'a.$.x': 12}}.
+Sort does not correctly implement the Mongo behavior where only array entries
+that match the query's selector are used as sort keys. Specifically, when
+sorting a document with sort specifier {a:1}, the document {a: [5, 10]} will
+usually have sort key 5 (the minimum is used for ascending sorts and the maximum
+for descending sorts). But if the query's selector is {a: {$gt: 7}}, then Mongo
+will actually use 10 as the sort key because 5 does not match.
 
-Sort does not support subkeys. You can sort on 'a', but not 'a.b'.
+Unsupported selectors:
+ - $elemMatch inside $all
+ - geoqueries other than $near ($nearSphere, $geoIntersects, $geoWithin)
+
+In MongoDB, a query with $near that matches a document which has multiple points
+matching the key (eg, `c.find({$near: 'a'})` with the document
+`{a: [[1,1], [2, 2]]}` can return the document multiple times (perhaps not
+even consecutively). In minimongo, queries never return a document multiple
+times.
 
 ## ON TYPES ##
 
@@ -29,9 +41,6 @@ integer type, and we don't support the integer type yet.)
 
 ## API ##
 
-find() doesn't support retrieving a subset of fields. It always
-returns the whole doc.
-
 find() doesn't support the min and max parameters.
 
 findAndModify isn't supported.
@@ -43,9 +52,6 @@ update() should have a clear stance on atomicity (both in terms of
 multiple ops on a single document, and on multi-document update mode.)
 It just hasn't been looked at/thought about yet.
 
-upsert combined with $-operators might work, but hasn't actually been
-looked at or tested.
-
 In general, the API needs tests, especially update. (On the other
 hand, the underlying selector and mutator code is quite well tested.)
 
@@ -53,13 +59,9 @@ hand, the underlying selector and mutator code is quite well tested.)
 
 We ignore the 'x' and 's' flags on regular expressions.
 
-We don't do as much type checking as we could, especially in
-selectors. If pass in something that's weirdly formed, you'll probably
-just get a random exception or error.
-
 "Natural order" isn't very well defined.
 
 We don't support capped collections.
 
-No performance optimization has been done. In particular, there are no
+Little performance optimization has been done. In particular, there are no
 indexes.

packages/minimongo/helpers.js

@@ -0,0 +1,39 @@
+// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as
+// arrays.
+// XXX maybe this should be EJSON.isArray
+isArray = function (x) {
+  return _.isArray(x) && !EJSON.isBinary(x);
+};
+
+// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about
+// RegExp
+// XXX note that _type(undefined) === 3!!!!
+isPlainObject = function (x) {
+  return x && LocalCollection._f._type(x) === 3;
+};
+
+isIndexable = function (x) {
+  return isArray(x) || isPlainObject(x);
+};
+
+isOperatorObject = function (valueSelector) {
+  if (!isPlainObject(valueSelector))
+    return false;
+
+  var theseAreOperators = undefined;
+  _.each(valueSelector, function (value, selKey) {
+    var thisIsOperator = selKey.substr(0, 1) === '$';
+    if (theseAreOperators === undefined) {
+      theseAreOperators = thisIsOperator;
+    } else if (theseAreOperators !== thisIsOperator) {
+      throw new Error("Inconsistent operator: " + valueSelector);
+    }
+  });
+  return !!theseAreOperators;  // {} has no operators
+};
+
+
+// string can be converted to integer
+isNumericKey = function (s) {
+  return /^[0-9]+$/.test(s);
+};

packages/minimongo/minimongo.js

@@ -20,7 +20,7 @@ LocalCollection = function (name) {
   //  results: array (ordered) or object (unordered) of current results
   //  results_snapshot: snapshot of results. null if not paused.
   //  cursor: Cursor object for the query.
-  //  selector_f, sort_f, (callbacks): functions
+  //  selector, sorter, (callbacks): functions
   this.queries = {};
 
   // null if not saving originals; a map from id to original document value if
@@ -31,6 +31,8 @@ LocalCollection = function (name) {
   this.paused = false;
 };
 
+Minimongo = {};
+
 // Object exported only for unit testing.
 // Use it to export private functions to test in Tinytest.
 MinimongoTest = {};
@@ -89,18 +91,13 @@ LocalCollection.Cursor = function (collection, selector, options) {
   if (LocalCollection._selectorIsId(selector)) {
     // stash for fast path
     self.selector_id = LocalCollection._idStringify(selector);
-    self.selector_f = LocalCollection._compileSelector(selector, self);
-    self.sort_f = undefined;
+    self.matcher = new Minimongo.Matcher(selector, self);
+    self.sorter = undefined;
   } else {
-    // MongoDB throws different errors on different branching operators
-    // containing $near
-    if (isGeoQuerySpecial(selector))
-      throw new Error("$near can't be inside $or/$and/$nor/$not");
-
     self.selector_id = undefined;
-    self.selector_f = LocalCollection._compileSelector(selector, self);
-    self.sort_f = (isGeoQuery(selector) || options.sort) ?
-      LocalCollection._compileSort(options.sort || [], self) : null;
+    self.matcher = new Minimongo.Matcher(selector, self);
+    self.sorter = (self.matcher.hasGeoQuery() || options.sort) ?
+      new Sorter(options.sort || []) : null;
   }
   self.skip = options.skip;
   self.limit = options.limit;
@@ -273,10 +270,14 @@ _.extend(LocalCollection.Cursor.prototype, {
     if (!options._allow_unordered && !ordered && (self.skip || self.limit))
       throw new Error("must use ordered observe with skip or limit");
 
-    // XXX merge this object w/ "this" Cursor.  they're the same.
+    if (self.fields && (self.fields._id === 0 || self.fields._id === false))
+      throw Error("You may not observe a cursor with {fields: {_id: 0}}");
+
     var query = {
-      selector_f: self.selector_f, // not fast pathed
-      sort_f: ordered && self.sort_f,
+      matcher: self.matcher, // not fast pathed
+      sorter: ordered && self.sorter,
+      distances: (
+        self.matcher.hasGeoQuery() && ordered && new LocalCollection._IdMap),
       results_snapshot: null,
       ordered: ordered,
       cursor: self,
@@ -292,7 +293,7 @@ _.extend(LocalCollection.Cursor.prototype, {
       qid = self.collection.next_qid++;
       self.collection.queries[qid] = query;
     }
-    query.results = self._getRawObjects(ordered);
+    query.results = self._getRawObjects(ordered, query.distances);
     if (self.collection.paused)
       query.results_snapshot = (ordered ? [] : {});
 
@@ -371,13 +372,21 @@ _.extend(LocalCollection.Cursor.prototype, {
 
 // Returns a collection of matching objects, but doesn't deep copy them.
 //
-// If ordered is set, returns a sorted array, respecting sort_f, skip, and limit
-// properties of the query.  if sort_f is falsey, no sort -- you get the natural
+// If ordered is set, returns a sorted array, respecting sorter, skip, and limit
+// properties of the query.  if sorter is falsey, no sort -- you get the natural
 // order.
 //
-// If ordered is not set, returns an object mapping from ID to doc (sort_f, skip
+// If ordered is not set, returns an object mapping from ID to doc (sorter, skip
 // and limit should not be set).
-LocalCollection.Cursor.prototype._getRawObjects = function (ordered) {
+//
+// If ordered is set and this cursor is a $near geoquery, then this function
+// will use an _IdMap to track each distance from the $near argument point in
+// order to use it as a sort key. If an _IdMap is passed in the 'distances'
+// argument, this function will clear it and use it for this purpose (otherwise
+// it will just create its own _IdMap). The observeChanges implementation uses
+// this to remember the distances after this function returns.
+LocalCollection.Cursor.prototype._getRawObjects = function (ordered,
+                                                            distances) {
   var self = this;
 
   var results = ordered ? [] : {};
@@ -401,16 +410,32 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) {
   }
 
   // slow path for arbitrary selector, sort, skip, limit
-  for (var id in self.collection.docs) {
-    var doc = self.collection.docs[id];
-    if (self.selector_f(doc)) {
-      if (ordered)
+
+  // in the observeChanges case, distances is actually part of the "query" (ie,
+  // live results set) object.  in other cases, distances is only used inside
+  // this function.
+  if (self.matcher.hasGeoQuery() && ordered) {
+    if (distances)
+      distances.clear();
+    else
+      distances = new LocalCollection._IdMap();
+  }
+
+  for (var idStringified in self.collection.docs) {
+    var doc = self.collection.docs[idStringified];
+    var id = LocalCollection._idParse(idStringified);  // XXX use more idmaps
+    var matchResult = self.matcher.documentMatches(doc);
+    if (matchResult.result) {
+      if (ordered) {
         results.push(doc);
-      else
-        results[id] = doc;
+        if (distances && matchResult.distance !== undefined)
+          distances.set(id, matchResult.distance);
+      } else {
+        results[idStringified] = doc;
+      }
     }
     // Fast path for limited unsorted queries.
-    if (self.limit && !self.skip && !self.sort_f &&
+    if (self.limit && !self.skip && !self.sorter &&
         results.length === self.limit)
       return results;
   }
@@ -418,8 +443,10 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) {
   if (!ordered)
     return results;
 
-  if (self.sort_f)
-    results.sort(self.sort_f);
+  if (self.sorter) {
+    var comparator = self.sorter.getComparator({distances: distances});
+    results.sort(comparator);
+  }
 
   var idx_start = self.skip || 0;
   var idx_end = self.limit ? (self.limit + idx_start) : results.length;
@@ -477,7 +504,10 @@ LocalCollection.prototype.insert = function (doc, callback) {
   // trigger live queries that match
   for (var qid in self.queries) {
     var query = self.queries[qid];
-    if (query.selector_f(doc)) {
+    var matchResult = query.matcher.documentMatches(doc);
+    if (matchResult.result) {
+      if (query.distances && matchResult.distance !== undefined)
+        query.distances.set(doc._id, matchResult.distance);
       if (query.cursor.skip || query.cursor.limit)
         queriesToRecompute.push(qid);
       else
@@ -505,23 +535,24 @@ LocalCollection.prototype.remove = function (selector, callback) {
   var remove = [];
 
   var queriesToRecompute = [];
-  var selector_f = LocalCollection._compileSelector(selector, self);
+  var matcher = new Minimongo.Matcher(selector, self);
 
   // Avoid O(n) for "remove a single doc by ID".
   var specificIds = LocalCollection._idsMatchedBySelector(selector);
   if (specificIds) {
     _.each(specificIds, function (id) {
       var strId = LocalCollection._idStringify(id);
-      // We still have to run selector_f, in case it's something like
+      // We still have to run matcher, in case it's something like
       //   {_id: "X", a: 42}
-      if (_.has(self.docs, strId) && selector_f(self.docs[strId]))
+      if (_.has(self.docs, strId)
+          && matcher.documentMatches(self.docs[strId]).result)
         remove.push(strId);
     });
   } else {
-    for (var id in self.docs) {
-      var doc = self.docs[id];
-      if (selector_f(doc)) {
-        remove.push(id);
+    for (var strId in self.docs) {
+      var doc = self.docs[strId];
+      if (matcher.documentMatches(doc).result) {
+        remove.push(strId);
       }
     }
   }
@@ -531,7 +562,7 @@ LocalCollection.prototype.remove = function (selector, callback) {
     var removeId = remove[i];
     var removeDoc = self.docs[removeId];
     _.each(self.queries, function (query, qid) {
-      if (query.selector_f(removeDoc)) {
+      if (query.matcher.documentMatches(removeDoc).result) {
         if (query.cursor.skip || query.cursor.limit)
           queriesToRecompute.push(qid);
         else
@@ -545,8 +576,10 @@ LocalCollection.prototype.remove = function (selector, callback) {
   // run live query callbacks _after_ we've removed the documents.
   _.each(queryRemove, function (remove) {
     var query = self.queries[remove.qid];
-    if (query)
+    if (query) {
+      query.distances && query.distances.remove(remove.doc._id);
       LocalCollection._removeFromResults(query, remove.doc);
+    }
   });
   _.each(queriesToRecompute, function (qid) {
     var query = self.queries[qid];
@@ -572,7 +605,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) {
   }
   if (!options) options = {};
 
-  var selector_f = LocalCollection._compileSelector(selector, self);
+  var matcher = new Minimongo.Matcher(selector, self);
 
   // Save the original results of any query that we might need to
   // _recomputeResults on, because _modifyAndNotify will mutate the objects in
@@ -590,10 +623,11 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) {
 
   for (var id in self.docs) {
     var doc = self.docs[id];
-    if (selector_f(doc)) {
+    var queryResult = matcher.documentMatches(doc);
+    if (queryResult.result) {
       // XXX Should we save the original even if mod ends up being a no-op?
       self._saveOriginal(id, doc);
-      self._modifyAndNotify(doc, mod, recomputeQids);
+      self._modifyAndNotify(doc, mod, recomputeQids, queryResult.arrayIndex);
       ++updateCount;
       if (!options.multi)
         break;
@@ -614,7 +648,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) {
   var insertedId;
   if (updateCount === 0 && options.upsert) {
     var newDoc = LocalCollection._removeDollarOperators(selector);
-    LocalCollection._modify(newDoc, mod, true);
+    LocalCollection._modify(newDoc, mod, {isInsert: true});
     if (! newDoc._id && options.insertedId)
       newDoc._id = options.insertedId;
     insertedId = self.insert(newDoc);
@@ -658,14 +692,14 @@ LocalCollection.prototype.upsert = function (selector, mod, options, callback) {
 };
 
 LocalCollection.prototype._modifyAndNotify = function (
-    doc, mod, recomputeQids) {
+    doc, mod, recomputeQids, arrayIndex) {
   var self = this;
 
   var matched_before = {};
   for (var qid in self.queries) {
     var query = self.queries[qid];
     if (query.ordered) {
-      matched_before[qid] = query.selector_f(doc);
+      matched_before[qid] = query.matcher.documentMatches(doc).result;
     } else {
       // Because we don't support skip or limit (yet) in unordered queries, we
       // can just do a direct lookup.
@@ -676,12 +710,15 @@ LocalCollection.prototype._modifyAndNotify = function (
 
   var old_doc = EJSON.clone(doc);
 
-  LocalCollection._modify(doc, mod);
+  LocalCollection._modify(doc, mod, {arrayIndex: arrayIndex});
 
   for (qid in self.queries) {
     query = self.queries[qid];
     var before = matched_before[qid];
-    var after = query.selector_f(doc);
+    var afterMatch = query.matcher.documentMatches(doc);
+    var after = afterMatch.result;
+    if (after && query.distances && afterMatch.distance !== undefined)
+      query.distances.set(doc._id, afterMatch.distance);
 
     if (query.cursor.skip || query.cursor.limit) {
       // We need to recompute any query where the doc may have been in the
@@ -713,12 +750,13 @@ LocalCollection._insertInResults = function (query, doc) {
   var fields = EJSON.clone(doc);
   delete fields._id;
   if (query.ordered) {
-    if (!query.sort_f) {
+    if (!query.sorter) {
       query.addedBefore(doc._id, fields, null);
       query.results.push(doc);
     } else {
       var i = LocalCollection._insertInSortedList(
-        query.sort_f, query.results, doc);
+        query.sorter.getComparator({distances: query.distances}),
+        query.results, doc);
       var next = query.results[i+1];
       if (next)
         next = next._id;
@@ -761,14 +799,15 @@ LocalCollection._updateInResults = function (query, doc, old_doc) {
 
   if (!_.isEmpty(changedFields))
     query.changed(doc._id, changedFields);
-  if (!query.sort_f)
+  if (!query.sorter)
     return;
 
   // just take it out and put it back in again, and see if the index
   // changes
   query.results.splice(orig_idx, 1);
   var new_idx = LocalCollection._insertInSortedList(
-    query.sort_f, query.results, doc);
+    query.sorter.getComparator({distances: query.distances}),
+    query.results, doc);
   if (orig_idx !== new_idx) {
     var next = query.results[new_idx+1];
     if (next)
@@ -790,7 +829,9 @@ LocalCollection._updateInResults = function (query, doc, old_doc) {
 LocalCollection._recomputeResults = function (query, oldResults) {
   if (!oldResults)
     oldResults = query.results;
-  query.results = query.cursor._getRawObjects(query.ordered);
+  if (query.distances)
+    query.distances.clear();
+  query.results = query.cursor._getRawObjects(query.ordered, query.distances);
 
   if (!query.paused) {
     LocalCollection._diffQueryChanges(
@@ -974,23 +1015,3 @@ LocalCollection._makeChangedFields = function (newDoc, oldDoc) {
   });
   return fields;
 };
-
-// Searches $near operator in the selector recursively
-// (including all $or/$and/$nor/$not branches)
-var isGeoQuery = function (selector) {
-  return _.any(selector, function (val, key) {
-    // Note: _.isObject matches objects and arrays
-    return key === "$near" || (_.isObject(val) && isGeoQuery(val));
-  });
-};
-
-// Checks if $near appears under some $or/$and/$nor/$not branch
-var isGeoQuerySpecial = function (selector) {
-  return _.any(selector, function (val, key) {
-    if (_.contains(['$or', '$and', '$nor', '$not'], key))
-      return isGeoQuery(val);
-    // Note: _.isObject matches objects and arrays
-    return _.isObject(val) && isGeoQuerySpecial(val);
-  });
-};
-

packages/minimongo/minimongo_server_tests.js

@@ -1,6 +1,7 @@
 Tinytest.add("minimongo - modifier affects selector", function (test) {
   function testSelectorPaths (sel, paths, desc) {
-    test.isTrue(_.isEqual(MinimongoTest.getSelectorPaths(sel), paths), desc);
+    var matcher = new Minimongo.Matcher(sel);
+    test.equal(matcher._getPaths(), paths, desc);
   }
 
   testSelectorPaths({
@@ -49,18 +50,24 @@ Tinytest.add("minimongo - modifier affects selector", function (test) {
     }
   }, ['a', 'b.c'], "literal object");
 
+  // Note that a and b do NOT end up in the path list, but x and y both do.
+  testSelectorPaths({
+    $or: [
+      {x: {$elemMatch: {a: 5}}},
+      {y: {$elemMatch: {b: 7}}}
+    ]
+  }, ['x', 'y'], "$or and elemMatch");
+
   function testSelectorAffectedByModifier (sel, mod, yes, desc) {
-    if (yes)
-      test.isTrue(LocalCollection._isSelectorAffectedByModifier(sel, mod, desc));
-    else
-      test.isFalse(LocalCollection._isSelectorAffectedByModifier(sel, mod, desc));
+    var matcher = new Minimongo.Matcher(sel);
+    test.equal(matcher.affectedByModifier(mod), yes, desc);
   }
 
   function affected(sel, mod, desc) {
-    testSelectorAffectedByModifier(sel, mod, 1, desc);
+    testSelectorAffectedByModifier(sel, mod, true, desc);
   }
   function notAffected(sel, mod, desc) {
-    testSelectorAffectedByModifier(sel, mod, 0, desc);
+    testSelectorAffectedByModifier(sel, mod, false, desc);
   }
 
   notAffected({ foo: 0 }, { $set: { bar: 1 } }, "simplest");
@@ -89,7 +96,8 @@ Tinytest.add("minimongo - modifier affects selector", function (test) {
 
 Tinytest.add("minimongo - selector and projection combination", function (test) {
   function testSelProjectionComb (sel, proj, expected, desc) {
-    test.equal(LocalCollection._combineSelectorAndProjection(sel, proj), expected, desc);
+    var matcher = new Minimongo.Matcher(sel);
+    test.equal(matcher.combineIntoProjection(proj), expected, desc);
   }
 
   // Test with inclusive projection
@@ -339,11 +347,15 @@ Tinytest.add("minimongo - selector and projection combination", function (test)
   var test = null; // set this global in the beginning of every test
   // T - should return true
   // F - should return false
+  var oneTest = function (sel, mod, expected, desc) {
+    var matcher = new Minimongo.Matcher(sel);
+    test.equal(matcher.canBecomeTrueByModifier(mod), expected, desc);
+  };
   function T (sel, mod, desc) {
-    test.isTrue(LocalCollection._canSelectorBecomeTrueByModifier(sel, mod), desc);
+    oneTest(sel, mod, true, desc);
   }
   function F (sel, mod, desc) {
-    test.isFalse(LocalCollection._canSelectorBecomeTrueByModifier(sel, mod), desc);
+    oneTest(sel, mod, false, desc);
   }
 
   Tinytest.add("minimongo - can selector become true by modifier - literals (structured tests)", function (t) {

packages/minimongo/minimongo_tests.js

@@ -243,33 +243,49 @@ Tinytest.add("minimongo - misc", function (test) {
 });
 
 Tinytest.add("minimongo - lookup", function (test) {
-  var lookupA = LocalCollection._makeLookupFunction('a');
-  test.equal(lookupA({}), [undefined]);
-  test.equal(lookupA({a: 1}), [1]);
-  test.equal(lookupA({a: [1]}), [[1]]);
+  var lookupA = MinimongoTest.makeLookupFunction('a');
+  test.equal(lookupA({}), [{value: undefined}]);
+  test.equal(lookupA({a: 1}), [{value: 1}]);
+  test.equal(lookupA({a: [1]}), [{value: [1]}]);
 
-  var lookupAX = LocalCollection._makeLookupFunction('a.x');
-  test.equal(lookupAX({a: {x: 1}}), [1]);
-  test.equal(lookupAX({a: {x: [1]}}), [[1]]);
-  test.equal(lookupAX({a: 5}), [undefined]);
+  var lookupAX = MinimongoTest.makeLookupFunction('a.x');
+  test.equal(lookupAX({a: {x: 1}}), [{value: 1}]);
+  test.equal(lookupAX({a: {x: [1]}}), [{value: [1]}]);
+  test.equal(lookupAX({a: 5}), [{value: undefined}]);
   test.equal(lookupAX({a: [{x: 1}, {x: [2]}, {y: 3}]}),
-             [1, [2], undefined]);
+             [{value: 1, arrayIndex: 0},
+              {value: [2], arrayIndex: 1},
+              {value: undefined, arrayIndex: 2}]);
 
-  var lookupA0X = LocalCollection._makeLookupFunction('a.0.x');
-  test.equal(lookupA0X({a: [{x: 1}]}), [1]);
-  test.equal(lookupA0X({a: [{x: [1]}]}), [[1]]);
-  test.equal(lookupA0X({a: 5}), [undefined]);
-  test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), [1]);
+  var lookupA0X = MinimongoTest.makeLookupFunction('a.0.x');
+  test.equal(lookupA0X({a: [{x: 1}]}), [
+    // From interpreting '0' as "0th array element".
+    {value: 1, arrayIndex: 0},
+    // From interpreting '0' as "after branching in the array, look in the
+    // object {x:1} for a field named 0".
+    {value: undefined, arrayIndex: 0}]);
+  test.equal(lookupA0X({a: [{x: [1]}]}), [
+    {value: [1], arrayIndex: 0},
+    {value: undefined, arrayIndex: 0}]);
+  test.equal(lookupA0X({a: 5}), [{value: undefined}]);
+  test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), [
+    // From interpreting '0' as "0th array element".
+    {value: 1, arrayIndex: 0},
+    // From interpreting '0' as "after branching in the array, look in the
+    // object {x:1} for a field named 0".
+    {value: undefined, arrayIndex: 0},
+    {value: undefined, arrayIndex: 1},
+    {value: undefined, arrayIndex: 2}
+  ]);
 });
 
 Tinytest.add("minimongo - selector_compiler", function (test) {
-  var matches = function (should_match, selector, doc) {
-    var does_match = MinimongoTest.matches(selector, doc);
-    if (does_match != should_match) {
+  var matches = function (shouldMatch, selector, doc) {
+    var doesMatch = new Minimongo.Matcher(selector).documentMatches(doc).result;
+    if (doesMatch != shouldMatch) {
       // XXX super janky
-      test.fail({type: "minimongo-ordering",
-                 message: "minimongo match failure: document " +
-                 (should_match ? "should match, but doesn't" :
+      test.fail({message: "minimongo match failure: document " +
+                 (shouldMatch ? "should match, but doesn't" :
                   "shouldn't match, but does"),
                  selector: JSON.stringify(selector),
                  document: JSON.stringify(doc)
@@ -317,6 +333,13 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   match({a: 12, b: 13}, {a: [11, 12, 13], b: [13, 14, 15]});
   nomatch({a: 12, b: 13}, {a: [11, 12, 13], b: [14, 15]});
 
+  // dates
+  var date1 = new Date;
+  var date2 = new Date(date1.getTime() + 1000);
+  match({a: date1}, {a: date1});
+  nomatch({a: date1}, {a: date2});
+
+
   // arrays
   match({a: [1,2]}, {a: [1, 2]});
   match({a: [1,2]}, {a: [[1, 2]]});
@@ -372,7 +395,7 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$lt: 10}}, {a: [11, 12]});
 
   // (there's a full suite of ordering test elsewhere)
-  match({a: {$lt: "null"}}, {a: null}); // tested against mongodb
+  nomatch({a: {$lt: "null"}}, {a: null});
   match({a: {$lt: {x: [2, 3, 4]}}}, {a: {x: [1, 3, 4]}});
   match({a: {$gt: {x: [2, 3, 4]}}}, {a: {x: [3, 3, 4]}});
   nomatch({a: {$gt: {x: [2, 3, 4]}}}, {a: {x: [1, 3, 4]}});
@@ -408,6 +431,17 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$all: [1, 2]}}, {a: [[1, 2]]}); // tested against mongodb
   nomatch({a: {$all: [1, 2]}}, {}); // tested against mongodb, field doesn't exist
   nomatch({a: {$all: [1, 2]}}, {a: {foo: 'bar'}}); // tested against mongodb, field is not an object
+  nomatch({a: {$all: []}}, {a: []});
+  nomatch({a: {$all: []}}, {a: [5]});
+  match({a: {$all: [/i/, /e/i]}}, {a: ["foo", "bEr", "biz"]});
+  nomatch({a: {$all: [/i/, /e/i]}}, {a: ["foo", "bar", "biz"]});
+  match({a: {$all: [{b: 3}]}}, {a: [{b: 3}]});
+  // Members of $all other than regexps are *equality matches*, not document
+  // matches.
+  nomatch({a: {$all: [{b: 3}]}}, {a: [{b: 3, k: 4}]});
+  test.throws(function () {
+    match({a: {$all: [{$gt: 4}]}}, {});
+  });
 
   // $exists
   match({a: {$exists: true}}, {a: 12});
@@ -425,11 +459,32 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$exists: false}}, {a: [1]});
   match({a: {$exists: false}}, {b: [1]});
 
+  match({a: {$exists: 1}}, {a: 5});
+  match({a: {$exists: 0}}, {b: 5});
+
+  nomatch({'a.x':{$exists: false}}, {a: [{}, {x: 5}]});
+  match({'a.x':{$exists: true}}, {a: [{}, {x: 5}]});
+  match({'a.x':{$exists: true}}, {a: [{}, {x: 5}]});
+  match({'a.x':{$exists: true}}, {a: {x: []}});
+  match({'a.x':{$exists: true}}, {a: {x: null}});
+
   // $mod
   match({a: {$mod: [10, 1]}}, {a: 11});
   nomatch({a: {$mod: [10, 1]}}, {a: 12});
   match({a: {$mod: [10, 1]}}, {a: [10, 11, 12]});
   nomatch({a: {$mod: [10, 1]}}, {a: [10, 12]});
+  _.each([
+    5,
+    [10],
+    [10, 1, 2],
+    "foo",
+    {bar: 1},
+    []
+  ], function (badMod) {
+    test.throws(function () {
+      match({a: {$mod: badMod}}, {a: 11});
+    });
+  });
 
   // $ne
   match({a: {$ne: 1}}, {a: 2});
@@ -448,6 +503,17 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   match({a: {$ne: {x: 1}}}, {a: {x: 2}});
   match({a: {$ne: {x: 1}}}, {a: {x: 1, y: 2}});
 
+  // This query means: All 'a.b' must be non-5, and some 'a.b' must be >6.
+  match({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 10}]});
+  nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 4}]});
+  nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 5}]});
+  nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 10}, {b: 5}]});
+  // Should work the same if the branch is at the bottom.
+  match({a: {$ne: 5, $gt: 6}}, {a: [2, 10]});
+  nomatch({a: {$ne: 5, $gt: 6}}, {a: [2, 4]});
+  nomatch({a: {$ne: 5, $gt: 6}}, {a: [2, 5]});
+  nomatch({a: {$ne: 5, $gt: 6}}, {a: [10, 5]});
+
   // $in
   match({a: {$in: [1, 2, 3]}}, {a: 2});
   nomatch({a: {$in: [1, 2, 3]}}, {a: 4});
@@ -461,6 +527,23 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   match({a: {$in: [1, 2, 3]}}, {a: [4, 2]});
   nomatch({a: {$in: [1, 2, 3]}}, {a: [4]});
 
+  match({a: {$in: ['x', /foo/i]}}, {a: 'x'});
+  match({a: {$in: ['x', /foo/i]}}, {a: 'fOo'});
+  match({a: {$in: ['x', /foo/i]}}, {a: ['f', 'fOo']});
+  nomatch({a: {$in: ['x', /foo/i]}}, {a: ['f', 'fOx']});
+
+  match({a: {$in: [1, null]}}, {});
+  match({'a.b': {$in: [1, null]}}, {});
+  match({'a.b': {$in: [1, null]}}, {a: {}});
+  match({'a.b': {$in: [1, null]}}, {a: {b: null}});
+  nomatch({'a.b': {$in: [1, null]}}, {a: {b: 5}});
+  nomatch({'a.b': {$in: [1]}}, {a: {b: null}});
+  nomatch({'a.b': {$in: [1]}}, {a: {}});
+  nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}]});
+  match({'a.b': {$in: [1, null]}}, {a: [{b: 5}, {}]});
+  nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}, []]});
+  nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}, 5]});
+
   // $nin
   nomatch({a: {$nin: [1, 2, 3]}}, {a: 2});
   match({a: {$nin: [1, 2, 3]}}, {a: 4});
@@ -476,6 +559,23 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   match({a: {$nin: [1, 2, 3]}}, {a: [4]});
   match({'a.b': {$nin: [1, 2, 3]}}, {a: [{b:4}]});
 
+  nomatch({a: {$nin: ['x', /foo/i]}}, {a: 'x'});
+  nomatch({a: {$nin: ['x', /foo/i]}}, {a: 'fOo'});
+  nomatch({a: {$nin: ['x', /foo/i]}}, {a: ['f', 'fOo']});
+  match({a: {$nin: ['x', /foo/i]}}, {a: ['f', 'fOx']});
+
+  nomatch({a: {$nin: [1, null]}}, {});
+  nomatch({'a.b': {$nin: [1, null]}}, {});
+  nomatch({'a.b': {$nin: [1, null]}}, {a: {}});
+  nomatch({'a.b': {$nin: [1, null]}}, {a: {b: null}});
+  match({'a.b': {$nin: [1, null]}}, {a: {b: 5}});
+  match({'a.b': {$nin: [1]}}, {a: {b: null}});
+  match({'a.b': {$nin: [1]}}, {a: {}});
+  match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}]});
+  nomatch({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, {}]});
+  match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, []]});
+  match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, 5]});
+
   // $size
   match({a: {$size: 0}}, {a: []});
   match({a: {$size: 1}}, {a: [2]});
@@ -524,6 +624,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$type: 11}}, {a: 'x'});
   nomatch({a: {$type: 11}}, {});
 
+  // The normal rule for {$type:4} (4 means array) is that it NOT good enough to
+  // just have an array that's the leaf that matches the path.  (An array inside
+  // that array is good, though.)
   nomatch({a: {$type: 4}}, {a: []});
   nomatch({a: {$type: 4}}, {a: [1]}); // tested against mongodb
   match({a: {$type: 1}}, {a: [1]});
@@ -535,6 +638,10 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$type: 1}}, {a: ["1", []]});
   match({a: {$type: 2}}, {a: ["1", []]});
   match({a: {$type: 4}}, {a: ["1", []]}); // tested against mongodb
+  // An exception to the normal rule is that an array found via numeric index is
+  // examined itself, and its elements are not.
+  match({'a.0': {$type: 4}}, {a: [[0]]});
+  nomatch({'a.0': {$type: 1}}, {a: [[0]]});
 
   // regular expressions
   match({a: /a/}, {a: 'cat'});
@@ -561,12 +668,24 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: /xxx/}, {});
   nomatch({a: {$regex: 'xxx'}}, {});
 
-  match({a: {$options: 'i'}}, {a: 12});
-  match({b: {$options: 'i'}}, {a: 12});
+  test.throws(function () {
+    match({a: {$options: 'i'}}, {a: 12});
+  });
 
   match({a: /a/}, {a: ['dog', 'cat']});
   nomatch({a: /a/}, {a: ['dog', 'puppy']});
 
+  // we don't support regexps in minimongo very well (eg, there's no EJSON
+  // encoding so it won't go over the wire), but run these tests anyway
+  match({a: /a/}, {a: /a/});
+  match({a: /a/}, {a: ['x', /a/]});
+  nomatch({a: /a/}, {a: /a/i});
+  nomatch({a: /a/m}, {a: /a/});
+  nomatch({a: /a/}, {a: /b/});
+  nomatch({a: /5/}, {a: 5});
+  nomatch({a: /t/}, {a: true});
+  match({a: /m/i}, {a: ['x', 'xM']});
+
   test.throws(function () {
     match({a: {$regex: /a/, $options: 'x'}}, {a: 'cat'});
   });
@@ -600,8 +719,45 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({"a.b": /a/}, {a: {b: "dog"}});
   match({"a.b.c": null}, {});
   match({"a.b.c": null}, {a: 1});
+  match({"a.b": null}, {a: 1});
   match({"a.b.c": null}, {a: {b: 4}});
 
+  // dotted keypaths, nulls, numeric indices, arrays
+  nomatch({"a.b": null}, {a: [1]});
+  match({"a.b": []}, {a: {b: []}});
+  var big = {a: [{b: 1}, 2, {}, {b: [3, 4]}]};
+  match({"a.b": 1}, big);
+  match({"a.b": [3, 4]}, big);
+  match({"a.b": 3}, big);
+  match({"a.b": 4}, big);
+  match({"a.b": null}, big);  // matches on slot 2
+  match({'a.1': 8}, {a: [7, 8, 9]});
+  nomatch({'a.1': 7}, {a: [7, 8, 9]});
+  nomatch({'a.1': null}, {a: [7, 8, 9]});
+  match({'a.1': [8, 9]}, {a: [7, [8, 9]]});
+  nomatch({'a.1': 6}, {a: [[6, 7], [8, 9]]});
+  nomatch({'a.1': 7}, {a: [[6, 7], [8, 9]]});
+  nomatch({'a.1': 8}, {a: [[6, 7], [8, 9]]});
+  nomatch({'a.1': 9}, {a: [[6, 7], [8, 9]]});
+  match({"a.1": 2}, {a: [0, {1: 2}, 3]});
+  match({"a.1": {1: 2}}, {a: [0, {1: 2}, 3]});
+  match({"x.1.y": 8}, {x: [7, {y: 8}, 9]});
+  // comes from trying '1' as key in the plain object
+  match({"x.1.y": null}, {x: [7, {y: 8}, 9]});
+  match({"a.1.b": 9}, {a: [7, {b: 9}, {1: {b: 'foo'}}]});
+  match({"a.1.b": 'foo'}, {a: [7, {b: 9}, {1: {b: 'foo'}}]});
+  match({"a.1.b": null}, {a: [7, {b: 9}, {1: {b: 'foo'}}]});
+  match({"a.1.b": 2}, {a: [1, [{b: 2}], 3]});
+  nomatch({"a.1.b": null}, {a: [1, [{b: 2}], 3]});
+  // this is new behavior in mongo 2.5
+  nomatch({"a.0.b": null}, {a: [5]});
+  match({"a.1": 4}, {a: [{1: 4}, 5]});
+  match({"a.1": 5}, {a: [{1: 4}, 5]});
+  nomatch({"a.1": null}, {a: [{1: 4}, 5]});
+  match({"a.1.foo": 4}, {a: [{1: {foo: 4}}, {foo: 5}]});
+  match({"a.1.foo": 5}, {a: [{1: {foo: 4}}, {foo: 5}]});
+  match({"a.1.foo": null}, {a: [{1: {foo: 4}}, {foo: 5}]});
+
   // trying to access a dotted field that is undefined at some point
   // down the chain
   nomatch({"a.b": 1}, {x: 2});
@@ -628,6 +784,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   test.throws(function () {
     match({$or: []}, {});
   });
+  test.throws(function () {
+    match({$or: [5]}, {});
+  });
   test.throws(function () {
     match({$or: []}, {a: 1});
   });
@@ -709,6 +868,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   test.throws(function () {
     match({$nor: []}, {});
   });
+  test.throws(function () {
+    match({$nor: [5]}, {});
+  });
   test.throws(function () {
     match({$nor: []}, {a: 1});
   });
@@ -787,6 +949,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   test.throws(function () {
     match({$and: []}, {});
   });
+  test.throws(function () {
+    match({$and: [5]}, {});
+  });
   test.throws(function () {
     match({$and: []}, {a: 1});
   });
@@ -866,7 +1031,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
 
   // $where
   match({$where: "this.a === 1"}, {a: 1});
+  match({$where: "obj.a === 1"}, {a: 1});
   nomatch({$where: "this.a !== 1"}, {a: 1});
+  nomatch({$where: "obj.a !== 1"}, {a: 1});
   nomatch({$where: "this.a === 1", a: 2}, {a: 1});
   match({$where: "this.a === 1", b: 2}, {a: 1, b: 2});
   match({$where: "this.a === 1 && this.b === 2"}, {a: 1, b: 2});
@@ -909,6 +1076,19 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
         {dogs: [{name: "Fido", age: 5}, {name: "Rex", age: 3}]});
   nomatch({dogs: {$elemMatch: {name: /e/, age: 5}}},
           {dogs: [{name: "Fido", age: 5}, {name: "Rex", age: 3}]});
+  match({x: {$elemMatch: {y: 9}}}, {x: [{y: 9}]});
+  nomatch({x: {$elemMatch: {y: 9}}}, {x: [[{y: 9}]]});
+  match({x: {$elemMatch: {$gt: 5, $lt: 9}}}, {x: [8]});
+  nomatch({x: {$elemMatch: {$gt: 5, $lt: 9}}}, {x: [[8]]});
+  match({'a.x': {$elemMatch: {y: 9}}},
+        {a: [{x: []}, {x: [{y: 9}]}]});
+  nomatch({a: {$elemMatch: {x: 5}}}, {a: {x: 5}});
+  match({a: {$elemMatch: {0: {$gt: 5, $lt: 9}}}}, {a: [[6]]});
+  match({a: {$elemMatch: {'0.b': {$gt: 5, $lt: 9}}}}, {a: [[{b:6}]]});
+
+  // $comment
+  match({a: 5, $comment: "asdf"}, {a: 5});
+  nomatch({a: 6, $comment: "asdf"}, {a: 5});
 
   // XXX still needs tests:
   // - non-scalar arguments to $gt, $lt, etc
@@ -1259,6 +1439,14 @@ Tinytest.add("minimongo - observe ordered with projection", function (test) {
   c.insert({_id: idA2, a:2});
   test.equal(operations.shift(), undefined);
 
+  var cursor = c.find({}, {fields: {a: 1, _id: 0}});
+  test.throws(function () {
+    cursor.observeChanges({added: function () {}});
+  });
+  test.throws(function () {
+    cursor.observe({added: function () {}});
+  });
+
   // test initial inserts (and backwards sort)
   handle = c.find({}, {sort: {a: -1}, fields: { a: 1 } }).observe(cbs);
   test.equal(operations.shift(), ['added', {a:2}, 0, null]);
@@ -1345,8 +1533,9 @@ Tinytest.add("minimongo - ordering", function (test) {
 
   // document ordering under a sort specification
   var verify = function (sorts, docs) {
-    _.each(sorts, function (sort) {
-      assert_ordering(test, LocalCollection._compileSort(sort), docs);
+    _.each(_.isArray(sorts) ? sorts : [sorts], function (sort) {
+      var sorter = new MinimongoTest.Sorter(sort);
+      assert_ordering(test, sorter.getComparator(), docs);
     });
   };
 
@@ -1373,14 +1562,94 @@ Tinytest.add("minimongo - ordering", function (test) {
          [{c: 1}, {a: 1, b: 2}, {a: 1, b: 3}, {a: 2, b: 0}]);
 
   test.throws(function () {
-    LocalCollection._compileSort("a");
+    new MinimongoTest.Sorter("a");
   });
 
   test.throws(function () {
-    LocalCollection._compileSort(123);
+    new MinimongoTest.Sorter(123);
   });
 
-  test.equal(LocalCollection._compileSort({})({a:1}, {a:2}), 0);
+  // No sort spec implies everything equal.
+  test.equal(new MinimongoTest.Sorter({}).getComparator()({a:1}, {a:2}), 0);
+
+  // All sorts of array edge cases!
+  // Increasing sort sorts by the smallest element it finds; 1 < 2.
+  verify({a: 1}, [
+    {a: [1, 10, 20]},
+    {a: [5, 2, 99]}
+  ]);
+  // Decreasing sorts by largest it finds; 99 > 20.
+  verify({a: -1}, [
+    {a: [5, 2, 99]},
+    {a: [1, 10, 20]}
+  ]);
+  // Can also sort by specific array indices.
+  verify({'a.1': 1}, [
+    {a: [5, 2, 99]},
+    {a: [1, 10, 20]}
+  ]);
+  // We do NOT expand sub-arrays, so the minimum in the second doc is 5, not
+  // -20. (Numbers always sort before arrays.)
+  verify({a: 1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [-5, -20], 18]}
+  ]);
+  // The maximum in each of these is the array, since arrays are "greater" than
+  // numbers. And [10, 15] is greater than [-5, -20].
+  verify({a: -1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [-5, -20], 18]}
+  ]);
+  // 'a.0' here ONLY means "first element of a", not "first element of something
+  // found in a", so it CANNOT find the 10 or -5.
+  verify({'a.0': 1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [-5, -20], 18]}
+  ]);
+  verify({'a.0': -1}, [
+    {a: [5, [-5, -20], 18]},
+    {a: [1, [10, 15], 20]}
+  ]);
+  // Similarly, this is just comparing [-5,-20] to [10, 15].
+  verify({'a.1': 1}, [
+    {a: [5, [-5, -20], 18]},
+    {a: [1, [10, 15], 20]}
+  ]);
+  verify({'a.1': -1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [-5, -20], 18]}
+  ]);
+  // Here we are just comparing [10,15] directly to [19,3] (and NOT also
+  // iterating over the numbers; this is implemented by setting dontIterate in
+  // makeLookupFunction).  So [10,15]<[19,3] even though 3 is the smallest
+  // number you can find there.
+  verify({'a.1': 1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [19, 3], 18]}
+  ]);
+  verify({'a.1': -1}, [
+    {a: [5, [19, 3], 18]},
+    {a: [1, [10, 15], 20]}
+  ]);
+  // Minimal elements are 1 and 5.
+  verify({a: 1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [19, 3], 18]}
+  ]);
+  // Maximal elements are [19,3] and [10,15] (because arrays sort higher than
+  // numbers), even though there's a 20 floating around.
+  verify({a: -1}, [
+    {a: [5, [19, 3], 18]},
+    {a: [1, [10, 15], 20]}
+  ]);
+  // Maximal elements are [10,15] and [3,19].  [10,15] is bigger even though 19
+  // is the biggest number in them, because array comparison is lexicographic.
+  verify({a: -1}, [
+    {a: [1, [10, 15], 20]},
+    {a: [5, [3, 19], 18]}
+  ]);
+
+
 });
 
 Tinytest.add("minimongo - sort", function (test) {
@@ -1529,26 +1798,27 @@ Tinytest.add("minimongo - binary search", function (test) {
 });
 
 Tinytest.add("minimongo - modify", function (test) {
-  var modify = function (doc, mod, result) {
-    var copy = EJSON.clone(doc);
-    LocalCollection._modify(copy, mod);
-    if (!LocalCollection._f._equal(copy, result)) {
-      // XXX super janky
-      test.fail({type: "minimongo-modifier",
-                 message: "modifier test failure",
-                 input_doc: JSON.stringify(doc),
-                 modifier: JSON.stringify(mod),
-                 expected: JSON.stringify(result),
-                 actual: JSON.stringify(copy)
-                });
-    } else {
-      test.ok();
-    }
+  var modifyWithQuery = function (doc, query, mod, expected) {
+    var coll = new LocalCollection;
+    coll.insert(doc);
+    // The query is relevant for 'a.$.b'.
+    coll.update(query, mod);
+    var actual = coll.findOne();
+    delete actual._id;  // added by insert
+    test.equal(actual, expected, EJSON.stringify({input: doc, mod: mod}));
+  };
+  var modify = function (doc, mod, expected) {
+    modifyWithQuery(doc, {}, mod, expected);
+  };
+  var exceptionWithQuery = function (doc, query, mod) {
+    var coll = new LocalCollection;
+    coll.insert(doc);
+    test.throws(function () {
+      coll.update(query, mod);
+    });
   };
   var exception = function (doc, mod) {
-    test.throws(function () {
-      LocalCollection._modify(EJSON.clone(doc), mod);
-    });
+    exceptionWithQuery(doc, {}, mod);
   };
 
   // document replacement
@@ -1574,18 +1844,13 @@ Tinytest.add("minimongo - modify", function (test) {
   modify({a: [null, null, null]}, {$set: {'a.3.b': 12}}, {
     a: [null, null, null, {b: 12}]});
   exception({a: []}, {$set: {'a.b': 12}});
-  test.expect_fail();
   exception({a: 12}, {$set: {'a.b': 99}}); // tested on mongo
-  test.expect_fail();
   exception({a: 'x'}, {$set: {'a.b': 99}});
-  test.expect_fail();
   exception({a: true}, {$set: {'a.b': 99}});
-  test.expect_fail();
   exception({a: null}, {$set: {'a.b': 99}});
   modify({a: {}}, {$set: {'a.3': 12}}, {a: {'3': 12}});
   modify({a: []}, {$set: {'a.3': 12}}, {a: [null, null, null, 12]});
   modify({}, {$set: {'': 12}}, {'': 12}); // tested on mongo
-  test.expect_fail();
   exception({}, {$set: {'.': 12}}); // tested on mongo
   modify({}, {$set: {'. ': 12}}, {'': {' ': 12}}); // tested on mongo
   modify({}, {$inc: {'... ': 12}}, {'': {'': {'': {' ': 12}}}}); // tested
@@ -1597,6 +1862,62 @@ Tinytest.add("minimongo - modify", function (test) {
   modify({x: [null, null]}, {$set: {'x.2.a': 1}}, {x: [null, null, {a: 1}]});
   exception({x: [null, null]}, {$set: {'x.1.a': 1}});
 
+  // a.$.b
+  modifyWithQuery({a: [{x: 2}, {x: 4}]}, {'a.x': 4}, {$set: {'a.$.z': 9}},
+                  {a: [{x: 2}, {x: 4, z: 9}]});
+  exception({a: [{x: 2}, {x: 4}]}, {$set: {'a.$.z': 9}});
+  exceptionWithQuery({a: [{x: 2}, {x: 4}], b: 5}, {b: 5}, {$set: {'a.$.z': 9}});
+  // can't have two $
+  exceptionWithQuery({a: [{x: [2]}]}, {'a.x': 2}, {$set: {'a.$.x.$': 9}});
+  modifyWithQuery({a: [5, 6, 7]}, {a: 6}, {$set: {'a.$': 9}}, {a: [5, 9, 7]});
+  modifyWithQuery({a: [{b: [{c: 9}, {c: 10}]}, {b: {c: 11}}]}, {'a.b.c': 10},
+                  {$unset: {'a.$.b': 1}}, {a: [{}, {b: {c: 11}}]});
+  modifyWithQuery({a: [{b: [{c: 9}, {c: 10}]}, {b: {c: 11}}]}, {'a.b.c': 11},
+                  {$unset: {'a.$.b': 1}},
+                  {a: [{b: [{c: 9}, {c: 10}]}, {}]});
+  modifyWithQuery({a: [1]}, {'a.0': 1}, {$set: {'a.$': 5}}, {a: [5]});
+  modifyWithQuery({a: [9]}, {a: {$mod: [2, 1]}}, {$set: {'a.$': 5}}, {a: [5]});
+  // Negatives don't set '$'.
+  exceptionWithQuery({a: [1]}, {$not: {a: 2}}, {$set: {'a.$': 5}});
+  exceptionWithQuery({a: [1]}, {'a.0': {$ne: 2}}, {$set: {'a.$': 5}});
+  // One $or clause works.
+  modifyWithQuery({a: [{x: 2}, {x: 4}]},
+                  {$or: [{'a.x': 4}]}, {$set: {'a.$.z': 9}},
+                  {a: [{x: 2}, {x: 4, z: 9}]});
+  // More $or clauses throw.
+  exceptionWithQuery({a: [{x: 2}, {x: 4}]},
+                     {$or: [{'a.x': 4}, {'a.x': 4}]},
+                     {$set: {'a.$.z': 9}});
+  // $and uses the last one.
+  modifyWithQuery({a: [{x: 1}, {x: 3}]},
+                  {$and: [{'a.x': 1}, {'a.x': 3}]},
+                  {$set: {'a.$.x': 5}},
+                  {a: [{x: 1}, {x: 5}]});
+  modifyWithQuery({a: [{x: 1}, {x: 3}]},
+                  {$and: [{'a.x': 3}, {'a.x': 1}]},
+                  {$set: {'a.$.x': 5}},
+                  {a: [{x: 5}, {x: 3}]});
+  // Same goes for the implicit AND of a document selector.
+  modifyWithQuery({a: [{x: 1}, {y: 3}]},
+                  {'a.x': 1, 'a.y': 3},
+                  {$set: {'a.$.z': 5}},
+                  {a: [{x: 1}, {y: 3, z: 5}]});
+  // with $near, make sure it finds the closest one
+  modifyWithQuery({a: [{b: [1,1]},
+                       {b: [ [3,3], [4,4] ]},
+                       {b: [9,9]}]},
+                  {'a.b': {$near: [5, 5]}},
+                  {$set: {'a.$.b': 'k'}},
+                  {a: [{b: [1,1]}, {b: 'k'}, {b: [9,9]}]});
+  modifyWithQuery({a: [{x: 1}, {y: 1}, {x: 1, y: 1}]},
+                  {a: {$elemMatch: {x: 1, y: 1}}},
+                  {$set: {'a.$.x': 2}},
+                  {a: [{x: 1}, {y: 1}, {x: 2, y: 1}]});
+  modifyWithQuery({a: [{b: [{x: 1}, {y: 1}, {x: 1, y: 1}]}]},
+                  {'a.b': {$elemMatch: {x: 1, y: 1}}},
+                  {$set: {'a.$.b': 3}},
+                  {a: [{b: 3}]});
+
   // $inc
   modify({a: 1, b: 2}, {$inc: {a: 10}}, {a: 11, b: 2});
   modify({a: 1, b: 2}, {$inc: {c: 10}}, {a: 1, b: 2, c: 10});
@@ -1777,10 +2098,11 @@ Tinytest.add("minimongo - modify", function (test) {
          {a: {}, q: {2: {r: 12}}});
   exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2'}}); // tested
   exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2.r'}}); // tested
-  test.expect_fail();
-  exception({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}}); // tested
-  test.expect_fail();
-  exception({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}}); // tested
+  // These strange MongoDB behaviors throw.
+  // modify({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}},
+  //        {a: {b: 12}, x: []}); // tested
+  // modify({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}},
+  //        {a: {b: 12}, x: []}); // tested
   exception({}, {$rename: {'a': 'a'}});
   exception({}, {$rename: {'a.b': 'a.b'}});
   modify({a: 12, b: 13}, {$rename: {a: 'b'}}, {b: 12});
@@ -2451,5 +2773,56 @@ Tinytest.add("minimongo - $near operator tests", function (test) {
       }]
     });
   });
+
+  // array tests
+  coll = new LocalCollection();
+  coll.insert({
+    _id: "x",
+    k: 9,
+    a: [
+      {b: [
+        [100, 100],
+        [1,  1]]},
+      {b: [150,  150]}]});
+  coll.insert({
+    _id: "y",
+    k: 9,
+    a: {b: [5, 5]}});
+  var testNear = function (near, md, expected) {
+    test.equal(
+      _.pluck(
+        coll.find({'a.b': {$near: near, $maxDistance: md}}).fetch(), '_id'),
+      expected);
+  };
+  testNear([149, 149], 4, ['x']);
+  testNear([149, 149], 1000, ['x', 'y']);
+  // It's important that we figure out that 'x' is closer than 'y' to [2,2] even
+  // though the first within-1000 point in 'x' (ie, [100,100]) is farther than
+  // 'y'.
+  testNear([2, 2], 1000, ['x', 'y']);
+
+  // Ensure that distance is used as a tie-breaker for sort.
+  test.equal(
+    _.pluck(coll.find({'a.b': {$near: [1, 1]}}, {sort: {k: 1}}).fetch(), '_id'),
+    ['x', 'y']);
+  test.equal(
+    _.pluck(coll.find({'a.b': {$near: [5, 5]}}, {sort: {k: 1}}).fetch(), '_id'),
+    ['y', 'x']);
+
+  var operations = [];
+  var cbs = log_callbacks(operations);
+  var handle = coll.find({'a.b': {$near: [7,7]}}).observe(cbs);
+
+  test.length(operations, 2);
+  test.equal(operations.shift(), ['added', {k:9, a:{b:[5,5]}}, 0, null]);
+  test.equal(operations.shift(),
+             ['added', {k: 9, a:[{b:[[100,100],[1,1]]},{b:[150,150]}]},
+              1, null]);
+  // This needs to be inserted in the MIDDLE of the two existing ones.
+  coll.insert({a: {b: [3,3]}});
+  test.length(operations, 1);
+  test.equal(operations.shift(), ['added', {a: {b: [3, 3]}}, 1, 'x']);
+
+  handle.stop();
 });
 

packages/minimongo/modify.js

@@ -6,63 +6,59 @@
 // XXX atomicity: if one modification fails, do we roll back the whole
 // change?
 //
-// isInsert is set when _modify is being called to compute the document to
-// insert as part of an upsert operation. We use this primarily to figure out
-// when to set the fields in $setOnInsert, if present.
-LocalCollection._modify = function (doc, mod, isInsert) {
-  var is_modifier = false;
-  for (var k in mod) {
-    // IE7 doesn't support indexing into strings (eg, k[0]), so use substr.
-    // Too bad -- it's far slower:
-    // http://jsperf.com/testing-the-first-character-of-a-string
-    is_modifier = k.substr(0, 1) === '$';
-    break; // just check the first key.
-  }
+// options:
+//   - isInsert is set when _modify is being called to compute the document to
+//     insert as part of an upsert operation. We use this primarily to figure
+//     out when to set the fields in $setOnInsert, if present.
+LocalCollection._modify = function (doc, mod, options) {
+  options = options || {};
+  if (!isPlainObject(mod))
+    throw MinimongoError("Modifier must be an object");
+  var isModifier = isOperatorObject(mod);
 
-  var new_doc;
+  var newDoc;
 
-  if (!is_modifier) {
+  if (!isModifier) {
     if (mod._id && !EJSON.equals(doc._id, mod._id))
       throw MinimongoError("Cannot change the _id of a document");
 
     // replace the whole document
     for (var k in mod) {
-      if (k.substr(0, 1) === '$')
-        throw MinimongoError(
-          "When replacing document, field name may not start with '$'");
       if (/\./.test(k))
         throw MinimongoError(
           "When replacing document, field name may not contain '.'");
     }
-    new_doc = mod;
+    newDoc = mod;
   } else {
-    // apply modifiers
-    var new_doc = EJSON.clone(doc);
+    // apply modifiers to the doc.
+    newDoc = EJSON.clone(doc);
 
-    for (var op in mod) {
-      var mod_func = LocalCollection._modifiers[op];
+    _.each(mod, function (operand, op) {
+      var modFunc = MODIFIERS[op];
       // Treat $setOnInsert as $set if this is an insert.
-      if (isInsert && op === '$setOnInsert')
-        mod_func = LocalCollection._modifiers['$set'];
-      if (!mod_func)
+      if (options.isInsert && op === '$setOnInsert')
+        modFunc = MODIFIERS['$set'];
+      if (!modFunc)
         throw MinimongoError("Invalid modifier specified " + op);
-      for (var keypath in mod[op]) {
+      _.each(operand, function (arg, keypath) {
         // XXX mongo doesn't allow mod field names to end in a period,
         // but I don't see why.. it allows '' as a key, as does JS
         if (keypath.length && keypath[keypath.length-1] === '.')
           throw MinimongoError(
             "Invalid mod field name, may not end in a period");
 
-        var arg = mod[op][keypath];
         var keyparts = keypath.split('.');
-        var no_create = !!LocalCollection._noCreateModifiers[op];
-        var forbid_array = (op === "$rename");
-        var target = LocalCollection._findModTarget(new_doc, keyparts,
-                                                    no_create, forbid_array);
+        var noCreate = _.has(NO_CREATE_MODIFIERS, op);
+        var forbidArray = (op === "$rename");
+        var target = findModTarget(newDoc, keyparts, {
+          noCreate: NO_CREATE_MODIFIERS[op],
+          forbidArray: (op === "$rename"),
+          arrayIndex: options.arrayIndex
+        });
         var field = keyparts.pop();
-        mod_func(target, field, arg, keypath, new_doc);
-      }
-    }
+        modFunc(target, field, arg, keypath, newDoc);
+      });
+    });
   }
 
   // move new document into place.
@@ -74,43 +70,71 @@ LocalCollection._modify = function (doc, mod, isInsert) {
     // isInsert: if we're constructing a document to insert (via upsert)
     // and we're in replacement mode, not modify mode, DON'T take the
     // _id from the query.  This matches mongo's behavior.
-    if (k !== '_id' || isInsert)
+    if (k !== '_id' || options.isInsert)
       delete doc[k];
   });
-  for (var k in new_doc) {
-    doc[k] = new_doc[k];
-  }
+  _.each(newDoc, function (v, k) {
+    doc[k] = v;
+  });
 };
 
 // for a.b.c.2.d.e, keyparts should be ['a', 'b', 'c', '2', 'd', 'e'],
 // and then you would operate on the 'e' property of the returned
-// object. if no_create is falsey, creates intermediate levels of
+// object.
+//
+// if options.noCreate is falsey, creates intermediate levels of
 // structure as necessary, like mkdir -p (and raises an exception if
 // that would mean giving a non-numeric property to an array.) if
-// no_create is true, return undefined instead. may modify the last
-// element of keyparts to signal to the caller that it needs to use a
-// different value to index into the returned object (for example,
-// ['a', '01'] -> ['a', 1]). if forbid_array is true, return null if
-// the keypath goes through an array.
-LocalCollection._findModTarget = function (doc, keyparts, no_create,
-                                      forbid_array) {
+// options.noCreate is true, return undefined instead.
+//
+// may modify the last element of keyparts to signal to the caller that it needs
+// to use a different value to index into the returned object (for example,
+// ['a', '01'] -> ['a', 1]).
+//
+// if forbidArray is true, return null if the keypath goes through an array.
+//
+// if options.arrayIndex is defined, use this for the (first) '$' in the path.
+var findModTarget = function (doc, keyparts, options) {
+  options = options || {};
+  var usedArrayIndex = false;
   for (var i = 0; i < keyparts.length; i++) {
     var last = (i === keyparts.length - 1);
     var keypart = keyparts[i];
-    var numeric = /^[0-9]+$/.test(keypart);
-    if (no_create && (!(typeof doc === "object") || !(keypart in doc)))
-      return undefined;
+    var indexable = isIndexable(doc);
+    if (!indexable) {
+      if (options.noCreate)
+        return undefined;
+      var e = MinimongoError(
+        "cannot use the part '" + keypart + "' to traverse " + doc);
+      e.setPropertyError = true;
+      throw e;
+    }
     if (doc instanceof Array) {
-      if (forbid_array)
+      if (options.forbidArray)
         return null;
-      if (!numeric)
+      if (keypart === '$') {
+        if (usedArrayIndex)
+          throw MinimongoError("Too many positional (i.e. '$') elements");
+        if (options.arrayIndex === undefined) {
+          throw MinimongoError("The positional operator did not find the " +
+                               "match needed from the query");
+        }
+        keypart = options.arrayIndex;
+        usedArrayIndex = true;
+      } else if (isNumericKey(keypart)) {
+        keypart = parseInt(keypart);
+      } else {
+        if (options.noCreate)
+          return undefined;
         throw MinimongoError(
           "can't append to array using string field name ["
                     + keypart + "]");
-      keypart = parseInt(keypart);
+      }
       if (last)
         // handle 'a.01'
         keyparts[i] = keypart;
+      if (options.noCreate && keypart >= doc.length)
+        return undefined;
       while (doc.length < keypart)
         doc.push(null);
       if (!last) {
@@ -121,9 +145,14 @@ LocalCollection._findModTarget = function (doc, keyparts, no_create,
                       "' of list value " + JSON.stringify(doc[keypart]));
       }
     } else {
-      // XXX check valid fieldname (no $ at start, no .)
-      if (!last && !(keypart in doc))
-        doc[keypart] = {};
+      if (keypart.length && keypart.substr(0, 1) === '$')
+        throw MinimongoError("can't set field named " + keypart);
+      if (!(keypart in doc)) {
+        if (options.noCreate)
+          return undefined;
+        if (!last)
+          doc[keypart] = {};
+      }
     }
 
     if (last)
@@ -134,7 +163,7 @@ LocalCollection._findModTarget = function (doc, keyparts, no_create,
   // notreached
 };
 
-LocalCollection._noCreateModifiers = {
+var NO_CREATE_MODIFIERS = {
   $unset: true,
   $pop: true,
   $rename: true,
@@ -142,7 +171,7 @@ LocalCollection._noCreateModifiers = {
   $pullAll: true
 };
 
-LocalCollection._modifiers = {
+var MODIFIERS = {
   $inc: function (target, field, arg) {
     if (typeof arg !== "number")
       throw MinimongoError("Modifier $inc allowed for numbers only");
@@ -218,7 +247,8 @@ LocalCollection._modifiers = {
       // XXX this allows us to use a $sort whose value is an array, but that's
       // actually an extension of the Node driver, so it won't work
       // server-side. Could be confusing!
-      sortFunction = LocalCollection._compileSort(arg.$sort);
+      // XXX is it correct that we don't do geo-stuff here?
+      sortFunction = new Sorter(arg.$sort).getComparator();
       for (var i = 0; i < toPush.length; i++) {
         if (LocalCollection._f._type(toPush[i]) !== 3) {
           throw MinimongoError("$push like modifiers using $sort " +
@@ -304,21 +334,21 @@ LocalCollection._modifiers = {
     else if (!(x instanceof Array))
       throw MinimongoError("Cannot apply $pull/pullAll modifier to non-array");
     else {
-      var out = []
+      var out = [];
       if (typeof arg === "object" && !(arg instanceof Array)) {
         // XXX would be much nicer to compile this once, rather than
         // for each document we modify.. but usually we're not
         // modifying that many documents, so we'll let it slide for
         // now
 
-        // XXX _compileSelector isn't up for the job, because we need
+        // XXX Minimongo.Matcher isn't up for the job, because we need
         // to permit stuff like {$pull: {a: {$gt: 4}}}.. something
         // like {$gt: 4} is not normally a complete selector.
         // same issue as $elemMatch possibly?
-        var match = LocalCollection._compileSelector(arg);
+        var matcher = new Minimongo.Matcher(arg);
         for (var i = 0; i < x.length; i++)
-          if (!match(x[i]))
-            out.push(x[i])
+          if (!matcher.documentMatches(x[i]).result)
+            out.push(x[i]);
       } else {
         for (var i = 0; i < x.length; i++)
           if (!LocalCollection._f._equal(x[i], arg))
@@ -338,7 +368,7 @@ LocalCollection._modifiers = {
     else if (!(x instanceof Array))
       throw MinimongoError("Cannot apply $pull/pullAll modifier to non-array");
     else {
-      var out = []
+      var out = [];
       for (var i = 0; i < x.length; i++) {
         var exclude = false;
         for (var j = 0; j < arg.length; j++) {
@@ -367,7 +397,7 @@ LocalCollection._modifiers = {
     delete target[field];
 
     var keyparts = arg.split('.');
-    var target2 = LocalCollection._findModTarget(doc, keyparts, false, true);
+    var target2 = findModTarget(doc, keyparts, {forbidArray: true});
     if (target2 === null)
       throw MinimongoError("$rename target field invalid");
     var field2 = keyparts.pop();
@@ -379,12 +409,3 @@ LocalCollection._modifiers = {
     throw MinimongoError("$bit is not supported");
   }
 };
-
-LocalCollection._removeDollarOperators = function (selector) {
-  var selectorDoc = {};
-  for (var k in selector)
-    if (k.substr(0, 1) !== '$')
-      selectorDoc[k] = selector[k];
-  return selectorDoc;
-};
-

packages/minimongo/package.js

@@ -5,6 +5,7 @@ Package.describe({
 
 Package.on_use(function (api) {
   api.export('LocalCollection');
+  api.export('Minimongo');
   api.export('MinimongoTest', { testOnly: true });
   api.use(['underscore', 'json', 'ejson', 'ordered-dict', 'deps',
            'random', 'ordered-dict']);
@@ -12,7 +13,9 @@ Package.on_use(function (api) {
   api.use('geojson-utils');
   api.add_files([
     'minimongo.js',
+    'helpers.js',
     'selector.js',
+    'sort.js',
     'projection.js',
     'modify.js',
     'diff.js',

packages/minimongo/selector_modifier.js

@@ -6,11 +6,12 @@
 //    - 'foo.bar': 42
 //  - $unset
 //    - 'abc.d': 1
-LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) {
+Minimongo.Matcher.prototype.affectedByModifier = function (modifier) {
+  var self = this;
   // safe check for $set/$unset being objects
   modifier = _.extend({ $set: {}, $unset: {} }, modifier);
   var modifiedPaths = _.keys(modifier.$set).concat(_.keys(modifier.$unset));
-  var meaningfulPaths = getPaths(selector);
+  var meaningfulPaths = self._getPaths();
 
   return _.any(modifiedPaths, function (path) {
     var mod = path.split('.');
@@ -19,17 +20,17 @@ LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) {
       var i = 0, j = 0;
 
       while (i < sel.length && j < mod.length) {
-        if (numericKey(sel[i]) && numericKey(mod[j])) {
+        if (isNumericKey(sel[i]) && isNumericKey(mod[j])) {
           // foo.4.bar selector affected by foo.4 modifier
           // foo.3.bar selector unaffected by foo.4 modifier
           if (sel[i] === mod[j])
             i++, j++;
           else
             return false;
-        } else if (numericKey(sel[i])) {
+        } else if (isNumericKey(sel[i])) {
           // foo.4.bar selector unaffected by foo.bar modifier
           return false;
-        } else if (numericKey(mod[j])) {
+        } else if (isNumericKey(mod[j])) {
           j++;
         } else if (sel[i] === mod[j])
           i++, j++;
@@ -43,42 +44,33 @@ LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) {
   });
 };
 
-getPathsWithoutNumericKeys = function (sel) {
-  return _.map(getPaths(sel), function (path) {
-    return _.reject(path.split('.'), numericKey).join('.');
-  });
-};
-
-// @param selector - Object: MongoDB selector. Currently doesn't support
-//                           $-operators and arrays well.
 // @param modifier - Object: MongoDB-styled modifier with `$set`s and `$unsets`
 //                           only. (assumed to come from oplog)
 // @returns - Boolean: if after applying the modifier, selector can start
 //                     accepting the modified value.
-LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier)
-{
-  if (!LocalCollection._isSelectorAffectedByModifier(selector, modifier))
+// Currently doesn't support $-operators and numeric indices precisely.
+Minimongo.Matcher.prototype.canBecomeTrueByModifier = function (modifier) {
+  var self = this;
+  if (!this.affectedByModifier(modifier))
     return false;
 
   modifier = _.extend({$set:{}, $unset:{}}, modifier);
 
-  if (_.any(_.keys(selector), pathHasNumericKeys) ||
-      _.any(_.keys(modifier.$unset), pathHasNumericKeys) ||
-      _.any(_.keys(modifier.$set), pathHasNumericKeys))
+  if (!self.isEquality())
     return true;
 
-  if (!isLiteralSelector(selector))
+  if (_.any(self._getPaths(), pathHasNumericKeys) ||
+      _.any(_.keys(modifier.$unset), pathHasNumericKeys) ||
+      _.any(_.keys(modifier.$set), pathHasNumericKeys))
     return true;
 
   // convert a selector into an object matching the selector
   // { 'a.b': { ans: 42 }, 'foo.bar': null, 'foo.baz': "something" }
   // => { a: { b: { ans: 42 } }, foo: { bar: null, baz: "something" } }
-  var doc = pathsToTree(_.keys(selector),
-                        function (path) { return selector[path]; },
+  var doc = pathsToTree(self._getPaths(),
+                        function (path) { return self._selector[path]; },
                         _.identity /*conflict resolution is no resolution*/);
 
-  var selectorFn = LocalCollection._compileSelector(selector);
-
   try {
     LocalCollection._modify(doc, modifier);
   } catch (e) {
@@ -97,11 +89,11 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier)
     throw e;
   }
 
-  return selectorFn(doc);
+  return self.documentMatches(doc).result;
 };
 
-// Returns a list of key paths the given selector is looking for
-var getPaths = MinimongoTest.getSelectorPaths = function (sel) {
+var getPaths = function (sel) {
+  return _.keys(new Minimongo.Matcher(sel)._paths);
   return _.chain(sel).map(function (v, k) {
     // we don't know how to handle $where because it can be anything
     if (k === "$where")
@@ -115,23 +107,6 @@ var getPaths = MinimongoTest.getSelectorPaths = function (sel) {
 };
 
 function pathHasNumericKeys (path) {
-  return _.any(path.split('.'), numericKey);
-}
-
-// string can be converted to integer
-function numericKey (s) {
-  return /^[0-9]+$/.test(s);
-}
-
-function isLiteralSelector (selector) {
-  return _.all(selector, function (subSelector, keyPath) {
-    if (keyPath.substr(0, 1) === "$" || _.isRegExp(subSelector))
-      return false;
-    if (!_.isObject(subSelector) || _.isArray(subSelector))
-      return true;
-    return _.all(subSelector, function (value, key) {
-      return key.substr(0, 1) !== "$";
-    });
-  });
+  return _.any(path.split('.'), isNumericKey);
 }
 

packages/minimongo/selector_projection.js

@@ -1,9 +1,9 @@
 // Knows how to combine a mongo selector and a fields projection to a new fields
 // projection taking into account active fields from the passed selector.
 // @returns Object - projection object (same as fields option of mongo cursor)
-LocalCollection._combineSelectorAndProjection = function (selector, projection)
-{
-  var selectorPaths = getPathsWithoutNumericKeys(selector);
+Minimongo.Matcher.prototype.combineIntoProjection = function (projection) {
+  var self = this;
+  var selectorPaths = self._getPathsElidingNumericKeys();
 
   // Special case for $where operator in the selector - projection should depend
   // on all fields of the document. getSelectorPaths returns a list of paths
@@ -40,6 +40,13 @@ LocalCollection._combineSelectorAndProjection = function (selector, projection)
   }
 };
 
+Minimongo.Matcher.prototype._getPathsElidingNumericKeys = function () {
+  var self = this;
+  return _.map(self._getPaths(), function (path) {
+    return _.reject(path.split('.'), isNumericKey).join('.');
+  });
+};
+
 // Returns a set of key paths similar to
 // { 'foo.bar': 1, 'a.b.c': 1 }
 var treeToPaths = function (tree, prefix) {

packages/minimongo/sort.js

@@ -0,0 +1,128 @@
+// Give a sort spec, which can be in any of these forms:
+//   {"key1": 1, "key2": -1}
+//   [["key1", "asc"], ["key2", "desc"]]
+//   ["key1", ["key2", "desc"]]
+//
+// (.. with the first form being dependent on the key enumeration
+// behavior of your javascript VM, which usually does what you mean in
+// this case if the key names don't look like integers ..)
+//
+// return a function that takes two objects, and returns -1 if the
+// first object comes first in order, 1 if the second object comes
+// first, or 0 if neither object comes before the other.
+
+Sorter = function (spec) {
+  var self = this;
+
+  var sortSpecParts = [];
+
+  if (spec instanceof Array) {
+    for (var i = 0; i < spec.length; i++) {
+      if (typeof spec[i] === "string") {
+        sortSpecParts.push({
+          lookup: makeLookupFunction(spec[i]),
+          ascending: true
+        });
+      } else {
+        sortSpecParts.push({
+          lookup: makeLookupFunction(spec[i][0]),
+          ascending: spec[i][1] !== "desc"
+        });
+      }
+    }
+  } else if (typeof spec === "object") {
+    for (var key in spec) {
+      sortSpecParts.push({
+        lookup: makeLookupFunction(key),
+        ascending: spec[key] >= 0
+      });
+    }
+  } else {
+    throw Error("Bad sort specification: ", JSON.stringify(spec));
+  }
+
+  // reduceValue takes in all the possible values for the sort key along various
+  // branches, and returns the min or max value (according to the bool
+  // findMin). Each value can itself be an array, and we look at its values
+  // too. (ie, we do a single level of flattening on branchValues, then find the
+  // min/max.)
+  //
+  // XXX This is actually wrong! In fact, the whole attempt to compile sort
+  // functions independently of selectors is wrong. In MongoDB, if you have
+  // documents {_id: 'x', a: [1, 10]} and {_id: 'y', a: [5, 15]},
+  // then C.find({}, {sort: {a: 1}}) puts x before y (1 comes before 5).
+  // But C.find({a: {$gt: 3}}, {sort: {a: 1}}) puts y before x (1 does not match
+  // the selector, and 5 comes before 10).
+  var reduceValue = function (branchValues, findMin) {
+    // Expand any leaf arrays that we find, and ignore those arrays themselves.
+    branchValues = expandArraysInBranches(branchValues, true);
+    var reduced = undefined;
+    var first = true;
+    // Iterate over all the values found in all the branches, and if a value is
+    // an array itself, iterate over the values in the array separately.
+    _.each(branchValues, function (branchValue) {
+      if (first) {
+        reduced = branchValue.value;
+        first = false;
+      } else {
+        // Compare the value we found to the value we found so far, saving it
+        // if it's less (for an ascending sort) or more (for a descending
+        // sort).
+        var cmp = LocalCollection._f._cmp(reduced, branchValue.value);
+        if ((findMin && cmp > 0) || (!findMin && cmp < 0))
+          reduced = branchValue.value;
+      }
+    });
+    return reduced;
+  };
+
+  var comparators = _.map(sortSpecParts, function (specPart) {
+    return function (a, b) {
+      var aValue = reduceValue(specPart.lookup(a), specPart.ascending);
+      var bValue = reduceValue(specPart.lookup(b), specPart.ascending);
+      var compare = LocalCollection._f._cmp(aValue, bValue);
+      return specPart.ascending ? compare : -compare;
+    };
+  });
+
+  self._baseComparator = composeComparators(comparators);
+};
+
+Sorter.prototype.getComparator = function (options) {
+  var self = this;
+
+  // If we have no distances, just use the comparator from the source
+  // specification (which defaults to "everything is equal".
+  if (!options || !options.distances) {
+    return self._baseComparator;
+  }
+
+  var distances = options.distances;
+
+  // Return a comparator which first tries the sort specification, and if that
+  // says "it's equal", breaks ties using $near distances.
+  return composeComparators([self._baseComparator, function (a, b) {
+    if (!distances.has(a._id))
+      throw Error("Missing distance for " + a._id);
+    if (!distances.has(b._id))
+      throw Error("Missing distance for " + b._id);
+    return distances.get(a._id) - distances.get(b._id);
+  }]);
+};
+
+MinimongoTest.Sorter = Sorter;
+
+// Given an array of comparators
+// (functions (a,b)->(negative or positive or zero)), returns a single
+// comparator which uses each comparator in order and returns the first
+// non-zero value.
+var composeComparators = function (comparatorArray) {
+  return function (a, b) {
+    for (var i = 0; i < comparatorArray.length; ++i) {
+      var compare = comparatorArray[i](a, b);
+      if (compare !== 0)
+        return compare;
+    }
+    return 0;
+  };
+};

packages/mongo-livedata/collection.js

@@ -278,8 +278,7 @@ Meteor.Collection._rewriteSelector = function (selector) {
       ret[key] = _.map(value, function (v) {
         return Meteor.Collection._rewriteSelector(v);
       });
-    }
-    else {
+    } else {
       ret[key] = value;
     }
   });

packages/mongo-livedata/mongo_driver.js

@@ -165,7 +165,7 @@ MongoConnection = function (url, options) {
     self._withDb(function (db) {
       dbNameFuture.return(db.databaseName);
     });
-    self._oplogHandle = new OplogHandle(options.oplogUrl, dbNameFuture);
+    self._oplogHandle = new OplogHandle(options.oplogUrl, dbNameFuture.wait());
   }
 };
 
@@ -491,7 +491,7 @@ var simulateUpsertWithInsertedId = function (collection, selector, mod,
     // the behavior of modifiers is concerned, whether `_modify`
     // is run on EJSON or on mongo-converted EJSON.
     var selectorDoc = LocalCollection._removeDollarOperators(selector);
-    LocalCollection._modify(selectorDoc, mod, true);
+    LocalCollection._modify(selectorDoc, mod, {isInsert: true});
     newDoc = selectorDoc;
   } else {
     newDoc = mod;
@@ -957,6 +957,14 @@ MongoConnection.prototype._observeChanges = function (
     return self._observeChangesTailable(cursorDescription, ordered, callbacks);
   }
 
+  // You may not filter out _id when observing changes, because the id is a core
+  // part of the observeChanges API.
+  if (cursorDescription.options.fields &&
+      (cursorDescription.options.fields._id === 0 ||
+       cursorDescription.options.fields._id === false)) {
+    throw Error("You may not observe a cursor with {fields: {_id: 0}}");
+  }
+
   var observeKey = JSON.stringify(
     _.extend({ordered: ordered}, cursorDescription));
 
@@ -987,21 +995,32 @@ MongoConnection.prototype._observeChanges = function (
 
   if (firstHandle) {
     var driverClass = PollingObserveDriver;
-    if (self._oplogHandle && !ordered && !callbacks._testOnlyPollCallback
-        && OplogObserveDriver.cursorSupported(cursorDescription)) {
-      driverClass = OplogObserveDriver;
+    var matcher;
+    if (self._oplogHandle && !ordered && !callbacks._testOnlyPollCallback) {
+      try {
+        matcher = new Minimongo.Matcher(cursorDescription.selector);
+      } catch (e) {
+        // Ignore and avoid oplog driver. eg, maybe we're trying to compile some
+        // newfangled $selector that minimongo doesn't support yet.
+        // XXX make all compilation errors MinimongoError or something
+        //     so that this doesn't ignore unrelated exceptions
+      }
+      if (matcher
+          && OplogObserveDriver.cursorSupported(cursorDescription, matcher)) {
+        driverClass = OplogObserveDriver;
+      }
     }
     observeDriver = new driverClass({
       cursorDescription: cursorDescription,
       mongoHandle: self,
       multiplexer: multiplexer,
       ordered: ordered,
+      matcher: matcher,  // ignored by polling
       _testOnlyPollCallback: callbacks._testOnlyPollCallback
     });
 
-    // This field is only set for the first ObserveHandle in an
-    // ObserveMultiplexer. It is only there for use tests.
-    observeHandle._observeDriver = observeDriver;
+    // This field is only set for use in tests.
+    multiplexer._observeDriver = observeDriver;
   }
 
   // Blocks until the initial adds have been sent.

packages/mongo-livedata/mongo_livedata_tests.js

@@ -391,13 +391,9 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test,
     }
   });
 
-  // XXX What if there are multiple observe handles on the ObserveMultiplexer?
-  //     There shouldn't be because the collection has a name unique to this
-  //     run.
   if (Meteor.isServer) {
-    // For now, has to be polling (not oplog).
-    test.isTrue(obs._observeDriver);
-    test.isTrue(obs._observeDriver._suspendPolling);
+    // For now, has to be polling (not oplog) because it is ordered observe.
+    test.isTrue(obs._multiplexer._observeDriver._suspendPolling);
   }
 
   var step = 0;
@@ -432,7 +428,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test,
 
     finishObserve(function () {
       if (Meteor.isServer)
-        obs._observeDriver._suspendPolling();
+        obs._multiplexer._observeDriver._suspendPolling();
 
       // Do a batch of 1-10 operations
       var batch_count = rnd(10) + 1;
@@ -465,7 +461,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test,
         }
       }
       if (Meteor.isServer)
-        obs._observeDriver._resumePolling();
+        obs._multiplexer._observeDriver._resumePolling();
 
     });
 
@@ -761,18 +757,46 @@ testAsyncMulti('mongo-livedata - empty documents, ' + idGeneration, [
     }
 
     var coll = new Meteor.Collection(collectionName, collectionOptions);
-    var docId;
 
     coll.insert({}, expect(function (err, id) {
       test.isFalse(err);
       test.isTrue(id);
-      docId = id;
       var cursor = coll.find();
       test.equal(cursor.count(), 1);
     }));
   }
 ]);
 
+// See https://github.com/meteor/meteor/issues/594.
+testAsyncMulti('mongo-livedata - document with length, ' + idGeneration, [
+  function (test, expect) {
+    var self = this;
+    var collectionName = Random.id();
+    if (Meteor.isClient) {
+      Meteor.call('createInsecureCollection', collectionName);
+      Meteor.subscribe('c-' + collectionName);
+    }
+
+    self.coll = new Meteor.Collection(collectionName, collectionOptions);
+
+    self.coll.insert({foo: 'x', length: 0}, expect(function (err, id) {
+      test.isFalse(err);
+      test.isTrue(id);
+      self.docId = id;
+      test.equal(self.coll.findOne(self.docId),
+                 {_id: self.docId, foo: 'x', length: 0});
+    }));
+  },
+  function (test, expect) {
+    var self = this;
+    self.coll.update(self.docId, {$set: {length: 5}}, expect(function (err) {
+      test.isFalse(err);
+      test.equal(self.coll.findOne(self.docId),
+                 {_id: self.docId, foo: 'x', length: 5});
+    }));
+  }
+]);
+
 testAsyncMulti('mongo-livedata - document with a date, ' + idGeneration, [
   function (test, expect) {
     var collectionName = Random.id();
@@ -1886,14 +1910,12 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - _disableOplog", functi
   if (MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle) {
     var observeWithOplog = coll.find({x: 5})
           .observeChanges({added: function () {}});
-    test.isTrue(observeWithOplog._observeDriver);
-    test.isTrue(observeWithOplog._observeDriver._usesOplog);
+    test.isTrue(observeWithOplog._multiplexer._observeDriver._usesOplog);
     observeWithOplog.stop();
   }
   var observeWithoutOplog = coll.find({x: 6}, {_disableOplog: true})
         .observeChanges({added: function () {}});
-  test.isTrue(observeWithoutOplog._observeDriver);
-  test.isFalse(observeWithoutOplog._observeDriver._usesOplog);
+  test.isFalse(observeWithoutOplog._multiplexer._observeDriver._usesOplog);
   observeWithoutOplog.stop();
 });
 

packages/mongo-livedata/observe_changes_tests.js

@@ -25,6 +25,7 @@ _.each ([{added:'added', forceOrdered: true},
                        function (logger) {
     var barid = c.insert({thing: "stuff"});
     var fooid = c.insert({noodles: "good", bacon: "bad", apples: "ok"});
+
     var handle = c.find(fooid).observeChanges(logger);
     if (added === 'added')
       logger.expectResult(added, [fooid, {noodles: "good", bacon: "bad",apples: "ok"}]);
@@ -43,6 +44,12 @@ _.each ([{added:'added', forceOrdered: true},
     c.insert({noodles: "good", bacon: "bad", apples: "ok"});
     logger.expectNoResult();
     handle.stop();
+
+    var badCursor = c.find({}, {fields: {noodles: 1, _id: false}});
+    test.throws(function () {
+      badCursor.observeChanges(logger);
+    });
+
     onComplete();
     });
   });

packages/mongo-livedata/oplog_observe_driver.js

@@ -32,14 +32,12 @@ OplogObserveDriver = function (options) {
 
   self._published = new LocalCollection._IdMap;
   var selector = self._cursorDescription.selector;
-  self._selectorFn = LocalCollection._compileSelector(
-    self._cursorDescription.selector);
+  self._matcher = options.matcher;
   var projection = self._cursorDescription.options.fields || {};
   self._projectionFn = LocalCollection._compileProjection(projection);
   // Projection function, result of combining important fields for selector and
   // existing fields projection
-  self._sharedProjection = LocalCollection._combineSelectorAndProjection(
-    selector, projection);
+  self._sharedProjection = self._matcher.combineIntoProjection(projection);
   self._sharedProjectionFn = LocalCollection._compileProjection(
     self._sharedProjection);
 
@@ -131,7 +129,7 @@ _.extend(OplogObserveDriver.prototype, {
     var self = this;
     newDoc = _.clone(newDoc);
 
-    var matchesNow = newDoc && self._selectorFn(newDoc);
+    var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result;
     if (mustMatchNow && !matchesNow) {
       throw Error("expected " + EJSON.stringify(newDoc) + " to match "
                   + EJSON.stringify(self._cursorDescription));
@@ -238,7 +236,7 @@ _.extend(OplogObserveDriver.prototype, {
 
       // XXX what if selector yields?  for now it can't but later it could have
       // $where
-      if (self._selectorFn(op.o))
+      if (self._matcher.documentMatches(op.o).result)
         self._add(op.o);
     } else if (op.op === 'u') {
       // Is this a modifier ($set/$unset, which may require us to poll the
@@ -263,8 +261,7 @@ _.extend(OplogObserveDriver.prototype, {
         LocalCollection._modify(newDoc, op.o);
         self._handleDoc(id, self._sharedProjectionFn(newDoc));
       } else if (!canDirectlyModifyDoc ||
-                 LocalCollection._canSelectorBecomeTrueByModifier(
-                   self._cursorDescription.selector, op.o)) {
+                 self._matcher.canBecomeTrueByModifier(op.o)) {
         self._needToFetch.set(id, op.ts.toString());
         if (self._phase === PHASE.STEADY)
           self._fetchModifiedDocuments();
@@ -462,7 +459,7 @@ _.extend(OplogObserveDriver.prototype, {
 // Does our oplog tailing code support this cursor? For now, we are being very
 // conservative and allowing only simple queries with simple options.
 // (This is a "static method".)
-OplogObserveDriver.cursorSupported = function (cursorDescription) {
+OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) {
   // First, check the options.
   var options = cursorDescription.options;
 
@@ -488,23 +485,13 @@ OplogObserveDriver.cursorSupported = function (cursorDescription) {
     }
   }
 
-  // For now, we're just dealing with equality queries: no $operators, regexps,
-  // or $and/$or/$where/etc clauses. We can expand the scope of what we're
-  // comfortable processing later. ($where will get pretty scary since it will
-  // allow selector processing to yield!)
-  return _.all(cursorDescription.selector, function (value, field) {
-    // No logical operators like $and.
-    if (field.substr(0, 1) === '$')
-      return false;
-    // We only allow scalars, not sub-documents or $operators or RegExp.
-    // XXX Date would be easy too, though I doubt anyone is doing equality
-    // lookups on dates
-    return typeof value === "string" ||
-      typeof value === "number" ||
-      typeof value === "boolean" ||
-      value === null ||
-      value instanceof Meteor.Collection.ObjectID;
-  });
+  // We don't allow the following selectors:
+  //   - $where (not confident that we provide the same JS environment
+  //             as Mongo, and can yield!)
+  //   - $near (has "interesting" properties in MongoDB, like the possibility
+  //            of returning an ID multiple times, though even polling maybe
+  //            have a bug there
+  return !matcher.hasWhere() && !matcher.hasGeoQuery();
 };
 
 var modifierCanBeDirectlyApplied = function (modifier) {

packages/mongo-livedata/oplog_tailing.js

@@ -1,6 +1,7 @@
 var Future = Npm.require('fibers/future');
 
 var OPLOG_COLLECTION = 'oplog.rs';
+var REPLSET_COLLECTION = 'system.replset';
 
 // Like Perl's quotemeta: quotes all regexp metacharacters. See
 //   https://github.com/substack/quotemeta/blob/master/index.js
@@ -27,10 +28,10 @@ idForOp = function (op) {
     throw Error("Unknown op: " + EJSON.stringify(op));
 };
 
-OplogHandle = function (oplogUrl, dbNameFuture) {
+OplogHandle = function (oplogUrl, dbName) {
   var self = this;
   self._oplogUrl = oplogUrl;
-  self._dbNameFuture = dbNameFuture;
+  self._dbName = dbName;
 
   self._oplogLastEntryConnection = null;
   self._oplogTailConnection = null;
@@ -41,26 +42,17 @@ OplogHandle = function (oplogUrl, dbNameFuture) {
     factPackage: "mongo-livedata", factName: "oplog-watchers"
   });
   self._lastProcessedTS = null;
-  // Lazily calculate the basic selector. Don't call _baseOplogSelector() at the
-  // top level of the constructor, because we don't want the constructor to
-  // block. Note that the _.once is per-handle.
-  self._baseOplogSelector = _.once(function () {
-    return {
-      ns: new RegExp('^' + quotemeta(self._dbNameFuture.wait()) + '\\.'),
-      $or: [
-        { op: {$in: ['i', 'u', 'd']} },
-        // If it is not db.collection.drop(), ignore it
-        { op: 'c', 'o.drop': { $exists: true } }]
-    };
-  });
+  self._baseOplogSelector = {
+    ns: new RegExp('^' + quotemeta(self._dbName) + '\\.'),
+    $or: [
+      { op: {$in: ['i', 'u', 'd']} },
+      // If it is not db.collection.drop(), ignore it
+      { op: 'c', 'o.drop': { $exists: true } }]
+  };
   // XXX doc
   self._catchingUpFutures = [];
 
-  // Setting up the connections and tail handler is a blocking operation, so we
-  // do it "later".
-  Meteor.defer(function () {
-    self._startTailing();
-  });
+  self._startTailing();
 };
 
 _.extend(OplogHandle.prototype, {
@@ -117,7 +109,7 @@ _.extend(OplogHandle.prototype, {
     // tailing selector (ie, we need to specify the DB name) or else we might
     // find a TS that won't show up in the actual tail stream.
     var lastEntry = self._oplogLastEntryConnection.findOne(
-      OPLOG_COLLECTION, self._baseOplogSelector(),
+      OPLOG_COLLECTION, self._baseOplogSelector,
       {fields: {ts: 1}, sort: {$natural: -1}});
 
     if (!lastEntry) {
@@ -168,13 +160,19 @@ _.extend(OplogHandle.prototype, {
     self._oplogLastEntryConnection = new MongoConnection(
       self._oplogUrl, {poolSize: 1});
 
-    // Find the last oplog entry. Blocks until the connection is ready.
+    // First, make sure that there actually is a repl set here. If not, oplog
+    // tailing won't ever find anything! (Blocks until the connection is ready.)
+    var replSetInfo = self._oplogLastEntryConnection.findOne(
+      REPLSET_COLLECTION, {});
+    if (!replSetInfo)
+      throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " +
+                  "a Mongo replica set");
+
+    // Find the last oplog entry.
     var lastOplogEntry = self._oplogLastEntryConnection.findOne(
       OPLOG_COLLECTION, {}, {sort: {$natural: -1}});
 
-    var dbName = self._dbNameFuture.wait();
-
-    var oplogSelector = _.clone(self._baseOplogSelector());
+    var oplogSelector = _.clone(self._baseOplogSelector);
     if (lastOplogEntry) {
       // Start after the last entry that currently exists.
       oplogSelector.ts = {$gt: lastOplogEntry.ts};
@@ -189,11 +187,13 @@ _.extend(OplogHandle.prototype, {
 
     self._tailHandle = self._oplogTailConnection.tail(
       cursorDescription, function (doc) {
-        if (!(doc.ns && doc.ns.length > dbName.length + 1 &&
-              doc.ns.substr(0, dbName.length + 1) === (dbName + '.')))
+        if (!(doc.ns && doc.ns.length > self._dbName.length + 1 &&
+              doc.ns.substr(0, self._dbName.length + 1) ===
+              (self._dbName + '.'))) {
           throw new Error("Unexpected ns");
+        }
 
-        var trigger = {collection: doc.ns.substr(dbName.length + 1),
+        var trigger = {collection: doc.ns.substr(self._dbName.length + 1),
                        dropCollection: false,
                        op: doc};
 

packages/mongo-livedata/oplog_tests.js

@@ -3,9 +3,9 @@ var OplogCollection = new Meteor.Collection("oplog-" + Random.id());
 Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) {
   var supported = function (expected, selector) {
     var cursor = OplogCollection.find(selector);
-    test.equal(
-      MongoTest.OplogObserveDriver.cursorSupported(cursor._cursorDescription),
-      expected);
+    var handle = cursor.observeChanges({added: function () {}});
+    test.equal(!!handle._multiplexer._observeDriver._usesOplog, expected);
+    handle.stop();
   };
 
   supported(true, "asdf");
@@ -25,8 +25,17 @@ Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) {
 
   supported(true, {});
 
-  supported(false, {$and: [{foo: "asdf"}, {bar: "baz"}]});
-  supported(false, {foo: {x: 1}});
-  supported(false, {foo: {$gt: 1}});
-  supported(false, {foo: [1, 2, 3]});
+  supported(true, {$and: [{foo: "asdf"}, {bar: "baz"}]});
+  supported(true, {foo: {x: 1}});
+  supported(true, {foo: {$gt: 1}});
+  supported(true, {foo: [1, 2, 3]});
+
+  // No $where.
+  supported(false, {$where: "xxx"});
+  supported(false, {$and: [{foo: "adsf"}, {$where: "xxx"}]});
+  // No geoqueries.
+  supported(false, {x: {$near: [1,1]}});
+  // Nothing Minimongo doesn't understand.  (Minimongo happens to fail to
+  // implement $elemMatch inside $all which MongoDB supports.)
+  supported(false, {x: {$all: [{$elemMatch: {y: 2}}]}});
 });

packages/oauth/oauth_server.js

@@ -162,7 +162,10 @@ var ensureConfigured = function(serviceName) {
 Oauth._renderOauthResults = function(res, query) {
   // We support ?close and ?redirect=URL. Any other query should
   // just serve a blank page
-  if ('close' in query) { // check with 'in' because we don't set a value
+  if (query.error) {
+    Log.warn("Error in Oauth Server: " + query.error);
+    closePopup(res);
+  } else if ('close' in query) { // check with 'in' because we don't set a value
     closePopup(res);
   } else if (query.redirect) {
     // Only redirect to URLs on the same domain as this app.

packages/oauth/package.js

@@ -6,7 +6,7 @@ Package.describe({
 Package.on_use(function (api) {
   api.use('routepolicy', 'server');
   api.use('webapp', 'server');
-  api.use(['underscore', 'service-configuration'], 'server');
+  api.use(['underscore', 'service-configuration', 'logging'], 'server');
 
   api.export('Oauth');
   api.export('OauthTest', 'server', {testOnly: true});

packages/retry/.gitignore

@@ -0,0 +1 @@
+.build*

packages/retry/package.js

@@ -0,0 +1,10 @@
+Package.describe({
+  summary: "Retry logic with exponential backoff",
+  internal: true
+});
+
+Package.on_use(function (api) {
+  api.use(['underscore', 'random'], ['client', 'server']);
+  api.export('Retry');
+  api.add_files('retry.js', ['client', 'server']);
+});

packages/retry/retry.js

@@ -1,24 +1,23 @@
 // Retry logic with an exponential backoff.
+//
+// options:
+//  baseTimeout: time for initial reconnect attempt (ms).
+//  exponent: exponential factor to increase timeout each attempt.
+//  maxTimeout: maximum time between retries (ms).
+//  minCount: how many times to reconnect "instantly".
+//  minTimeout: time to wait for the first `minCount` retries (ms).
+//  fuzz: factor to randomize retry times by (to avoid retry storms).
 
 Retry = function (options) {
   var self = this;
   _.extend(self, _.defaults(_.clone(options || {}), {
-    // time for initial reconnect attempt.
-    baseTimeout: 1000,
-    // exponential factor to increase timeout each attempt.
+    baseTimeout: 1000, // 1 second
     exponent: 2.2,
-    // maximum time between reconnects. keep this intentionally
-    // high-ish to ensure a server can recover from a failure caused
-    // by load
+    // The default is high-ish to ensure a server can recover from a
+    // failure caused by load.
     maxTimeout: 5 * 60000, // 5 minutes
-    // time to wait for the first 2 retries.  this helps page reload
-    // speed during dev mode restarts, but doesn't hurt prod too
-    // much (due to CONNECT_TIMEOUT)
     minTimeout: 10,
-    // how many times to try to reconnect 'instantly'
     minCount: 2,
-    // fuzz factor to randomize reconnect times by. avoid reconnect
-    // storms.
     fuzz: 0.5 // +- 25%
   }));
   self.retryTimer = null;

packages/spark/spark.js

@@ -1055,8 +1055,7 @@ Spark.list = function (cursor, itemFunc, elseFunc) {
         var frag = itemDict.get(id).liveRange.extract();
         if (before === null) {
           itemDict.lastValue().liveRange.insertAfter(frag);
-        }
-        else {
+        } else {
           itemDict.get(before).liveRange.insertBefore(frag);
         }
         itemDict.moveBefore(id, before);

packages/test-helpers/connection.js

@@ -17,8 +17,7 @@ makeTestConnection = function (test, succeeded, failed) {
     if (serverConns[serverConn.id]) {
       test.fail("onConnection callback called multiple times for same session id");
       failed();
-    }
-    else {
+    } else {
       serverConns[serverConn.id] = serverConn;
     }
   });
@@ -30,8 +29,7 @@ makeTestConnection = function (test, succeeded, failed) {
     if (! serverConn) {
       test.fail("No onConnection received server side for connected client");
       failed();
-    }
-    else {
+    } else {
       onConnectionHandle.stop();
       succeeded(clientConn, serverConn);
     }

packages/underscore/package.js

@@ -20,5 +20,10 @@ Package.on_use(function (api) {
 
   api.export('_');
 
+  // NOTE: we patch _.each and various other functions that polymorphically take
+  // objects, arrays, and array-like objects (such as the querySelectorAll
+  // return value, document.images, and 'arguments') such that objects with a
+  // numeric length field whose constructor === Object are still treated as
+  // objects, not as arrays.  Search for looksLikeArray.
   api.add_files(['pre.js', 'underscore.js', 'post.js']);
 });

packages/underscore/underscore.js

@@ -70,6 +70,16 @@
   // Collection Functions
   // --------------------
 
+  // METEOR CHANGE: _.each({length: 5}) should be treated like an object, not an
+  // array. This looksLikeArray function is introduced by Meteor, and replaces
+  // all instances of `obj.length === +obj.length`.
+  // https://github.com/meteor/meteor/issues/594
+  // https://github.com/jashkenas/underscore/issues/770
+  var looksLikeArray = function (obj) {
+    return (obj.length === +obj.length
+            && (_.isArguments(obj) || obj.constructor !== Object));
+  };
+
   // The cornerstone, an `each` implementation, aka `forEach`.
   // Handles objects with the built-in `forEach`, arrays, and raw objects.
   // Delegates to **ECMAScript 5**'s native `forEach` if available.
@@ -77,7 +87,7 @@
     if (obj == null) return;
     if (nativeForEach && obj.forEach === nativeForEach) {
       obj.forEach(iterator, context);
-    } else if (obj.length === +obj.length) {
+    } else if (looksLikeArray(obj)) {
       for (var i = 0, length = obj.length; i < length; i++) {
         if (iterator.call(context, obj[i], i, obj) === breaker) return;
       }
@@ -134,7 +144,7 @@
       return initial ? obj.reduceRight(iterator, memo) : obj.reduceRight(iterator);
     }
     var length = obj.length;
-    if (length !== +length) {
+    if (!looksLikeArray(obj)) {
       var keys = _.keys(obj);
       length = keys.length;
     }
@@ -381,14 +391,14 @@
   _.toArray = function(obj) {
     if (!obj) return [];
     if (_.isArray(obj)) return slice.call(obj);
-    if (obj.length === +obj.length) return _.map(obj, _.identity);
+    if (looksLikeArray(obj)) return _.map(obj, _.identity);
     return _.values(obj);
   };
 
   // Return the number of elements in an object.
   _.size = function(obj) {
     if (obj == null) return 0;
-    return (obj.length === +obj.length) ? obj.length : _.keys(obj).length;
+    return (looksLikeArray(obj)) ? obj.length : _.keys(obj).length;
   };
 
   // Array Functions

packages/webapp/webapp_server.js

@@ -580,8 +580,9 @@ var runWebAppServer = function () {
           // on a per-job basis.  Discuss w/ teammates.
           proxyBinding = AppConfig.configureService(
             "proxy",
+            "pre0",
             function (proxyService) {
-              if (proxyService.providers.proxy) {
+              if (proxyService && ! _.isEmpty(proxyService)) {
                 var proxyConf;
                 // XXX Figure out a per-job way to specify bind location
                 // (besides hardcoding the location for ADMIN_APP jobs).
@@ -602,9 +603,9 @@ var runWebAppServer = function () {
                   proxyConf = configuration.proxy;
                 }
                 Log("Attempting to bind to proxy at " +
-                    proxyService.providers.proxy);
+                    proxyService);
                 WebAppInternals.bindToProxy(_.extend({
-                  proxyEndpoint: proxyService.providers.proxy
+                  proxyEndpoint: proxyService
                 }, proxyConf));
               }
             }

scripts/admin/banner.txt

@@ -1,5 +1,4 @@
-=> Meteor 0.7.0: use MongoDB's operations log to observe equality
-   queries instead of polling the database server on each update.
+=> Meteor 0.7.0.1: Fix failure to initialize local MongoDB server.
 
    This release is being downloaded in the background. Update your
-   project to Meteor 0.7.0 by running 'meteor update'.
+   project to Meteor 0.7.0.1 by running 'meteor update'.

scripts/admin/install-engine.sh

@@ -1,5 +1,19 @@
 #!/bin/sh
 
+# This is the Meteor install script!
+# Are you looking at this in your web browser, and would like to install Meteor?
+# Just open up your terminal and type:
+#
+#    curl https://install.meteor.com/ | sh
+#
+# Meteor currently supports:
+#   - Mac: OS X 10.6 and above
+#   - Linux: x86 and x86_64 systems
+
+
+
+# Now, on to the actual installer!
+
 ## NOTE sh NOT bash. This script should be POSIX sh only, since we don't
 ## know what shell the user has. Debian uses 'dash' for 'sh', for
 ## example.

scripts/admin/notices.json

@@ -75,6 +75,9 @@
   {
     "release": "0.7.0"
   },
+  {
+    "release": "0.7.0.1"
+  },
   {
     "release": "NEXT"
   }

tools/auth.js

@@ -196,15 +196,28 @@ var tryRevokeOldTokens = function (options) {
     }
     var response = result.response;
 
-    if (response.statusCode === 200) {
-      // Server confirms that the tokens have been revoked
-      // (Be careful to reread session data in case httpHelpers changed it)
-      data = readSessionData();
-      var session = getSession(data, domain);
-      session.pendingRevoke = _.difference(session.pendingRevoke, tokenIds);
-      if (! session.pendingRevoke.length)
-        delete session.pendingRevoke;
-      writeSessionData(data);
+    if (response.statusCode === 200 &&
+        response.body) {
+      try {
+        var body = JSON.parse(response.body);
+        if (body.tokenRevoked) {
+          // Server confirms that the tokens have been revoked. Checking for a
+          // `tokenRevoked` key in the response confirms that we hit an actual
+          // accounts server that understands that we were trying to revoke some
+          // tokens, not just a random URL that happened to return a 200
+          // response.
+
+          // (Be careful to reread session data in case httpHelpers changed it)
+          data = readSessionData();
+          var session = getSession(data, domain);
+          session.pendingRevoke = _.difference(session.pendingRevoke, tokenIds);
+          if (! session.pendingRevoke.length)
+            delete session.pendingRevoke;
+          writeSessionData(data);
+        }
+      } catch (e) {
+        logoutFailWarning(domain);
+      }
     } else {
       logoutFailWarning(domain);
     }
@@ -309,12 +322,14 @@ var logInToGalaxy = function (galaxyName) {
       method: 'GET',
       strictSSL: true,
       headers: {
-        cookie: 'GALAXY_OAUTH_SESSION=' + session
+        cookie: 'GALAXY_OAUTH_SESSION=' + session +
+          '; GALAXY_USER_AGENT_TOOL=' +
+          encodeURIComponent(JSON.stringify(utils.getAgentInfo()))
       }
     });
     var body = JSON.parse(galaxyResult.body);
   } catch (e) {
-    return { error: 'no-galaxy' };
+    return { error: (body && body.error) || 'no-galaxy' };
   }
   response = galaxyResult.response;
 
@@ -324,7 +339,7 @@ var logInToGalaxy = function (galaxyName) {
   if (response.statusCode !== 200 ||
       ! body ||
       ! _.has(galaxyResult.setCookie, 'GALAXY_AUTH'))
-    return { error: 'access-denied' };
+    return { error: (body && body.error) || 'access-denied' };
 
   return {
     token: galaxyResult.setCookie.GALAXY_AUTH,
@@ -418,9 +433,20 @@ exports.loginCommand = function (options) {
     var galaxyLoginResult = logInToGalaxy(galaxy);
     if (galaxyLoginResult.error) {
       // XXX add human readable error messages
-      process.stdout.write('\nLogin to ' + galaxy + ' failed: ' +
-                           galaxyLoginResult.error + '\n');
-      return 1
+      process.stdout.write('\nLogin to ' + galaxy + ' failed. ');
+
+      if (galaxyLoginResult.error === 'unauthorized') {
+        process.stdout.write('You are not authorized for this galaxy.\n');
+      } else if (galaxyLoginResult.error === 'no_oauth_server') {
+        process.stdout.write('The galaxy could not ' +
+                             'contact Meteor Accounts.\n');
+      } else if (galaxyLoginResult.error === 'no_identity') {
+        process.stdout.write('Your login information could not be found.\n');
+      } else {
+        process.stdout.write('Error: ' + galaxyLoginResult.error + '\n');
+      }
+
+      return 1;
     }
     data = readSessionData(); // be careful to reread data file after RPC
     var session = getSession(data, galaxy);
@@ -437,6 +463,7 @@ exports.loginCommand = function (options) {
                        (currentUsername(data) ?
                         " as " + currentUsername(data) : "") + ".\n" +
                        "Thanks for being a Meteor developer!\n");
+  return 0;
 };
 
 exports.logoutCommand = function (options) {

tools/bundler.js

@@ -1474,7 +1474,8 @@ var writeSiteArchive = function (targets, outputPath, options) {
       builder.write('README', { data: new Buffer(
 "This is a Meteor application bundle. It has only one dependency:\n" +
 "Node.js 0.10 (with the 'fibers' package). The current release of Meteor\n" +
-"has been tested with Node 0.10.21. To run the application:\n" +
+"has been tested with Node 0.10.22 and works best with 0.10.22 through\n" +
+"0.10.24. To run the application:\n" +
 "\n" +
 "  $ rm -r programs/server/node_modules/fibers\n" +
 "  $ npm install fibers@1.0.1\n" +

tools/tests/test-bundler-npm.js

@@ -57,7 +57,7 @@ var _assertCorrectPackageNpmDir = function (deps) {
     // known to the test author. We set keys on val always in this
     // order so that comparison works well.
     var val = {};
-    _.each(['version', 'from', 'resolved', 'dependencies'], function (key) {
+    _.each(['version', 'dependencies'], function (key) {
       if (expected[key])
         val[key] = expected[key];
       else if (actualMeteorNpmShrinkwrapDependencies[name] && actualMeteorNpmShrinkwrapDependencies[name][key])