Merge branch 'devel' into oplog-limits-buffered

History.md

@@ -1,5 +1,21 @@
 ## v.NEXT
 
+* Use "faye-websocket" (0.7.2) npm module instead of "websocket" (1.0.8) for
+  server-to-server DDP.
+
+* minimongo: Support {a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}}  #1875
+
+* minimongo: Support {a: {$regex: '', $options: 'i'}}  #1874
+
+## v0.7.1.2
+
+* Fix bug in tool error handling that caused `meteor` to crash on Mac
+  OSX when no computer name is set.
+
+* Work around a bug that caused MongoDB to fail an assertion when using
+  tailable cursors on non-oplog collections.
+
+
 ## v0.7.1.1
 
 * Integrate with Meteor developer accounts, a new way of managing your
@@ -29,7 +45,7 @@
 * Add and improve support for minimongo operators.
   - Support `$comment`.
   - Support `obj` name in `$where`.
-  - `$regexp` matches actual regexps properly.
+  - `$regex` matches actual regexps properly.
   - Improve support for `$nin`, `$ne`, `$not`.
   - Support using `{ $in: [/foo/, /bar/] }`. #1707
   - Support `{$exists: false}`.

docs/.meteor/release

@@ -1 +1 @@
-0.7.1.1
+0.7.1.2

docs/lib/release-override.js

@@ -1,5 +1,5 @@
 // While galaxy apps are on their own special meteor releases, override
 // Meteor.release here.
 if (Meteor.isClient) {
-  Meteor.release = Meteor.release ? "0.7.1.1" : undefined;
+  Meteor.release = Meteor.release ? "0.7.1.2" : undefined;
 }

examples/leaderboard/.meteor/release

@@ -1 +1 @@
-0.7.1.1
+0.7.1.2

examples/parties/.meteor/release

@@ -1 +1 @@
-0.7.1.1
+0.7.1.2

examples/todos/.meteor/release

@@ -1 +1 @@
-0.7.1.1
+0.7.1.2

examples/wordplay/.meteor/release

@@ -1 +1 @@
-0.7.1.1
+0.7.1.2

packages/livedata/.npm/package/npm-shrinkwrap.json

@@ -16,8 +16,13 @@
         }
       }
     },
-    "websocket": {
-      "version": "1.0.8"
+    "faye-websocket": {
+      "version": "0.7.2",
+      "dependencies": {
+        "websocket-driver": {
+          "version": "0.3.2"
+        }
+      }
     }
   }
 }

packages/livedata/livedata_connection.js

@@ -49,7 +49,10 @@ var Connection = function (url, options) {
     self._stream = new LivedataTest.ClientStream(url, {
       retry: options.retry,
       headers: options.headers,
-      _sockjsOptions: options._sockjsOptions
+      _sockjsOptions: options._sockjsOptions,
+      // To keep some tests quiet (because we don't have a real API for handling
+      // client-stream-level errors).
+      _dontPrintErrors: options._dontPrintErrors
     });
   }
 

packages/livedata/livedata_server.js

@@ -1160,7 +1160,7 @@ _.extend(Server.prototype, {
       // drop all future data coming over this connection on the
       // floor. We don't want to confuse things.
       socket.removeAllListeners('data');
-      setTimeout(function () {
+      Meteor.setTimeout(function () {
         socket.send(stringifyDDP({msg: 'failed', version: version}));
         socket.close();
       }, timeout);

packages/livedata/livedata_tests.js

@@ -683,9 +683,10 @@ if (Meteor.isServer) {
   testAsyncMulti("livedata - connect fails to unknown place", [
     function (test, expect) {
       var self = this;
-      self.conn = DDP.connect("example.com");
+      self.conn = DDP.connect("example.com", {_dontPrintErrors: true});
       Meteor.setTimeout(expect(function () {
         test.isFalse(self.conn.status().connected, "Not connected");
+        self.conn.close();
       }), 500);
     }
   ]);

packages/livedata/package.js

@@ -3,7 +3,11 @@ Package.describe({
   internal: true
 });
 
-Npm.depends({sockjs: "0.3.8", websocket: "1.0.8"});
+// We use 'faye-websocket' for connections in server-to-server DDP, mostly
+// because it's the same library used as a server in sockjs, and it's easiest to
+// deal with a single websocket implementation.  (Plus, its maintainer is easy
+// to work with on pull requests.)
+Npm.depends({sockjs: "0.3.8", "faye-websocket": "0.7.2"});
 
 Package.on_use(function (api) {
   api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps',

packages/livedata/stream_client_nodejs.js

@@ -11,43 +11,16 @@
 // ping frames or with DDP-level messages.)
 LivedataTest.ClientStream = function (endpoint, options) {
   var self = this;
+  options = options || {};
+
   self.options = _.extend({
     retry: true
   }, options);
 
-  // WebSocket-Node https://github.com/Worlize/WebSocket-Node
-  // Chosen because it can run without native components. It has a
-  // somewhat idiosyncratic API. We may want to use 'ws' instead in the
-  // future.
-  //
-  // Since server-to-server DDP is still an experimental feature, we only
-  // require the module if we actually create a server-to-server
-  // connection. This is a minor efficiency improvement, but moreover: while
-  // 'websocket' doesn't require native components, it tries to use some
-  // optional native components and prints a warning if it can't load
-  // them. Since native components in packages don't work when transferred to
-  // other architectures yet, this means that require('websocket') prints a
-  // spammy log message when deployed to another architecture. Delaying the
-  // require means you only get the log message if you're actually using the
-  // feature.
-  self.client = new (Npm.require('websocket').client)();
+  self.client = null;  // created in _launchConnection
   self.endpoint = endpoint;
-  self.currentConnection = null;
 
-  options = options || {};
-  self.headers = options.headers || {};
-
-  self.client.on('connect', Meteor.bindEnvironment(
-    function (connection) {
-      return self._onConnect(connection);
-    },
-    "stream connect callback"
-  ));
-
-  self.client.on('connectFailed', function (error) {
-    // XXX: Make this do something better than make the tests hang if it does not work.
-    return self._lostConnection();
-  });
+  self.headers = self.options.headers || {};
 
   self._initCommon();
 
@@ -63,7 +36,7 @@ _.extend(LivedataTest.ClientStream.prototype, {
   send: function (data) {
     var self = this;
     if (self.currentStatus.connected) {
-      self.currentConnection.send(data);
+      self.client.send(data);
     }
   },
 
@@ -73,80 +46,37 @@ _.extend(LivedataTest.ClientStream.prototype, {
     self.endpoint = url;
   },
 
-  _onConnect: function (connection) {
+  _onConnect: function (client) {
     var self = this;
 
+    if (client !== self.client) {
+      // This connection is not from the last call to _launchConnection.
+      // But _launchConnection calls _cleanup which closes previous connections.
+      // It's our belief that this stifles future 'open' events, but maybe
+      // we are wrong?
+      throw new Error("Got open from inactive client");
+    }
+
     if (self._forcedToDisconnect) {
       // We were asked to disconnect between trying to open the connection and
       // actually opening it. Let's just pretend this never happened.
-      connection.close();
+      self.client.close();
+      self.client = null;
       return;
     }
 
     if (self.currentStatus.connected) {
-      // We already have a connection. It must have been the case that
-      // we started two parallel connection attempts (because we
-      // wanted to 'reconnect now' on a hanging connection and we had
-      // no way to cancel the connection attempt.) Just ignore/close
-      // the latecomer.
-      connection.close();
-      return;
+      // We already have a connection. It must have been the case that we
+      // started two parallel connection attempts (because we wanted to
+      // 'reconnect now' on a hanging connection and we had no way to cancel the
+      // connection attempt.) But this shouldn't happen (similarly to the client
+      // !== self.client check above).
+      throw new Error("Two parallel connections?");
     }
 
-    if (self.connectionTimer) {
-      clearTimeout(self.connectionTimer);
-      self.connectionTimer = null;
-    }
-
-    var onError = Meteor.bindEnvironment(
-      function (_this, error) {
-        if (self.currentConnection !== _this)
-          return;
-
-        Meteor._debug("stream error", error.toString(),
-                      (new Date()).toDateString());
-        self._lostConnection();
-      },
-      "stream error callback"
-    );
-
-    connection.on('error', function (error) {
-      // We have to pass in `this` explicitly because bindEnvironment
-      // doesn't propagate it for us.
-      onError(this, error);
-    });
-
-    var onClose = Meteor.bindEnvironment(
-      function (_this) {
-        if (self.options._testOnClose)
-          self.options._testOnClose();
-
-        if (self.currentConnection !== _this)
-          return;
-
-        self._lostConnection();
-      },
-      "stream close callback"
-    );
-
-    connection.on('close', function () {
-      // We have to pass in `this` explicitly because bindEnvironment
-      // doesn't propagate it for us.
-      onClose(this);
-    });
-
-    connection.on('message', function (message) {
-      if (self.currentConnection !== this)
-        return; // old connection still emitting messages
-
-      if (message.type === "utf8") // ignore binary frames
-        _.each(self.eventCallbacks.message, function (callback) {
-          callback(message.utf8Data);
-        });
-    });
+    self._clearConnectionTimer();
 
     // update status
-    self.currentConnection = connection;
     self.currentStatus.status = "connected";
     self.currentStatus.connected = true;
     self.currentStatus.retryCount = 0;
@@ -161,10 +91,10 @@ _.extend(LivedataTest.ClientStream.prototype, {
     var self = this;
 
     self._clearConnectionTimer();
-    if (self.currentConnection) {
-      var conn = self.currentConnection;
-      self.currentConnection = null;
-      conn.close();
+    if (self.client) {
+      var client = self.client;
+      self.client = null;
+      client.close();
     }
   },
 
@@ -181,25 +111,63 @@ _.extend(LivedataTest.ClientStream.prototype, {
     var self = this;
     self._cleanup(); // cleanup the old socket, if there was one.
 
-    // launch a connect attempt. we have no way to track it. we either
-    // get an _onConnect event, or we don't.
+    // Since server-to-server DDP is still an experimental feature, we only
+    // require the module if we actually create a server-to-server
+    // connection.
+    var FayeWebSocket = Npm.require('faye-websocket');
 
-    // XXX: set up a timeout on this.
+    // We would like to specify 'ddp' as the subprotocol here. The npm module we
+    // used to use as a client would fail the handshake if we ask for a
+    // subprotocol and the server doesn't send one back (and sockjs doesn't).
+    // Faye doesn't have that behavior; it's unclear from reading RFC 6455 if
+    // Faye is erroneous or not.  So for now, we don't specify protocols.
+    var client = self.client = new FayeWebSocket.Client(
+      toWebsocketUrl(self.endpoint),
+      [/*no subprotocols*/],
+      {headers: self.headers}
+    );
 
-    // we would like to specify 'ddp' as the protocol here, but
-    // unfortunately WebSocket-Node fails the handshake if we ask for
-    // a protocol and the server doesn't send one back (and sockjs
-    // doesn't). also, related: I guess we have to accept that
-    // 'stream' is ddp-specific
-    self.client.connect(toWebsocketUrl(self.endpoint),
-                        undefined, // protocols
-                        undefined, // origin
-                        self.headers);
-
-    if (self.connectionTimer)
-      clearTimeout(self.connectionTimer);
-    self.connectionTimer = setTimeout(
+    self._clearConnectionTimer();
+    self.connectionTimer = Meteor.setTimeout(
       _.bind(self._lostConnection, self),
       self.CONNECT_TIMEOUT);
+
+    self.client.on('open', Meteor.bindEnvironment(function () {
+      return self._onConnect(client);
+    }, "stream connect callback"));
+
+    var clientOnIfCurrent = function (event, description, f) {
+      self.client.on(event, Meteor.bindEnvironment(function () {
+        // Ignore events from any connection we've already cleaned up.
+        if (client !== self.client)
+          return;
+        f.apply(this, arguments);
+      }, description));
+    };
+
+    clientOnIfCurrent('error', 'stream error callback', function (error) {
+      if (!self.options._dontPrintErrors)
+        Meteor._debug("stream error", error.message);
+
+      // XXX: Make this do something better than make the tests hang if it does
+      // not work.
+      self._lostConnection();
+    });
+
+
+    clientOnIfCurrent('close', 'stream close callback', function () {
+      self._lostConnection();
+    });
+
+
+    clientOnIfCurrent('message', 'stream message callback', function (message) {
+      // Ignore binary frames, where message.data is a Buffer
+      if (typeof message.data !== "string")
+        return;
+
+      _.each(self.eventCallbacks.message, function (callback) {
+        callback(message.data);
+      });
+    });
   }
 });

packages/livedata/stream_client_tests.js

@@ -1,17 +1,24 @@
 var Fiber = Npm.require('fibers');
 
-Tinytest.addAsync("stream client - callbacks run in a fiber", function (test, onComplete) {
-  stream = new LivedataTest.ClientStream(
-    Meteor.absoluteUrl(),
-    {
-      _testOnClose: function () {
-        test.isTrue(Fiber.current);
-        onComplete();
-      }
-    }
-  );
-  stream.on('reset', function () {
-    test.isTrue(Fiber.current);
-    stream.disconnect();
-  });
-});
+testAsyncMulti("stream client - callbacks run in a fiber", [
+  function (test, expect) {
+    var stream = new LivedataTest.ClientStream(Meteor.absoluteUrl());
+
+    var messageFired = false;
+    var resetFired = false;
+
+    stream.on('message', expect(function () {
+      test.isTrue(Fiber.current);
+      if (resetFired)
+        stream.disconnect();
+      messageFired = true;
+    }));
+
+    stream.on('reset', expect(function () {
+      test.isTrue(Fiber.current);
+      if (messageFired)
+        stream.disconnect();
+      resetFired = true;
+    }));
+  }
+]);

packages/minimongo/helpers.js

@@ -16,7 +16,10 @@ isIndexable = function (x) {
   return isArray(x) || isPlainObject(x);
 };
 
-isOperatorObject = function (valueSelector) {
+// Returns true if this is an object with at least one key and all keys begin
+// with $.  Unless inconsistentOK is set, throws if some keys begin with $ and
+// others don't.
+isOperatorObject = function (valueSelector, inconsistentOK) {
   if (!isPlainObject(valueSelector))
     return false;
 
@@ -26,7 +29,10 @@ isOperatorObject = function (valueSelector) {
     if (theseAreOperators === undefined) {
       theseAreOperators = thisIsOperator;
     } else if (theseAreOperators !== thisIsOperator) {
-      throw new Error("Inconsistent operator: " + valueSelector);
+      if (!inconsistentOK)
+        throw new Error("Inconsistent operator: " +
+                        JSON.stringify(valueSelector));
+      theseAreOperators = false;
     }
   });
   return !!theseAreOperators;  // {} has no operators

packages/minimongo/minimongo_tests.js

@@ -678,6 +678,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$regex: 'a'}}, {a: 'cut'});
   nomatch({a: {$regex: 'a'}}, {a: 'CAT'});
   match({a: {$regex: 'a', $options: 'i'}}, {a: 'CAT'});
+  match({a: {$regex: '', $options: 'i'}}, {a: 'foo'});
+  nomatch({a: {$regex: '', $options: 'i'}}, {});
+  nomatch({a: {$regex: '', $options: 'i'}}, {a: 5});
   nomatch({a: /undefined/}, {});
   nomatch({a: {$regex: 'undefined'}}, {});
   nomatch({a: /xxx/}, {});
@@ -817,6 +820,12 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({$or: [{a: 2}, {a: 3}], b: 2}, {a: 1, b: 2});
   nomatch({$or: [{a: 1}, {a: 2}], b: 3}, {a: 1, b: 2});
 
+  // Combining $or with equality
+  match({x: 1, $or: [{a: 1}, {b: 1}]}, {x: 1, b: 1});
+  match({$or: [{a: 1}, {b: 1}], x: 1}, {x: 1, b: 1});
+  nomatch({x: 1, $or: [{a: 1}, {b: 1}]}, {b: 1});
+  nomatch({x: 1, $or: [{a: 1}, {b: 1}]}, {x: 1});
+
   // $or and $lt, $lte, $gt, $gte
   match({$or: [{a: {$lte: 1}}, {a: 2}]}, {a: 1});
   nomatch({$or: [{a: {$lt: 1}}, {a: 2}]}, {a: 1});
@@ -1100,6 +1109,16 @@ Tinytest.add("minimongo - selector_compiler", function (test) {
   nomatch({a: {$elemMatch: {x: 5}}}, {a: {x: 5}});
   match({a: {$elemMatch: {0: {$gt: 5, $lt: 9}}}}, {a: [[6]]});
   match({a: {$elemMatch: {'0.b': {$gt: 5, $lt: 9}}}}, {a: [[{b:6}]]});
+  match({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+        {a: [{x: 1, b: 1}]});
+  match({a: {$elemMatch: {$or: [{a: 1}, {b: 1}], x: 1}}},
+        {a: [{x: 1, b: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{b: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{x: 1}]});
+  nomatch({a: {$elemMatch: {x: 1, $or: [{a: 1}, {b: 1}]}}},
+          {a: [{x: 1}, {b: 1}]});
 
   // $comment
   match({a: 5, $comment: "asdf"}, {a: 5});

packages/minimongo/selector.js

@@ -384,7 +384,7 @@ var VALUE_OPERATORS = {
   },
   // $options just provides options for $regex; its logic is inside $regex
   $options: function (operand, valueSelector) {
-    if (!valueSelector.$regex)
+    if (!_.has(valueSelector, '$regex'))
       throw Error("$options needs a $regex");
     return everythingMatcher;
   },
@@ -653,7 +653,7 @@ var ELEMENT_OPERATORS = {
         throw Error("$elemMatch need an object");
 
       var subMatcher, isDocMatcher;
-      if (isOperatorObject(operand)) {
+      if (isOperatorObject(operand, true)) {
         subMatcher = compileValueSelector(operand, matcher);
         isDocMatcher = false;
       } else {

packages/retry/retry.js

@@ -57,7 +57,7 @@ _.extend(Retry.prototype, {
     var timeout = self._timeout(count);
     if (self.retryTimer)
       clearTimeout(self.retryTimer);
-    self.retryTimer = setTimeout(fn, timeout);
+    self.retryTimer = Meteor.setTimeout(fn, timeout);
     return timeout;
   }
 

scripts/admin/banner.txt

@@ -1,7 +1,4 @@
-=> Meteor 0.7.1.1: Extend oplog tailing driver to support most common
-   MongoDB queries.  Introduce Meteor developer accounts, a new way of
-   managing your meteor.com deployed sites. When you use `meteor
-   deploy`, you will be prompted to create a developer account.
+=> Meteor 0.7.1.2: Fix crash on OSX machines with no hostname set.
 
    This release is being downloaded in the background. Update your
-   project to Meteor 0.7.1.1 by running 'meteor update'.
+   project to Meteor 0.7.1.2 by running 'meteor update'.

scripts/admin/notices.json

@@ -88,6 +88,9 @@
                  "http://jquery.com/upgrade-guide/1.9/"]
     }
   },
+  {
+    "release": "0.7.1.2"
+  },
   {
     "release": "NEXT"
   }

tools/deploy-galaxy.js

@@ -78,7 +78,7 @@ var ServiceConnection = function (galaxy, service) {
   // from the hostname of endpointUrl, and run the login command for
   // that galaxy.
   if (! authToken)
-    throw new Error("not logged in to galaxy?")
+    throw new Error("not logged in to galaxy?");
 
   self.connection = Package.livedata.DDP.connect(endpointUrl, {
     headers: {
@@ -117,10 +117,12 @@ _.extend(ServiceConnection.prototype, {
     var args = _.toArray(arguments);
     var name = args.shift();
     self.connection.apply(name, args, function (err, result) {
-      if (err)
+      if (err) {
         fut['throw'](err);
-      else
+      } else {
+        self._cleanUpTimer();
         fut['return'](result);
+      }
     });
 
     return fut.wait();
@@ -141,6 +143,7 @@ _.extend(ServiceConnection.prototype, {
     args.push({
       onReady: function () {
         ready = true;
+        self._cleanUpTimer();
         fut['return']();
       },
       onError: function (e) {
@@ -151,8 +154,16 @@ _.extend(ServiceConnection.prototype, {
       }
     });
 
-    self.connection.subscribe.apply(self.connection, args);
-    return fut.wait();
+    var sub = self.connection.subscribe.apply(self.connection, args);
+    fut.wait();
+    return sub;
+  },
+
+  _cleanUpTimer: function () {
+    var self = this;
+    var Package = getPackage();
+    Package.meteor.Meteor.clearTimeout(self.connectionTimer);
+    self.connectionTimer = null;
   },
 
   close: function () {
@@ -163,9 +174,7 @@ _.extend(ServiceConnection.prototype, {
     }
     if (self.connectionTimer) {
       // Clean up the timer so that Node can exit cleanly
-      var Package = getPackage();
-      Package.meteor.Meteor.clearTimeout(self.connectionTimer);
-      self.connectionTimer = null;
+      self._cleanUpTimer();
     }
   }
 });
@@ -456,17 +465,6 @@ exports.logs = function (options) {
       throw new Error("Can't listen to messages on the logs collection");
 
     var logsSubscription = null;
-    // In case of reconnect recover the state so user sees only new logs
-    logReader.connection.onReconnect = function () {
-      logsSubscription && logsSubscription.stop();
-      var opts = { streaming: options.streaming };
-      if (lastLogId)
-        opts.resumeAfterId = lastLogId;
-      // XXX correctly handle errors on resubscribe
-      logsSubscription = logReader.subscribeAndWait("logsForApp",
-                                                    options.app, opts);
-    };
-
     try {
       logsSubscription =
         logReader.subscribeAndWait("logsForApp", options.app,
@@ -477,6 +475,29 @@ exports.logs = function (options) {
       });
     }
 
+    // In case of reconnect recover the state so user sees only new logs.
+    // Only set up the onReconnect handler after the subscribe and wait
+    // has returned; if we set it up before, then we'll end up with two
+    // subscriptions, because the onReconnect handler will run for the
+    // first time before the subscribeAndWait returns.
+    logReader.connection.onReconnect = function () {
+      logsSubscription && logsSubscription.stop();
+      var opts = { streaming: options.streaming };
+      if (lastLogId)
+        opts.resumeAfterId = lastLogId;
+      // Don't use subscribeAndWait here; it'll deadlock. We can't
+      // process the sub messages until `onReconnect` returns, and
+      // `onReconnect` won't return unless the sub messages have been
+      // processed. There's no reason we need to wait for the sub to be
+      // ready here anyway.
+      // XXX correctly handle errors on resubscribe
+      logsSubscription = logReader.connection.subscribe(
+        "logsForApp",
+        options.app,
+        opts
+      );
+    };
+
     return options.streaming ? null : 0;
   } finally {
     // If not streaming, close the connection to log-reader so that

tools/watch.js

@@ -191,7 +191,7 @@ WatchSet.fromJSON = function (json) {
   set.files = _.clone(json.files);
 
   var reFromJSON = function (j) {
-    if (j.$regex)
+    if (_.has(j, '$regex'))
       return new RegExp(j.$regex, j.$options);
     return new RegExp(j);
   };