Merge remote-tracking branch 'origin/devel' into sso

docs/client/packages/browser-policy.html

@@ -40,8 +40,8 @@ cross-site scripting attacks by disabling all scripts except those loaded from a
 `script src` attribute.
 
 Meteor determines the browser policy when the server starts up, so you should
-call `BrowserPolicy` functions in top-level application code or in
-`Meteor.startup`.
+call `BrowserPolicy` functions on the server in top-level application code or in
+`Meteor.startup`. `BrowserPolicy` functions cannot be used in client code.
 
 #### Frame options
 

meteor

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-BUNDLE_VERSION=0.3.24
+BUNDLE_VERSION=0.3.25
 
 # OS Check. Put here because here is where we download the precompiled
 # bundles that are arch specific.

packages/application-configuration/config.js

@@ -58,9 +58,6 @@ try {
       packages: {
         'mongo-livedata': {
           url: process.env.MONGO_URL
-        },
-        'email': {
-          url: process.env.MAIL_URL
         }
       }
     };
@@ -83,10 +80,13 @@ AppConfig.getAppConfig = function () {
 AppConfig.configurePackage = function (packageName, configure) {
   var appConfig = AppConfig.getAppConfig(); // Will either be based in the env var,
                                          // or wait for galaxy to connect.
-  var lastConfig = appConfig && appConfig.packages && appConfig.packages[packageName];
-  if (lastConfig) {
-    configure(lastConfig);
-  }
+  var lastConfig =
+        (appConfig && appConfig.packages && appConfig.packages[packageName]) || {};
+  // Always call the configure callback "soon" even if the initial configuration
+  // is empty (synchronously, though deferred would be OK).
+  // XXX make sure that all callers of configurePackage deal well with multiple
+  // callback invocations!  eg, email does not
+  configure(lastConfig);
   var configureIfDifferent = function (app) {
     if (!EJSON.equals(app.config && app.config.packages && app.config.packages[packageName],
                       lastConfig)) {

packages/email/email.js

@@ -33,19 +33,26 @@ var makePool = function (mailUrlString) {
 
 // We construct smtpPool at the first call to Email.send, so that
 // Meteor.startup code can set $MAIL_URL.
-var smtpPool = null;
-var maybeMakePool = function () {
-  // We check MAIL_URL in case someone else set it in Meteor.startup code.
-  var poolFuture = new Future();
-  AppConfig.configurePackage('email', function (config) {
-    // TODO: allow reconfiguration.
-    if (!smtpPool && (config.url || process.env.MAIL_URL)) {
-      smtpPool = makePool(config.url || process.env.MAIL_URL);
-    }
-    poolFuture.return();
-  });
+var smtpPoolFuture = new Future;;
+var configured = false;
 
-  poolFuture.wait();
+var getPool = function () {
+  // We check MAIL_URL in case someone else set it in Meteor.startup code.
+  if (!configured) {
+    configured = true;
+    AppConfig.configurePackage('email', function (config) {
+      // XXX allow reconfiguration when the app config changes
+      if (smtpPoolFuture.isResolved())
+        return;
+      var url = config.url || process.env.MAIL_URL;
+      var pool = null;
+      if (url)
+        pool = makePool(url);
+      smtpPoolFuture.return(pool);
+    });
+  }
+
+  return smtpPoolFuture.wait();
 };
 
 var next_devmode_mail_id = 0;
@@ -81,8 +88,8 @@ var devModeSend = function (mc) {
   future.wait();
 };
 
-var smtpSend = function (mc) {
-  smtpPool._future_wrapped_sendMail(mc).wait();
+var smtpSend = function (pool, mc) {
+  pool._future_wrapped_sendMail(mc).wait();
 };
 
 /**
@@ -141,10 +148,9 @@ Email.send = function (options) {
     mc.addHeader(name, value);
   });
 
-  maybeMakePool();
-
-  if (smtpPool) {
-    smtpSend(mc);
+  var pool = getPool();
+  if (pool) {
+    smtpSend(pool, mc);
   } else {
     devModeSend(mc);
   }

packages/follower-livedata/follower.js

@@ -30,7 +30,6 @@ Follower = {
     }, options);
     // start each elector as untried/assumed connectable.
 
-    // for options.priority, low-priority things are tried first.
     var makeElectorTries = function (urlSet) {
 
       electorTries = {};
@@ -101,9 +100,9 @@ Follower = {
       }
 
       if (conn) {
-        prevReconnect.apply(conn, {
+        prevReconnect.apply(conn, [{
           url: url
-        });
+        }]);
       } else {
         conn = DDP.connect(url);
         prevReconnect = conn.reconnect;
@@ -197,7 +196,7 @@ Follower = {
       if (!intervalHandle)
         intervalHandle = monitorConnection();
       if (arguments[0] && arguments[0].url) {
-        makeElectorTries(arguments[0].url, {reset: true});
+        makeElectorTries(arguments[0].url);
         tryElector();
       } else {
         prevReconnect.apply(conn, arguments);

packages/handlebars/.npm/package/npm-shrinkwrap.json

@@ -1,7 +1,7 @@
 {
   "dependencies": {
     "handlebars": {
-      "version": "1.0.7",
+      "from": "https://github.com/meteor/handlebars.js/tarball/543ec6689cf663cfda2d8f26c3c27de40aad7bd5",
       "dependencies": {
         "optimist": {
           "version": "0.3.7",
@@ -10,9 +10,6 @@
               "version": "0.0.2"
             }
           }
-        },
-        "uglify-js": {
-          "version": "1.2.6"
         }
       }
     }

packages/handlebars/package.js

@@ -3,7 +3,11 @@ Package.describe({
   internal: true
 });
 
-Npm.depends({handlebars: '1.0.7'});
+Npm.depends({
+  // Fork of 1.0.7 dropping a used-only-by-bin/handlebars dependency on the very
+  // large uglify-js 1.2.6.
+  handlebars: 'https://github.com/meteor/handlebars.js/tarball/543ec6689cf663cfda2d8f26c3c27de40aad7bd5'
+});
 
 Package.on_use(function (api) {
   api.use('underscore');

packages/livedata/.npm/package/npm-shrinkwrap.json

@@ -10,7 +10,7 @@
           "version": "0.7.0",
           "dependencies": {
             "websocket-driver": {
-              "version": "0.3.0"
+              "version": "0.3.1"
             }
           }
         }

packages/livedata/stream_server.js

@@ -48,8 +48,15 @@ StreamServer = function () {
   if (!Package.webapp) {
     throw new Error("Cannot create a DDP server without the webapp package");
   }
+  // Install the sockjs handlers, but we want to keep around our own particular
+  // request handler that adjusts idle timeouts while we have an outstanding
+  // request.  This compensates for the fact that sockjs removes all listeners
+  // for "request" to add its own.
+  Package.webapp.WebApp.httpServer.removeListener('request', Package.webapp.WebApp._timeoutAdjustmentRequestCallback);
   self.server.installHandlers(Package.webapp.WebApp.httpServer);
-  Package.webapp.WebApp.httpServer.on('closing', function () {
+  Package.webapp.WebApp.httpServer.addListener('request', Package.webapp.WebApp._timeoutAdjustmentRequestCallback);
+
+  Package.webapp.WebApp.httpServer.on('meteor-closing', function () {
     _.each(self.open_sockets, function (socket) {
       socket.end();
     });

packages/minimongo/minimongo.js

@@ -1099,7 +1099,14 @@ LocalCollection._compileProjection = function (fields) {
   // Find the non-_id keys (_id is handled specially because it is included unless
   // explicitly excluded). Sort the keys, so that our code to detect overlaps
   // like 'foo' and 'foo.bar' can assume that 'foo' comes first.
-  var fieldsKeys = _.reject(_.keys(fields).sort(), function (key) { return key === '_id'; });
+  var fieldsKeys = _.keys(fields).sort();
+
+  // If there are other rules other than '_id', treat '_id' differently in a
+  // separate case. If '_id' is the only rule, use it to understand if it is
+  // including/excluding projection.
+  if (fieldsKeys.length > 0 && !(fieldsKeys.length === 1 && fieldsKeys[0] === '_id'))
+    fieldsKeys = _.reject(fieldsKeys, function (key) { return key === '_id'; });
+
   var including = null; // Unknown
   var projectionRulesTree = {}; // Tree represented as nested objects
 

packages/minimongo/minimongo_tests.js

@@ -992,6 +992,30 @@ Tinytest.add("minimongo - projection_compiler", function (test) {
      "blacklist nested - path not found in doc"]
   ]);
 
+  testProjection({ _id: 1 }, [
+    [{ _id: 42, x: 1, y: { z: "2" } },
+     { _id: 42 },
+     "_id whitelisted"],
+    [{ _id: 33 },
+     { _id: 33 },
+     "_id whitelisted, _id only"],
+    [{ x: 1 },
+     {},
+     "_id whitelisted, no _id"]
+  ]);
+
+  testProjection({ _id: 0 }, [
+    [{ _id: 42, x: 1, y: { z: "2" } },
+     { x: 1, y: { z: "2" } },
+     "_id blacklisted"],
+    [{ _id: 33 },
+     {},
+     "_id blacklisted, _id only"],
+    [{ x: 1 },
+     { x: 1 },
+     "_id blacklisted, no _id"]
+  ]);
+
   test.throws(function () {
     testProjection({ 'inc': 1, 'excl': 0 }, [
       [ { inc: 42, excl: 42 }, { inc: 42 }, "Can't combine incl/excl rules" ]

packages/webapp/webapp_server.js

@@ -12,6 +12,9 @@ var optimist = Npm.require('optimist');
 var useragent = Npm.require('useragent');
 var send = Npm.require('send');
 
+var SHORT_SOCKET_TIMEOUT = 5*1000;
+var LONG_SOCKET_TIMEOUT = 120*1000;
+
 WebApp = {};
 WebAppInternals = {};
 
@@ -193,6 +196,27 @@ Meteor.startup(function () {
 });
 
 
+
+// When we have a request pending, we want the socket timeout to be long, to
+// give ourselves a while to serve it, and to allow sockjs long polls to
+// complete.  On the other hand, we want to close idle sockets relatively
+// quickly, so that we can shut down relatively promptly but cleanly, without
+// cutting off anyone's response.
+WebApp._timeoutAdjustmentRequestCallback = function (req, res) {
+  // this is really just req.socket.setTimeout(LONG_SOCKET_TIMEOUT);
+  req.setTimeout(LONG_SOCKET_TIMEOUT);
+  // Insert our new finish listener to run BEFORE the existing one which removes
+  // the response from the socket.
+  var finishListeners = res.listeners('finish');
+  // XXX Apparently in Node 0.12 this event is now called 'prefinish'.
+  // https://github.com/joyent/node/commit/7c9b6070
+  res.removeAllListeners('finish');
+  res.on('finish', function () {
+    res.setTimeout(SHORT_SOCKET_TIMEOUT);
+  });
+  _.each(finishListeners, function (l) { res.on('finish', l); });
+};
+
 var runWebAppServer = function () {
   var shuttingDown = false;
   // read the control for the client we'll be serving up
@@ -414,44 +438,25 @@ var runWebAppServer = function () {
   var httpServer = http.createServer(app);
   var onListeningCallbacks = [];
 
-  var longPollingSockets = {};
+  // After 5 seconds w/o data on a socket, kill it.  On the other hand, if
+  // there's an outstanding request, give it a higher timeout instead (to avoid
+  // killing long-polling requests)
+  httpServer.setTimeout(SHORT_SOCKET_TIMEOUT);
 
-  // After 5 seconds of a socket being open, assume it is a long-polling
-  // connection that we have to keep track of to shut down when we're shutting
-  // down the server overall.
-  httpServer.setTimeout(5000, Meteor.bindEnvironment(function (socket) {
-    if (shuttingDown) {
-      socket.end();
-    } else {
-      socket._meteorLongPollingId = Random.id();
-      longPollingSockets[socket._meteorLongPollingId] = socket;
-      // give the socket another minute to live.
-      var destroy = Meteor.setTimeout(function () {
-        delete longPollingSockets[socket._meteorLongPollingId];
-        socket.removeListener('close', onClose);
-        socket.destroy();
-      }, 60*1000);
-
-      var onClose =  function () {
-        delete longPollingSockets[socket._meteorLongPollingId];
-        Meteor.clearTimeout(destroy);
-      };
-      socket.on('close', onClose);
-    }
-  }, function (err) {
-    console.log(err);
-  }));
+  // Do this here, and then also in livedata/stream_server.js, because
+  // stream_server.js kills all the current request handlers when installing its
+  // own.
+  httpServer.on('request', WebApp._timeoutAdjustmentRequestCallback);
 
 
   // For now, handle SIGHUP here.  Later, this should be in some centralized
   // Meteor shutdown code.
   process.on('SIGHUP', Meteor.bindEnvironment(function () {
     shuttingDown = true;
-    _.each(longPollingSockets, function (socket, id) {
-      socket.end();
-    });
     // tell others with websockets open that we plan to close this.
-    httpServer.emit('closing');
+    // XXX: Eventually, this should be done with a standard meteor shut-down
+    // logic path.
+    httpServer.emit('meteor-closing');
     httpServer.close( function () {
       process.exit(0);
     });

scripts/generate-dev-bundle.sh

@@ -100,7 +100,6 @@ which npm
 cd "$DIR/lib/node_modules"
 npm install optimist@0.6.0
 npm install semver@2.1.0
-npm install handlebars@1.0.7
 npm install request@2.27.0
 npm install keypress@0.2.1
 npm install underscore@1.5.2

tools/app.html.in

@@ -1,17 +0,0 @@
-<!DOCTYPE html>
-<html##HTML_ATTRIBUTES##>
-<head>
-{{#each stylesheets}}  <link rel="stylesheet" href="##ROOT_URL_PATH_PREFIX##{{this}}">
-{{/each}}
-
-##RUNTIME_CONFIG##
-
-{{#each scripts}}  <script type="text/javascript" src="##ROOT_URL_PATH_PREFIX##{{this}}"></script>
-{{/each}}
-
-{{{head_extra}}}
-</head>
-<body>
-{{{body_extra}}}
-</body>
-</html>

tools/bundler.js

@@ -769,19 +769,39 @@ _.extend(ClientTarget.prototype, {
     self.css[0].setUrlToHash(".css");
   },
 
+  // XXX Instead of packaging the boilerplate in the client program, the
+  // template should be part of WebApp, and we should make sure that all
+  // information that it needs is in the manifest (ie, make sure to include head
+  // and body).  Then it will just need to do one level of templating instead
+  // of two.  Alternatively, use spacebars with unipackage.load here.
   generateHtmlBoilerplate: function () {
     var self = this;
 
-    var templatePath = path.join(__dirname, "app.html.in");
-    var template = watch.readAndWatchFile(self.watchSet, templatePath);
-
-    var f = require('handlebars').compile(template.toString());
-    return new Buffer(f({
-      scripts: _.pluck(self.js, 'url'),
-      stylesheets: _.pluck(self.css, 'url'),
-      head_extra: self.head.join('\n'),
-      body_extra: self.body.join('\n')
-    }), 'utf8');
+    var html = [];
+    html.push('<!DOCTYPE html>\n' +
+              '<html##HTML_ATTRIBUTES##>\n' +
+              '<head>\n');
+    _.each(self.css, function (css) {
+      html.push('  <link rel="stylesheet" href="##ROOT_URL_PATH_PREFIX##');
+      html.push(_.escape(css.url));
+      html.push('">\n');
+    });
+    html.push('\n\n##RUNTIME_CONFIG##\n\n');
+    _.each(self.js, function (js) {
+      html.push('  <script type="text/javascript" src="##ROOT_URL_PATH_PREFIX##');
+      html.push(_.escape(js.url));
+      html.push('"></script>\n');
+    });
+    html.push('\n\n');
+    html.push(self.head.join('\n'));  // unescaped!
+    html.push('\n' +
+              '</head>\n' +
+              '<body>\n');
+    html.push(self.body.join('\n'));  // unescaped!
+    html.push('\n' +
+              '</body>\n' +
+              '</html>\n');
+    return new Buffer(html.join(''), 'utf8');
   },
 
   // Output the finished target to disk

tools/library.js

@@ -25,15 +25,7 @@ var Library = function (options) {
 
   // Trim down localPackageDirs to just those that actually exist (and
   // that are actually directories)
-  self.localPackageDirs = _.filter(options.localPackageDirs, function (dir) {
-      try {
-        // use stat rather than lstat since symlink to dir is OK
-        var stats = fs.statSync(dir);
-      } catch (e) {
-        return false;
-      }
-      return stats.isDirectory();
-    });
+  self.localPackageDirs = _.filter(options.localPackageDirs, isDirectory);
 
   self.overrides = {}; // package name to package directory
 
@@ -95,6 +87,9 @@ _.extend(Library.prototype, {
   // called from Package initialization code. Intended primarily for comparison
   // to the packageDirForBuildInfo field on a Package object; also used
   // internally to implement 'get'.
+  //
+  // If it finds a directory named name inside one of the localPackageDirs which
+  // contains nothing but ".build", it deletes that directory.
   findPackageDirectory: function (name) {
     var self = this;
 
@@ -110,6 +105,9 @@ _.extend(Library.prototype, {
 
     for (var i = 0; i < self.localPackageDirs.length; ++i) {
       var packageDir = path.join(self.localPackageDirs[i], name);
+      if (!isDirectory(packageDir))
+        continue;
+
       // A directory is a package if it either contains 'package.js' (a package
       // source tree) or 'unipackage.json' (a compiled unipackage). (Actually,
       // for now, unipackages contain a dummy package.js too.)
@@ -129,6 +127,13 @@ _.extend(Library.prototype, {
           fs.existsSync(path.join(packageDir, 'unipackage.json'))) {
         return packageDir;
       }
+
+      // Does this package directory just contain a ".build" subdirectory and
+      // nothing else? Most likely, this package was created on another branch
+      // of meteor, and when you checked this branch out it left around the
+      // gitignored .build directory. Clean it up.
+      if (_.isEqual(fs.readdirSync(packageDir), ['.build']))
+        files.rm_recursive(packageDir);
     }
 
     // Try the Meteor distribution, if we have one.
@@ -464,3 +469,13 @@ _.extend(exports, {
     return out;
   }
 });
+
+var isDirectory = function (dir) {
+  try {
+    // use stat rather than lstat since symlink to dir is OK
+    var stats = fs.statSync(dir);
+  } catch (e) {
+    return false;
+  }
+  return stats.isDirectory();
+};

tools/meteor_npm.js

@@ -210,6 +210,10 @@ _.extend(exports, {
     var installedDependencies = self._installedDependencies(packageNpmDir);
 
     // If we already have the right things installed, life is good.
+    // XXX this check is wrong: what if we just pulled a commit that changes
+    //     a sub-module in npm-shrinkwrap.json? See #1648
+    //     But while it might be "correct" to just drop this check we should
+    //     be careful not to make the common case of no changes too slow.
     if (_.isEqual(installedDependencies, npmDependencies))
       return;