Merge branch 'master' into auth

Conflicts:
	examples/todos/.meteor/packages

History.md

@@ -1,8 +1,15 @@
 
 ## vNEXT
 
+## v0.3.9
+
+* Add `spiderable` package to allow web crawlers to index Meteor apps.
+
+* `meteor deploy` uses SSL to protect application deployment.
+
 * Fix `stopImmediatePropagation()`. #205
 
+
 ## v0.3.8
 
 * HTTPS support

README.md

@@ -74,3 +74,4 @@ in several ways:
 * IRC: ```#meteor``` on ```irc.freenode.net```
 * Ask a question: http://stackoverflow.com/questions/tagged/meteor
 * Email us: ```contact@meteor.com```
+* How to contribute to Meteor: https://github.com/meteor/meteor/wiki

admin/debian/changelog

@@ -1,4 +1,4 @@
-meteor (0.3.8-1) unstable; urgency=low
+meteor (0.3.9-1) unstable; urgency=low
 
   * Automated debian build.
 

admin/install-s3.sh

@@ -5,7 +5,7 @@
 ## example.
 
 URLBASE="https://d3sqy0vbqsdhku.cloudfront.net"
-VERSION="0.3.8"
+VERSION="0.3.9"
 PKGVERSION="${VERSION}-1"
 
 UNAME=`uname`

admin/manifest.json

@@ -1,6 +1,6 @@
 {
-  "version": "0.3.8",
-  "deb_version": "0.3.8-1",
-  "rpm_version": "0.3.8-1",
+  "version": "0.3.9",
+  "deb_version": "0.3.9-1",
+  "rpm_version": "0.3.9-1",
   "urlbase": "https://d3sqy0vbqsdhku.cloudfront.net"
 }

admin/meteor.spec

@@ -5,7 +5,7 @@
 Summary: Meteor platform and JavaScript application server
 Vendor: Meteor
 Name: meteor
-Version: 0.3.8
+Version: 0.3.9
 Release: 1
 License: MIT
 Group: Networking/WWW

app/lib/updater.js

@@ -1,4 +1,4 @@
-exports.CURRENT_VERSION = "0.3.8";
+exports.CURRENT_VERSION = "0.3.9";
 
 var fs = require("fs");
 var http = require("http");

app/meteor/deploy.js

@@ -23,7 +23,7 @@ var DEPLOY_HOSTNAME = 'deploy.meteor.com';
 // interactively prompt for here.
 
 var meteor_rpc = function (rpc_name, method, site, query_params, callback) {
-  var url = "http://" + DEPLOY_HOSTNAME + '/' + rpc_name + '/' + site;
+  var url = "https://" + DEPLOY_HOSTNAME + '/' + rpc_name + '/' + site;
 
   if (!_.isEmpty(query_params))
     url += '?' + qs.stringify(query_params);
@@ -309,7 +309,7 @@ var read_password = function (callback) {
 // called exactly once. Calls callback with the entered password, or
 // undefined if no password is required.
 var with_password = function (site, callback) {
-  var check_url = "http://" + DEPLOY_HOSTNAME + "/has_password/" + site;
+  var check_url = "https://" + DEPLOY_HOSTNAME + "/has_password/" + site;
 
   request(check_url, function (error, response, body) {
     if (error || response.statusCode !== 200) {

app/meteor/post-upgrade.js

@@ -2,7 +2,7 @@ try {
   // XXX can't get this from updater.js because in 0.3.7 and before the
   // updater didn't have the right NODE_PATH set. At some point we can
   // remove this and just use updater.CURRENT_VERSION.
-  var VERSION = "0.3.8";
+  var VERSION = "0.3.9";
 
   var fs = require('fs');
   var path = require('path');

docs/.meteor/packages

@@ -9,3 +9,4 @@ showdown
 code-prettify
 jquery-waypoints
 less
+spiderable

docs/client/docs.html

@@ -11,7 +11,7 @@
   </div>
   <div id="main">
     <div id="top"></div>
-    <h2 class="main-headline">Meteor 0.3.8</h1>
+    <h1 class="main-headline">Meteor 0.3.9</h1>
     {{> introduction }}
     {{> concepts }}
     {{> api }}

docs/client/docs.js

@@ -1,4 +1,4 @@
-METEOR_VERSION = "0.3.8";
+METEOR_VERSION = "0.3.9";
 
 Meteor.startup(function () {
   // XXX this is broken by the new multi-page layout.  Also, it was
@@ -194,6 +194,7 @@ var toc = [
     "jquery",
     "less",
     "sass",
+    "spiderable",
     "stylus",
     "showdown",
     "underscore"

docs/client/packages.html

@@ -24,6 +24,7 @@ and removed with:
 {{> pkg_jquery}}
 {{> pkg_less}}
 {{> pkg_sass}}
+{{> pkg_spiderable}}
 {{> pkg_stylus}}
 {{> pkg_showdown}}
 {{> pkg_underscore}}

docs/client/packages/spiderable.html

@@ -0,0 +1,41 @@
+<template name="pkg_spiderable">
+{{#better_markdown}}
+## `spiderable`
+
+
+The `spiderable` package is a temporary solution to allow web search
+engines to index a Meteor application. It uses the <a target="_blank"
+href="https://developers.google.com/webmasters/ajax-crawling/">AJAX
+Crawling specification</a> published by Google to serve HTML to
+compatible spiders (Google, Bing, Yandex, and more).
+
+When a spider requests an HTML snapshot of a page the Meteor server runs
+the client half of the application inside <a target="_blank"
+href="http://phantomjs.org/">phantomjs</a>, a headless browser, and
+returns the full HTML generated by the client code.
+
+{{#warning}}
+This is a temporary approach to allow Meteor applications to be
+searchable. Expect significant changes to this package.
+{{/warning}}
+
+In order to have links between multiple pages on a site visible to
+spiders, apps must use real links (eg `<a href="/about">`) rather than
+simply re-rendering portions of the page when an element is
+clicked. Apps should render their content based on the URL of the page
+and can use HTML5 push-state to alter the URL on the client without
+triggering a page reload. See the <a target="_blank"
+href="http://meteor.com/examples/todos">Todos example</a> for a
+demonstration.
+
+
+{{#warning}}
+If you deploy your application with `meteor bundle`, you must install
+`phantomjs` (<a target="_blank"
+href="http://phantomjs.org/">http://phantomjs.org</a>) somewhere in your
+`$PATH`. If you use `meteor deploy` this is already taken care of.
+{{/warning}}
+
+
+{{/better_markdown}}
+</template>

examples/todos/.meteor/packages

@@ -5,6 +5,7 @@
 
 underscore
 backbone
+spiderable
 accounts
 accounts-ui
 accounts-weibo

examples/todos/client/todos.css

@@ -138,6 +138,8 @@ h3 {
 
 #lists .list-name {
     cursor: pointer;
+    color: black;
+    text-decoration: none;
 }
 
 #createList {

examples/todos/client/todos.html

@@ -30,9 +30,9 @@
           </div>
         {{else}}
           <div class="display">
-            <div class="list-name {{name_class}}">
+            <a class="list-name {{name_class}}" href="/{{_id}}">
               {{name}}
-            </div>
+            </a>
           </div>
         {{/if}}
       </div>

examples/todos/client/todos.js

@@ -87,6 +87,10 @@ Template.lists.events = {
   'mousedown .list': function (evt) { // select list
     Router.setList(this._id);
   },
+  'click .list': function (evt) {
+    // prevent clicks on <a> from refreshing the page.
+    evt.preventDefault();
+  },
   'dblclick .list': function (evt) { // start editing list name
     Session.set('editing_listname', this._id);
     Meteor.flush(); // force DOM redraw, so we can focus the edit field

packages/livedata/livedata_connection.js

@@ -760,7 +760,9 @@ _.extend(Meteor, {
   //     "http://subdomain.meteor.com/sockjs" (deprecated),
   //     "/sockjs" (deprecated)
   connect: function (url, _restartOnUpdate) {
-    return new Meteor._LivedataConnection(url, _restartOnUpdate);
+    var ret = new Meteor._LivedataConnection(url, _restartOnUpdate);
+    Meteor._LivedataConnection._allConnections.push(ret); // hack. see below.
+    return ret;
   },
 
   autosubscribe: function (sub_func) {
@@ -789,3 +791,14 @@ _.extend(Meteor, {
   }
 });
 
+
+// Hack for `spiderable` package: a way to see if the page is done
+// loading all the data it needs.
+Meteor._LivedataConnection._allConnections = [];
+Meteor._LivedataConnection._allSubscriptionsReady = function () {
+  return _.all(Meteor._LivedataConnection._allConnections, function (conn) {
+    for (var k in conn.sub_ready_callbacks)
+      return false;
+    return true;
+  });
+};

packages/spiderable/package.js

@@ -0,0 +1,10 @@
+Package.describe({
+  summary: "Makes the application crawlable to web spiders."
+});
+
+Package.on_use(function (api) {
+  api.use(['templating'], 'client');
+
+  api.add_files('spiderable.html', 'client');
+  api.add_files('spiderable.js', 'server');
+});

packages/spiderable/spiderable.html

@@ -0,0 +1 @@
+<head><meta name="fragment" content="!"></head>

packages/spiderable/spiderable.js

@@ -0,0 +1,90 @@
+(function () {
+  var fs = __meteor_bootstrap__.require('fs');
+  var spawn = __meteor_bootstrap__.require('child_process').spawn;
+  var querystring = __meteor_bootstrap__.require('querystring');
+  var app = __meteor_bootstrap__.app;
+
+  // how long to let phantomjs run before we kill it
+  var REQUEST_TIMEOUT = 15*1000;
+
+  app.use(function (req, res, next) {
+    if (/\?.*_escaped_fragment_=/.test(req.url)) {
+      // get escaped fragment out of the url.
+      var idx = req.url.indexOf('?');
+      var preQuery = req.url.substr(0, idx);
+      var queryStr = req.url.substr(idx + 1);
+      var parsed = querystring.parse(queryStr);
+      delete parsed['_escaped_fragment_'];
+      var newQuery = querystring.stringify(parsed);
+      var newPath = preQuery + (newQuery ? "?" + newQuery : "");
+      var url = "http://" + req.headers.host + newPath;
+
+      // run phantomjs
+      //
+      // Use '/dev/stdin' to avoid writing to a temporary file. Can't
+      // just omit the file, as PhantomJS takes that to mean 'use a
+      // REPL' and exits as soon as stdin closes.
+      var cp = spawn('phantomjs', ['--load-images=no', '/dev/stdin']);
+
+      var data = '';
+      cp.stdout.setEncoding('utf8');
+      cp.stdout.on('data', function (chunk) {
+        data += chunk;
+      });
+
+      cp.on('exit', function (code) {
+        if (0 === code && /<html>/i.test(data)) {
+          res.writeHead(200, {'Content-Type': 'text/html; charset=UTF-8'});
+          res.end(data);
+        } else {
+          // phantomjs failed. Don't send the error, instead send the
+          // normal page.
+          if (code === 127)
+            Meteor._debug("spiderable: phantomjs not installed. Download and install from http://phantomjs.org/");
+          else
+            Meteor._debug("spiderable: phantomjs failed:", code, data);
+
+          next();
+        }
+      });
+
+      // don't crash w/ EPIPE if phantomjs isn't installed.
+      cp.stdin.on('error', function () {});
+
+      cp.stdin.write(
+        "var url = '" + url + "';" +
+"var page = require('webpage').create();" +
+"page.open(url);" +
+
+"setInterval(function() {" +
+"  var ready = page.evaluate(function () {" +
+"    if (typeof Meteor !== 'undefined' && Meteor.status().connected) {" +
+"      Meteor.flush();" +
+"      return Meteor._LivedataConnection._allSubscriptionsReady();" +
+"    }" +
+"    return false;" +
+"  });" +
+
+"  if (ready) {" +
+"    var out = page.content;" +
+"    out = out.replace(/<script[^>]+>(.|\\n|\\r)*?<\\/script\\s*>/ig, '');" +
+"    out = out.replace('<meta name=\"fragment\" content=\"!\">', '');" +
+
+"    console.log(out);" +
+"    phantom.exit();" +
+"  }" +
+"}, 100);");
+      cp.stdin.end();
+
+      // Just kill it if it takes too long.
+      setTimeout(function () {
+        if (cp && cp.pid) {
+          cp.kill();
+        }
+      }, REQUEST_TIMEOUT);
+
+    } else {
+      next();
+    }
+  });
+})();