diff --git a/packages/spiderable/package.js b/packages/spiderable/package.js new file mode 100644 index 0000000000..6951f41fc0 --- /dev/null +++ b/packages/spiderable/package.js @@ -0,0 +1,10 @@ +Package.describe({ + summary: "Makes the application crawlable to web spiders." +}); + +Package.on_use(function (api) { + api.use(['templating'], 'client'); + + api.add_files('spiderable.html', 'client'); + api.add_files('spiderable.js', 'server'); +}); diff --git a/packages/spiderable/spiderable.html b/packages/spiderable/spiderable.html new file mode 100644 index 0000000000..7cbdf71b89 --- /dev/null +++ b/packages/spiderable/spiderable.html @@ -0,0 +1 @@ + diff --git a/packages/spiderable/spiderable.js b/packages/spiderable/spiderable.js new file mode 100644 index 0000000000..fb162ad348 --- /dev/null +++ b/packages/spiderable/spiderable.js @@ -0,0 +1,71 @@ +(function () { + var fs = __meteor_bootstrap__.require('fs'); + var spawn = __meteor_bootstrap__.require('child_process').spawn; + var querystring = __meteor_bootstrap__.require('querystring'); + var app = __meteor_bootstrap__.app; + + app.use(function (req, res, next) { + if (/\?.*_escaped_fragment_=/.test(req.url)) { + // get escaped fragment out of the url. Gross! + var preQuery = req.url.split("?")[0]; + var queryStr = req.url.split("?")[1]; + var parsed = querystring.parse(queryStr); + delete parsed['_escaped_fragment_']; + var newQuery = querystring.stringify(parsed); + var newPath = preQuery + (newQuery ? "?" + newQuery : ""); + var url = "http://" + req.headers.host + newPath; + console.log("GB", url); + + // run phantomjs + var tmpfile = "/tmp/" + + (((1+Math.random())*0x1000000)|0).toString(16) + ".js"; + fs.writeFile( + tmpfile, + "var url = '" + url + "';" + +"var page = require('webpage').create();" + + +"page.open(url);" + + +"var lastContent;" + +"var settledCount = 0;" + +"var count = 0;" + + +"setInterval(function() {" + +" var connected = page.evaluate(function () {" + +" return typeof Meteor !== 'undefined' && Meteor.status().connected;" + +" });" + +" if (!connected || page.content !== lastContent) {" + +" settledCount = 0;" + +" lastContent = page.content;" + +" } else {" + +" settledCount += 1;" + +" }" + + +" if (settledCount >= 3 || count >= 100) {" + +" var out = page.content;" + +" out = out.replace(/]+>(.|\\n|\\r)*?<\\/script\\s*>/ig, '');" + +" out = out.replace('', '');" + + +" console.log(out);" + +" phantom.exit();" + +" }" + +"}, 100);", + function (err) { + if (err) { // can't write file? + next(); + return; + } + // XXX make sure phantomjs in the path! + var cp = spawn('phantomjs', ['--load-images=no', tmpfile]); + cp.on('exit', function (code) { + // XXX look at code + res.end(); + fs.unlink(tmpfile); + }); + cp.stdout.pipe(res); + }); + } else { + next(); + } + }); +})();