From 2c343a0788568cd676183631faa609f32abfa9cb Mon Sep 17 00:00:00 2001
From: Avital Oliver <avital@thewe.net>
Date: Wed, 13 May 2015 19:28:14 -0700
Subject: [PATCH] spiderable: Let apps configure timeout for phantomjs

`Spiderable.requestTimeout` can now be changed in server code in an app
to the number of milliseconds to wait until spiderable gives up on
phantomjs.

This is motivated by frontpage hitting 15 seconds at times (due to some
other problem we have), but regardless slow page loads are better than
non-crawlable ones.
---
 History.md                               | 3 +++
 packages/spiderable/spiderable_server.js | 7 ++++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/History.md b/History.md
index 4ed784023c..5f92f18be5 100644
--- a/History.md
+++ b/History.md
@@ -125,6 +125,9 @@
   package on `EmailInternals.NpmModules`. Allow specifying a `MailComposer`
   object to `Email.send` instead of individual options.  #4209
 
+* Expose `Spiderable.requestTimeout` from `spiderable` package to
+  allow apps to set the timeout for running phantomjs.
+
 
 ### Other bug fixes and improvements
 
diff --git a/packages/spiderable/spiderable_server.js b/packages/spiderable/spiderable_server.js
index 1773a6f20b..6d5d4db894 100644
--- a/packages/spiderable/spiderable_server.js
+++ b/packages/spiderable/spiderable_server.js
@@ -15,8 +15,9 @@ var urlParser = Npm.require('url');
 Spiderable.userAgentRegExps = [
     /^facebookexternalhit/i, /^linkedinbot/i, /^twitterbot/i];
 
-// how long to let phantomjs run before we kill it
-var REQUEST_TIMEOUT = 15*1000;
+// how long to let phantomjs run before we kill it (and send down the
+// regular page instead). Users may modify this number.
+Spiderable.requestTimeout = 15*1000;
 // maximum size of result HTML. node's default is 200k which is too
 // small for our docs.
 var MAX_BUFFER = 5*1024*1024; // 5MB
@@ -107,7 +108,7 @@ WebApp.connectHandlers.use(function (req, res, next) {
       ['-c',
        ("exec phantomjs " + phantomJsArgs + " /dev/stdin <<'END'\n" +
         phantomScript + "END\n")],
-      {timeout: REQUEST_TIMEOUT, maxBuffer: MAX_BUFFER},
+      {timeout: Spiderable.requestTimeout, maxBuffer: MAX_BUFFER},
       function (error, stdout, stderr) {
         if (!error && /<html/i.test(stdout)) {
           res.writeHead(200, {'Content-Type': 'text/html; charset=UTF-8'});