From 4ca704f93f22300499b09e0d83e457a1a85e54e6 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Mon, 8 Sep 2014 09:14:21 -0700 Subject: [PATCH 01/19] Simple (generic) http proxy for meteor run & test-packages Start it by passing --http-proxy-port=9999 --- tools/commands.js | 4 + tools/run-all.js | 17 ++++ tools/run-httpproxy.js | 196 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 217 insertions(+) create mode 100644 tools/run-httpproxy.js diff --git a/tools/commands.js b/tools/commands.js index 1677338723..28e10c2066 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -151,6 +151,7 @@ main.registerCommand({ options: { port: { type: String, short: "p", default: '3000' }, 'app-port': { type: String }, + 'http-proxy-port': { type: String }, production: { type: Boolean }, 'raw-logs': { type: Boolean }, settings: { type: String }, @@ -278,6 +279,7 @@ main.registerCommand({ return runAll.run(options.appDir, { proxyPort: parsedHostPort.port, proxyHost: parsedHostPort.host, + httpProxyPort: options['http-proxy-port'], appPort: appPort, appHost: appHost, settingsFile: options.settings, @@ -1119,6 +1121,7 @@ main.registerCommand({ maxArgs: Infinity, options: { port: { type: String, short: "p", default: "localhost:3000" }, + 'http-proxy-port': { type: String }, deploy: { type: String }, production: { type: Boolean }, settings: { type: String }, @@ -1394,6 +1397,7 @@ var runTestAppForPackages = function (testPackages, testRunnerAppDir, options) { // a switch to a different release appDirForVersionCheck: options.appDir, proxyPort: options.port, + httpProxyPort: options['http-proxy-port'], disableOplog: options['disable-oplog'], settingsFile: options.settings, banner: "Tests", diff --git a/tools/run-all.js b/tools/run-all.js index 9dec9c890c..79d0c162bd 100644 --- a/tools/run-all.js +++ b/tools/run-all.js @@ -5,6 +5,7 @@ var release = require('./release.js'); var runLog = require('./run-log.js'); var Proxy = require('./run-proxy.js').Proxy; +var HttpProxy = require('./run-httpproxy.js').HttpProxy; var AppRunner = require('./run-app.js').AppRunner; var MongoRunner = require('./run-mongo.js').MongoRunner; var Updater = require('./run-updater.js').Updater; @@ -44,6 +45,13 @@ var Runner = function (appDir, options) { onFailure: options.onFailure }); + self.httpProxy = null; + if (options.httpProxyPort) { + self.httpProxy = new HttpProxy({ + listenPort: options.httpProxyPort + }) + } + self.mongoRunner = null; var mongoUrl, oplogUrl; if (options.mongoUrl) { @@ -99,6 +107,14 @@ _.extend(Runner.prototype, { self.updater.start(); } + // print the banner only once we've successfully bound the port + if (! self.stopped && self.httpProxy) { + self.httpProxy.start(); + if (! self.quiet) { + runLog.log("=> Started http proxy."); + } + } + if (! self.stopped && self.mongoRunner) { var spinner = ['-', '\\', '|', '/']; // I looked at some Unicode indeterminate progress indicators, such as: @@ -161,6 +177,7 @@ _.extend(Runner.prototype, { self.stopped = true; self.proxy.stop(); + self.httpProxy && self.httpProxy.stop(); self.updater.stop(); self.mongoRunner && self.mongoRunner.stop(); self.appRunner.stop(); diff --git a/tools/run-httpproxy.js b/tools/run-httpproxy.js new file mode 100644 index 0000000000..1b29cfa158 --- /dev/null +++ b/tools/run-httpproxy.js @@ -0,0 +1,196 @@ +// The HTTP proxy is primarily so we can use localhost:3000 with OAuth, +// on devices which don't run a webserver e.g. Android / iOS +// This is a generic HTTP proxy, like a mini-Squid +// (whereas run-proxy.js is just for our app) +var _ = require('underscore'); +var Future = require('fibers/future'); +var runLog = require('./run-log.js'); +var url = require('url'); + +// options: listenPort, proxyToPort, proxyToHost, onFailure +var HttpProxy = function (options) { + var self = this; + + self.listenPort = options.listenPort; + self.listenHost = options.listenHost; + + self.onFailure = options.onFailure || function () {}; + + self.mode = "proxy"; + self.httpQueue = []; // keys: req, res + self.websocketQueue = []; // keys: req, socket, head + + self.proxy = null; + self.server = null; +}; + +_.extend(HttpProxy.prototype, { + // Start the proxy server, block (yield) until it is ready to go + // (actively listening on outer and proxying to inner), and then + // return. + start: function () { + var self = this; + + if (self.server) + throw new Error("already running?"); + + self.started = false; + + var http = require('http'); + var net = require('net'); + var httpProxy = require('http-proxy'); + + self.proxy = httpProxy.createProxyServer({ + // agent is required to handle keep-alive, and http-proxy 1.0 is a little + // buggy without it: https://github.com/nodejitsu/node-http-proxy/pull/488 + agent: new http.Agent({ maxSockets: 100 }), + xfwd: false //true + }); + + var server = self.server = http.createServer(function (req, res) { + // Normal HTTP request + self.httpQueue.push({ req: req, res: res }); + self._tryHandleConnections(); + }); + + self.server.on('upgrade', function (req, socket, head) { + // Websocket connection + self.websocketQueue.push({ req: req, socket: socket, head: head }); + self._tryHandleConnections(); + }); + + var fut = new Future; + self.server.on('error', function (err) { + if (err.code === 'EADDRINUSE') { + var port = self.listenPort; + runLog.log( + "HTTP proxy server can't listen on port " + port + ". \n" + + "If something else is using port " + port + ", you can\n" + + "specify an alternative port with --http-proxy-port ."); + } else if (self.listenHost && + (err.code === 'ENOTFOUND' || err.code === 'EADDRNOTAVAIL')) { + // This handles the case of "entered a DNS name that's unknown" + // (ENOTFOUND from getaddrinfo) and "entered some random IP that we + // can't bind to" (EADDRNOTAVAIL from listen). + runLog.log( + "Can't listen on host " + self.listenHost + + " (" + err.code + " from " + err.syscall + ")."); + } else { + runLog.log('' + err); + } + self.onFailure(); + // Allow start() to return. + fut.isResolved() || fut['return'](); + }); + + // Don't crash if the app doesn't respond; instead return an error + // immediately. + self.proxy.on('error', function (err, req, resOrSocket) { + if (resOrSocket instanceof http.ServerResponse) { + resOrSocket.writeHead(503, { + 'Content-Type': 'text/plain' + }); + resOrSocket.end('Unexpected error.'); + } else if (resOrSocket instanceof net.Socket) { + resOrSocket.end(); + } + }); + + self.server.listen(self.listenPort, self.listenHost || '0.0.0.0', function () { + if (self.server) { + self.started = true; + } else { + // stop() got called while we were invoking listen! Close the server (we + // still have the var server). The rest of the cleanup shouldn't be + // necessary. + server.close(); + } + fut.isResolved() || fut['return'](); + }); + + fut.wait(); + }, + + // Idempotent. + stop: function () { + var self = this; + + if (! self.server) + return; + + if (! self.started) { + // This probably means that we failed to listen. However, there could be a + // race condition and we could be in the middle of starting to listen! In + // that case, the listen callback will notice that we nulled out server + // here. + self.server = null; + return; + } + + // This stops listening but allows existing connections to + // complete gracefully. + self.server.close(); + self.server = null; + + // It doesn't seem to be necessary to do anything special to + // destroy an httpProxy proxyserver object. + self.proxy = null; + + // Drop any held connections. + _.each(self.httpQueue, function (c) { + c.res.statusCode = 500; + c.res.end(); + }); + self.httpQueue = []; + + _.each(self.websocketQueue, function (c) { + c.socket.destroy(); + }); + self.websocketQueue = []; + + self.mode = "hold"; + }, + + _tryHandleConnections: function () { + var self = this; + + while (self.httpQueue.length) { + if (self.mode !== "proxy") + break; + + var c = self.httpQueue.shift(); + var req = c.req; + var targetUrl = req.url; + runLog.log("Proxy request: " + req.method + " " +req.url); + var newUrl = req.url + self.proxy.web(c.req, c.res, { + target: targetUrl + }); + } + + while (self.websocketQueue.length) { + if (self.mode !== "proxy") + break; + + var c = self.websocketQueue.shift(); + var req = c.req; + var targetUrl = req.url; + runLog.log("Proxy request (websocket): " + req.method + " " +req.url); + self.proxy.ws(c.req, c.socket, c.head, { + target: targetUrl + }); + } + }, + + // The proxy can be in one of three modes: + // - "proxy": connections are proxied + // + // The initial mode is "proxy". + setMode: function (mode) { + var self = this; + self.mode = mode; + self._tryHandleConnections(); + } +}); + +exports.HttpProxy = HttpProxy; From efdb867ff4d5ee74fa1fefb89daf35b298ff96cc Mon Sep 17 00:00:00 2001 From: Justin SB Date: Mon, 8 Sep 2014 11:32:03 -0700 Subject: [PATCH 02/19] Support CONNECT method, which is what Android uses --- tools/run-httpproxy.js | 84 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/tools/run-httpproxy.js b/tools/run-httpproxy.js index 1b29cfa158..175357dea1 100644 --- a/tools/run-httpproxy.js +++ b/tools/run-httpproxy.js @@ -19,6 +19,7 @@ var HttpProxy = function (options) { self.mode = "proxy"; self.httpQueue = []; // keys: req, res self.websocketQueue = []; // keys: req, socket, head + self.connectQueue = []; // keys: req, socket, head self.proxy = null; self.server = null; @@ -53,6 +54,11 @@ _.extend(HttpProxy.prototype, { self._tryHandleConnections(); }); + self.server.on('connect', function (req, socket, head) { + self.connectQueue.push({ req: req, socket: socket, head: head }); + self._tryHandleConnections(); + }); + self.server.on('upgrade', function (req, socket, head) { // Websocket connection self.websocketQueue.push({ req: req, socket: socket, head: head }); @@ -148,6 +154,11 @@ _.extend(HttpProxy.prototype, { }); self.websocketQueue = []; + _.each(self.connectQueue, function (c) { + c.socket.destroy(); + }); + self.connectQueue = []; + self.mode = "hold"; }, @@ -180,6 +191,15 @@ _.extend(HttpProxy.prototype, { target: targetUrl }); } + + while (self.connectQueue.length) { + if (self.mode !== "proxy") + break; + + var c = self.connectQueue.shift(); + runLog.log("Proxy request (connect): " + c.req.method + " " + c.req.url); + proxyConnectMethod(c.req, c.socket, c.head); + } }, // The proxy can be in one of three modes: @@ -193,4 +213,68 @@ _.extend(HttpProxy.prototype, { } }); + + +// This is what http-proxy does +// XXX: We should submit connect support upstream +var setupSocket = function(socket) { + socket.setTimeout(0); + socket.setNoDelay(true); + + socket.setKeepAlive(true, 0); + + return socket; +}; + + +var proxyConnectMethod = function (req, socket, options, head, server, clb) { + if (req.method !== 'CONNECT') { + socket.destroy(); + return true; + } + + var tokens = req.url.split(':'); + + if (tokens.length != 2) { + runLog.log("Bad request: " + req.url); + socket.destroy(); + return true; + } + + var host = tokens[0]; + var port = tokens[1]; + + if (port != 443) { + runLog.log("Blocking request to non-443 port: " + req.url); + socket.destroy(); + return true; + } + + setupSocket(socket); + + // XXX: Needed? + // if (head && head.length) socket.unshift(head); + + var net = require('net'); + var proxySocket = net.createConnection(port, host); + setupSocket(proxySocket); + + socket.on('error', function (err) { + runLog.log("Error on socket: " + err); + proxySocket.end(); + }); + proxySocket.on('error', function (err) { + runLog.log("Error on proxySocket: " + err); + socket.end(); + }); + + proxySocket.on('connect', function(connect) { + runLog.log("Connection established to " + host + ":" + port); + socket.write("HTTP/1.0 200 Connection established\n\n"); + socket.pipe(proxySocket); + proxySocket.pipe(socket); + }); +}; + + exports.HttpProxy = HttpProxy; From 1e1a8f724bca4467b76d3557885bc932d68dcae4 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Mon, 8 Sep 2014 15:58:26 -0700 Subject: [PATCH 03/19] Automatically use proxy on cordova Doesn't seem to automatically apply the settings though --- .../boilerplate_web.cordova.html | 4 +-- tools/commands-cordova.js | 9 +++++- tools/commands.js | 32 ++++++++++++++----- tools/run-httpproxy.js | 2 +- 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/packages/boilerplate-generator/boilerplate_web.cordova.html b/packages/boilerplate-generator/boilerplate_web.cordova.html index 6cf5ceeafe..5de2c5975e 100644 --- a/packages/boilerplate-generator/boilerplate_web.cordova.html +++ b/packages/boilerplate-generator/boilerplate_web.cordova.html @@ -11,8 +11,8 @@ if (/Android/i.test(navigator.userAgent)) { // When Android app is emulated, it cannot connect to localhost, // instead it should connect to 10.0.2.2 - __meteor_runtime_config__.ROOT_URL = (__meteor_runtime_config__.ROOT_URL || '').replace(/localhost/i, '10.0.2.2'); - __meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL = (__meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL || '').replace(/localhost/i, '10.0.2.2'); + //__meteor_runtime_config__.ROOT_URL = (__meteor_runtime_config__.ROOT_URL || '').replace(/localhost/i, '10.0.2.2'); + //__meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL = (__meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL || '').replace(/localhost/i, '10.0.2.2'); } __meteor_manifest__ = {{meteorManifest}}; diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index 1e2740bb24..537a3d12a5 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -634,9 +634,16 @@ var execCordovaOnPlatform = function (localPath, platformName, options) { 'platforms', 'ios', '*.xcodeproj')]); } else { verboseLog('Running emulator:', localCordova, args); + var emulatorOptions = { verbose: options.verbose, cwd: cordovaPath }; + emulatorOptions.env = {}; + if (options.httpProxyPort) { + // XXX: Is this Android only? + // XXX: Is 10.0.2.2 always the IP? + emulatorOptions.env['http_proxy'] = '10.0.2.2:' + options.httpProxyPort; + } execFileAsyncOrThrow( localCordova, args, - { verbose: options.verbose, cwd: cordovaPath }); + emulatorOptions); } var Log = getLoadedPackages().logging.Log; diff --git a/tools/commands.js b/tools/commands.js index 28e10c2066..80beaf8374 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -200,6 +200,8 @@ main.registerCommand({ return 1; } + options.httpProxyPort = options['http-proxy-port']; + // If we are targeting the remote devices if (_.intersection(options.args, ['ios-device', 'android-device']).length) { cordova.verboseLog('A run on a device requested'); @@ -218,11 +220,16 @@ main.registerCommand({ if (options.args.length) { // will asynchronously start mobile emulators/devices try { - // --clean encpasulates the behavior of once + // --clean encapsulates the behavior of once if (options.clean) { options.once = true; } + if (!options.httpProxyPort) { + console.log('Forcing http proxy on port 3002 for mobile'); + options.httpProxyPort = '3002'; + } + cordova.verboseLog('Will compile mobile builds'); var appName = path.basename(options.appDir); var localPath = path.join(options.appDir, '.meteor', 'local'); @@ -279,7 +286,7 @@ main.registerCommand({ return runAll.run(options.appDir, { proxyPort: parsedHostPort.port, proxyHost: parsedHostPort.host, - httpProxyPort: options['http-proxy-port'], + httpProxyPort: options.httpProxyPort, appPort: appPort, appHost: appHost, settingsFile: options.settings, @@ -1154,6 +1161,8 @@ main.registerCommand({ return 1; } + options.httpProxyPort = options['http-proxy-port']; + // XXX not good to change the options this way _.extend(options, parsedHostPort); @@ -1199,6 +1208,11 @@ main.registerCommand({ }); if (! _.isEmpty(mobilePlatforms)) { + if (!options.httpProxyPort) { + console.log('Forcing http proxy on port 3002 for mobile'); + options.httpProxyPort = '3002' + } + var localPath = path.join(testRunnerAppDir, '.meteor', 'local'); var platforms = @@ -1209,11 +1223,13 @@ main.registerCommand({ project.addCordovaPlatforms(platforms); try { - cordova.buildPlatforms(localPath, mobilePlatforms, - _.extend({}, options, { - appName: path.basename(testRunnerAppDir), - debug: ! options.production - })); + var cordovaOptions = _.extend({}, options, { + appName: path.basename(testRunnerAppDir), + debug: ! options.production + }); + + cordova.buildPlatforms(localPath, mobilePlatforms, cordovaOptions); + cordova.runPlatforms(localPath, mobilePlatforms, options); } catch (err) { process.stderr.write(err.message + '\n'); @@ -1397,7 +1413,7 @@ var runTestAppForPackages = function (testPackages, testRunnerAppDir, options) { // a switch to a different release appDirForVersionCheck: options.appDir, proxyPort: options.port, - httpProxyPort: options['http-proxy-port'], + httpProxyPort: options.httpProxyPort, disableOplog: options['disable-oplog'], settingsFile: options.settings, banner: "Tests", diff --git a/tools/run-httpproxy.js b/tools/run-httpproxy.js index 175357dea1..e63c05fe34 100644 --- a/tools/run-httpproxy.js +++ b/tools/run-httpproxy.js @@ -7,7 +7,7 @@ var Future = require('fibers/future'); var runLog = require('./run-log.js'); var url = require('url'); -// options: listenPort, proxyToPort, proxyToHost, onFailure +// options: listenPort, listenHost, onFailure var HttpProxy = function (options) { var self = this; From 4c4317badf264889f7c1b1a84e95f15b8c399454 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 07:14:58 -0700 Subject: [PATCH 04/19] Use a more common mirror for apache ant download --- scripts/generate-android-bundle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate-android-bundle.sh b/scripts/generate-android-bundle.sh index e6e51d6a59..91c59c863a 100755 --- a/scripts/generate-android-bundle.sh +++ b/scripts/generate-android-bundle.sh @@ -46,7 +46,7 @@ else fi { - curl -O http://www.motorlogy.com/apache//ant/binaries/apache-ant-1.9.4-bin.tar.gz + curl -O http://apache.osuosl.org/ant/binaries/apache-ant-1.9.4-bin.tar.gz tar xzf apache-ant-1.9.4-bin.tar.gz rm apache-ant-1.9.4-bin.tar.gz From 05b84fcc5e7ca8dc0004963d37fc5caf4414eda0 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 07:26:20 -0700 Subject: [PATCH 05/19] Create avd with x86 (ARM cannot be accelerated) --- scripts/generate-android-bundle.sh | 5 ++++- tools/cordova-scripts/ensure_android_bundle.sh | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/scripts/generate-android-bundle.sh b/scripts/generate-android-bundle.sh index 91c59c863a..7486deeca8 100755 --- a/scripts/generate-android-bundle.sh +++ b/scripts/generate-android-bundle.sh @@ -58,9 +58,12 @@ fi # the platform that cordova likes echo y | android-sdk/tools/android update sdk -t android-19 -u - # system image for android 19 + # system image for android 19 - arm echo y | android-sdk/tools/android update sdk -t sys-img-armeabi-v7a-android-19 --all -u + # system image for android 19 - x86 + echo y | android-sdk/tools/android update sdk -t sys-img-x86-android-19 --all -u + # build tools echo y | android-sdk/tools/android update sdk -t "build-tools-20.0.0" -u diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index 97d7f792c2..1d8ebe30ec 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -95,8 +95,11 @@ command -v javac >/dev/null 2>&1 || { # create avd if necessary if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] ; then + #ABI="default/armeabi-v7a" + ABI="default/x86" + echo " -" | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi default/armeabi-v7a --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2 +" | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi ${ABI} --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2 fi From 9ebd90bddc7cd3061632d3e5e108e18786a63987 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 07:55:37 -0700 Subject: [PATCH 06/19] Start cordova using the runner infrastructure Cleaner, but also we need to start the http proxy first --- tools/commands-cordova.js | 47 ++++++++++++++++++++++++++++++++------- tools/commands.js | 15 +++++++++---- tools/run-all.js | 16 +++++++++++++ 3 files changed, 66 insertions(+), 12 deletions(-) diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index 1e2740bb24..174fa7ccb3 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -605,6 +605,44 @@ cordova.buildPlatforms = function (localPath, platforms, options) { buildCordova(localPath, 'build', options); }; + +cordova.buildPlatformRunners = function (localPath, platforms, options) { + var runners = []; + _.each(platforms, function (platformName) { + runners.push(new CordovaRunner(localPath, platformName, options)); + }); + return runners; +}; + + +// This is a runner, that we pass to Runner (run-all.js) +var CordovaRunner = function (localPath, platformName, options) { + var self = this; + + self.localPath = localPath; + self.platformName = platformName; + self.options = options; + + self.title = 'Cordova (' + self.platformName + ')'; +}; + +_.extend(CordovaRunner.prototype, { + start: function () { + var self = this; + + execCordovaOnPlatform(self.localPath, + self.platformName, + self.options); + }, + + stop: function () { + var self = this; + + // XXX: A no-op for now (we leave it running because it's slow!) + } +}); + + // Start the simulator or physical device for a specific platform. // platformName is of the form ios/ios-device/android/android-device // options: @@ -722,14 +760,7 @@ var execCordovaOnPlatform = function (localPath, platformName, options) { return 0; }; -// Start the simulator or physical device for a list of platforms -// options: -// - verbose: print all build logs -cordova.runPlatforms = function (localPath, platforms, options) { - _.each(platforms, function (platformName) { - execCordovaOnPlatform(localPath, platformName, options); - }); -}; + // packages - list of strings cordova.filterPackages = function (packages) { diff --git a/tools/commands.js b/tools/commands.js index 28e10c2066..8d6192e09f 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -214,6 +214,8 @@ main.registerCommand({ // Always bundle for the browser by default. var webArchs = project.getWebArchs(); + var runners = []; + // If additional args were specified, then also start a mobile build. if (options.args.length) { // will asynchronously start mobile emulators/devices @@ -230,7 +232,7 @@ main.registerCommand({ cordova.buildPlatforms(localPath, options.args, _.extend({ appName: appName, debug: ! options.production }, options, parsedHostPort)); - cordova.runPlatforms(localPath, options.args, options); + runners = runners.concat(cordova.buildPlatformRunners(localPath, options.args, options)); } catch (err) { if (options.verbose) { process.stderr.write('Error while running for mobile platforms ' + @@ -291,7 +293,8 @@ main.registerCommand({ rootUrl: process.env.ROOT_URL, mongoUrl: process.env.MONGO_URL, oplogUrl: process.env.MONGO_OPLOG_URL, - once: options.once + once: options.once, + extraRunners: runners }); }); @@ -1190,6 +1193,8 @@ main.registerCommand({ [options['driver-package'] || 'test-in-browser'], 'add'); + var runners = []; + var mobileOptions = ['ios', 'ios-device', 'android', 'android-device']; var mobilePlatforms = []; @@ -1214,13 +1219,14 @@ main.registerCommand({ appName: path.basename(testRunnerAppDir), debug: ! options.production })); - cordova.runPlatforms(localPath, mobilePlatforms, options); + runners = runners.concat(cordova.buildPlatformRunners(localPath, mobilePlatforms, options)); } catch (err) { process.stderr.write(err.message + '\n'); return 1; } } + options.extraRunners = runners; return runTestAppForPackages(testPackages, testRunnerAppDir, options); }); @@ -1406,7 +1412,8 @@ var runTestAppForPackages = function (testPackages, testRunnerAppDir, options) { mongoUrl: process.env.MONGO_URL, oplogUrl: process.env.MONGO_OPLOG_URL, once: options.once, - recordPackageUsage: false + recordPackageUsage: false, + extraRunners: options.extraRunners }); } diff --git a/tools/run-all.js b/tools/run-all.js index 79d0c162bd..f20dce0d3e 100644 --- a/tools/run-all.js +++ b/tools/run-all.js @@ -37,6 +37,8 @@ var Runner = function (appDir, options) { self.rootUrl = 'http://localhost:' + listenPort + '/'; } + self.extraRunners = options.extraRunners; + self.proxy = new Proxy({ listenPort: listenPort, listenHost: options.proxyHost, @@ -152,6 +154,17 @@ _.extend(Runner.prototype, { } } + _.forEach(self.extraRunners, function (extraRunner) { + if (! self.stopped) { + var title = extraRunner.title; + if (! self.quiet) + runLog.logTemporary("=> Starting " + title + "..."); + self.extraRunner.start(); + if (! self.quiet && ! self.stopped) + runLog.log("=> Started " + title + "."); + } + }); + if (! self.stopped) { if (! self.quiet) runLog.logTemporary("=> Starting your app..."); @@ -180,6 +193,9 @@ _.extend(Runner.prototype, { self.httpProxy && self.httpProxy.stop(); self.updater.stop(); self.mongoRunner && self.mongoRunner.stop(); + _.forEach(self.extraRunners, function (extraRunner) { + extraRunner.stop(); + }); self.appRunner.stop(); // XXX does calling this 'finish' still make sense now that runLog is a // singleton? From 36db11a63d4def984b9ed5845ded6411701935ba Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 08:12:29 -0700 Subject: [PATCH 07/19] Fix 'typo' --- tools/run-all.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/run-all.js b/tools/run-all.js index f20dce0d3e..acd6bb9477 100644 --- a/tools/run-all.js +++ b/tools/run-all.js @@ -159,7 +159,7 @@ _.extend(Runner.prototype, { var title = extraRunner.title; if (! self.quiet) runLog.logTemporary("=> Starting " + title + "..."); - self.extraRunner.start(); + extraRunner.start(); if (! self.quiet && ! self.stopped) runLog.log("=> Started " + title + "."); } From 742603f9499b89d068e2e52e2f9da5cb3fa7a717 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 08:19:15 -0700 Subject: [PATCH 08/19] Fix env logic, so we don't kill the user's PATH --- tools/commands-cordova.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index 9302fc0a4b..f432079aac 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -57,7 +57,11 @@ var execFileAsyncOrThrow = function (file, args, opts, cb) { } // XXX a hack to always tell the scripts where warehouse is - if (opts) opts.env = _.extend({ "WAREHOUSE_DIR": tropo.root, "USE_GLOBAL_ADK": process.env.USE_GLOBAL_ADK || "", HOME: process.env.HOME }, opts.env); + opts = opts || {}; + opts.env = _.extend({ "USE_GLOBAL_ADK": "" }, + process.env, + opts.env || {}, + { "WAREHOUSE_DIR": tropo.root }); var execFileAsync = require('./utils.js').execFileAsync; if (_.contains([localCordova, localAdb, localAndroid], file) && @@ -87,7 +91,11 @@ var execFileSyncOrThrow = function (file, args, opts) { verboseLog('Running synchronously: ', file, args); // XXX a hack to always tell the scripts where warehouse is - if (opts) opts.env = _.extend({ "WAREHOUSE_DIR": tropo.root, "USE_GLOBAL_ADK": process.env.USE_GLOBAL_ADK || "", HOME: process.env.HOME }, opts.env); + opts = opts || {}; + opts.env = _.extend({ "USE_GLOBAL_ADK": "" }, + process.env, + opts.env || {}, + { "WAREHOUSE_DIR": tropo.root }); var childProcess = execFileSync(file, args, opts); if (! childProcess.success) From 452da56297390cbc37d8c2a810ff097fee61ba08 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 08:23:17 -0700 Subject: [PATCH 09/19] Allow passing of arguments to configure-android Args are passed through to the android tool Useful commands: configure-android avd configure-android list avd configure-android -- delete avd --name meteor (Note the extra -- to delineate the args) (Note #2: the delete avd command doesn't seem to actually work, though this appears to be an Android bug - it says that is has deleted it, but it is still there!) --- tools/commands-cordova.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index f432079aac..0e35200f7d 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -878,8 +878,14 @@ main.registerCommand({ name: "configure-android", options: { verbose: { type: Boolean, short: "v" } - } + }, + minArgs: 0, + maxArgs: Infinity }, function (options) { - return execFileSyncOrThrow(localAndroid, [], options); + cordova.setVerboseness(options.verbose); + + var androidArgs = options.args || []; + var execOptions = { pipeOutput: true, verbose: options.verbose }; + execFileSyncOrThrow(localAndroid, androidArgs, execOptions); }); From 801b9648155e7aa3f9c7787328d35a95e881f312 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 12:34:07 -0700 Subject: [PATCH 10/19] Set image options to look a bit like a Nexus 4 Also fixes keyboard support --- .../cordova-scripts/ensure_android_bundle.sh | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index 1d8ebe30ec..0a06b3dcce 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -92,6 +92,17 @@ command -v javac >/dev/null 2>&1 || { echo >&2 "To add the android platform, please install a JDK. Here are some directions: http://openjdk.java.net/install/"; exit 1; } +set_config () { + KEY=$1 + VALUE=$2 + + CONFIG_FILE=${ANDROID_BUNDLE}/meteor_avd/config.ini + + TEMP_FILE=`mktemp` + grep -v "^${KEY}=" ${CONFIG_FILE} > ${TEMP_FILE} + echo "${KEY}=${VALUE}" >> ${TEMP_FILE} + mv -f ${TEMP_FILE} ${CONFIG_FILE} +} # create avd if necessary if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] ; then @@ -100,6 +111,21 @@ if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] echo " " | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi ${ABI} --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2 + + # Nice keyboard support + set_config "hw.keyboard" "yes" + set_config "hw.mainKeys" "no" + + # Look like a nexus 4, but with 1GB not 2GB (so we don't kill a 4GB machine) + set_config "hw.ramSize" "1024" + set_config "vm.heapSize" "64" + set_config "skin.dynamic" "yes" + set_config "skin.name" "768x1280" + set_config "skin.path" "768x1280" + set_config "hw.lcd.density" "7320" + + # XXX: hw.gpu.enabled=yes ? + fi From d7824753580f323c483cd7c9960fa415d8c0f9ae Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 12:35:05 -0700 Subject: [PATCH 11/19] Fix typo --- tools/cordova-scripts/ensure_android_bundle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index 0a06b3dcce..ca19d58107 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -122,7 +122,7 @@ if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] set_config "skin.dynamic" "yes" set_config "skin.name" "768x1280" set_config "skin.path" "768x1280" - set_config "hw.lcd.density" "7320" + set_config "hw.lcd.density" "320" # XXX: hw.gpu.enabled=yes ? From 5a5888ec52036054c71dbdc3939f655239a9fe9c Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 12:55:16 -0700 Subject: [PATCH 12/19] Don't use a huge screen by default after all; it'll be slow --- tools/cordova-scripts/ensure_android_bundle.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index ca19d58107..f33b5dce44 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -116,13 +116,16 @@ if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] set_config "hw.keyboard" "yes" set_config "hw.mainKeys" "no" - # Look like a nexus 4, but with 1GB not 2GB (so we don't kill a 4GB machine) + # More RAM than the default set_config "hw.ramSize" "1024" set_config "vm.heapSize" "64" - set_config "skin.dynamic" "yes" - set_config "skin.name" "768x1280" - set_config "skin.path" "768x1280" - set_config "hw.lcd.density" "320" + + # These are the settings for a Nexus 4, but it's a bit big for some screens + # (and likely a bit slow without GPU & KVM/HAXM acceleration) + #set_config "skin.dynamic" "yes" + #set_config "hw.lcd.density" "320" + #set_config "hw.device.name" "Nexus 4" + #set_config "hw.device.manufacturer" "Google" # XXX: hw.gpu.enabled=yes ? From 1103261918c51d7e14ab0b5575713d1cf6aa2dd1 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 12:59:42 -0700 Subject: [PATCH 13/19] If verbose, pass --verbose to cordova run --- tools/commands-cordova.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index 0e35200f7d..ca55778d33 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -666,9 +666,12 @@ var execCordovaOnPlatform = function (localPath, platformName, options) { verboseLog('isDevice:', isDevice); - var args = [ 'run', - isDevice ? '--device' : '--emulator', - platform ]; + var args = [ 'run' ]; + if (options.verbose) { + args.push('--verbose'); + } + args.push(isDevice ? '--device' : '--emulator'); + args.push(platform); // XXX assert we have a valid Cordova project if (platform === 'ios' && isDevice) { From 2fc07ee80194944e00ee01aec4cf497ba4fb082e Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:02:32 -0700 Subject: [PATCH 14/19] Pass http_proxy to the emulator if the httpProxyPort is set It doesn't work for Cordova, though it does work for the browser This may get fixed later --- tools/commands-cordova.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/commands-cordova.js b/tools/commands-cordova.js index ca55778d33..03ccee4d49 100644 --- a/tools/commands-cordova.js +++ b/tools/commands-cordova.js @@ -684,11 +684,11 @@ var execCordovaOnPlatform = function (localPath, platformName, options) { } else { verboseLog('Running emulator:', localCordova, args); var emulatorOptions = { verbose: options.verbose, cwd: cordovaPath }; - emulatorOptions.env = {}; + emulatorOptions.env = _.extend({}, process.env); if (options.httpProxyPort) { // XXX: Is this Android only? - // XXX: Is 10.0.2.2 always the IP? - emulatorOptions.env['http_proxy'] = '10.0.2.2:' + options.httpProxyPort; + // This is odd; the IP address is on the host, not inside the emulator + emulatorOptions.env['http_proxy'] = '127.0.0.1:' + options.httpProxyPort; } execFileAsyncOrThrow( localCordova, args, From a1ec61dec01e6efa20ed1c792fe48dd72d83c130 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:06:58 -0700 Subject: [PATCH 15/19] Don't override ANDROID_SDK_HOME when running with system ADT --- tools/cordova-scripts/common_env.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/cordova-scripts/common_env.sh b/tools/cordova-scripts/common_env.sh index c845a699e6..3d0d008a78 100644 --- a/tools/cordova-scripts/common_env.sh +++ b/tools/cordova-scripts/common_env.sh @@ -47,9 +47,8 @@ if [ -z "$USE_GLOBAL_ADK" ] ; then export HOME="${ANDROID_BUNDLE}" export ANDROID_SDK_HOME="${ANDROID_BUNDLE}" else - # to use a global ADK we don't set PATH, ANT_HOME + # to use a global ADK we don't set PATH, ANT_HOME, ANDROID_SDK_HOME # relying that they are installed and available globally - export ANDROID_SDK_HOME="${HOME}" true fi From ffc69aa0105c329c62e88a2d1d1ece4d00618775 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:17:53 -0700 Subject: [PATCH 16/19] Super-simple script to set android proxy For discussion / demo really --- tools/cordova-scripts/set_android_proxy.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 tools/cordova-scripts/set_android_proxy.sh diff --git a/tools/cordova-scripts/set_android_proxy.sh b/tools/cordova-scripts/set_android_proxy.sh new file mode 100755 index 0000000000..a8a1b8f07d --- /dev/null +++ b/tools/cordova-scripts/set_android_proxy.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +# import all the environment +source $(dirname $0)/common_env.sh + +adb shell sqlite3 /data/data/com.android.providers.telephony/databases/telephony.db \ + "update carriers set proxy='10.0.2.2', port='3002' where current=1" +adb shell stop +adb shell start From 8c1b8154fe3bc30af69918d89e481c76a23304f1 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:58:37 -0700 Subject: [PATCH 17/19] Turn on error checking in ensure_android_bundle.sh --- tools/cordova-scripts/ensure_android_bundle.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index f33b5dce44..bb410db57e 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -1,4 +1,7 @@ #!/bin/bash + +set -e + BUNDLE_VERSION=0.1 # OS Check. Put here because here is where we download the precompiled From 04e5bc700c3dee7fe5a5229c3137a7f552e7f475 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:58:54 -0700 Subject: [PATCH 18/19] Pass a template for mktemp Mac requires one --- tools/cordova-scripts/ensure_android_bundle.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index bb410db57e..7cf27510b9 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -101,7 +101,7 @@ set_config () { CONFIG_FILE=${ANDROID_BUNDLE}/meteor_avd/config.ini - TEMP_FILE=`mktemp` + TEMP_FILE=`mktemp tmp.XXXXXXXXXX` grep -v "^${KEY}=" ${CONFIG_FILE} > ${TEMP_FILE} echo "${KEY}=${VALUE}" >> ${TEMP_FILE} mv -f ${TEMP_FILE} ${CONFIG_FILE} From b504fa8afaaff5020b1ddba5203377d95feafa78 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 9 Sep 2014 13:59:10 -0700 Subject: [PATCH 19/19] Auto-install the x86 ABI if it isn't installed This could be big actually; it means we don't need to bundle it --- tools/cordova-scripts/ensure_android_bundle.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/cordova-scripts/ensure_android_bundle.sh b/tools/cordova-scripts/ensure_android_bundle.sh index 7cf27510b9..2f07597814 100755 --- a/tools/cordova-scripts/ensure_android_bundle.sh +++ b/tools/cordova-scripts/ensure_android_bundle.sh @@ -107,11 +107,20 @@ set_config () { mv -f ${TEMP_FILE} ${CONFIG_FILE} } +install_x86 () { + echo "Android x86 System image not found. Found targets:" + android list target + echo "Downloading x86 system image..." + echo y | android update sdk -t sys-img-x86-android-19 --all -u +} + # create avd if necessary if [[ ! $("${ANDROID_BUNDLE}/android-sdk/tools/android" list avd | grep Name) ]] ; then #ABI="default/armeabi-v7a" ABI="default/x86" + (android list target | grep ABIs | grep default/x86 > /dev/null) || install_x86 + echo " " | "${ANDROID_BUNDLE}/android-sdk/tools/android" create avd --target 1 --name meteor --abi ${ABI} --path ${ANDROID_BUNDLE}/meteor_avd/ 1>&2