From 830acf651c2cd71eacbaf8a1d5ad30d6911adddd Mon Sep 17 00:00:00 2001 From: Ben Newman Date: Mon, 13 Oct 2014 17:24:10 -0400 Subject: [PATCH] Implement a `meteor shell` command. Summary: Invoking `meteor shell` starts an interactive REPL for evaluating server-side code. Shell commands are evaluated in the server process, so the shell's input and output have to be piped through a socket from/to the parent process that runs the `meteor shell` command. This separation accounts for most of the complexity of this diff, but the end result is pretty seamless: tab completion, history, ctlr-c and ctrl-d, etc. all work as expected. Task: https://app.asana.com/0/15750483766338/16576093991355 Test Plan: Run `meteor run` in one terminal and `meteor shell` in another, evaluate some commands, and verify that the commands seem to be running the context of the server process. Also check tab completion and history, and ensure that the server is running simultaneously with the shell. Reviewers: dgreenspan, avital, slava, emily, nim, sashko CC: sashko Differential Revision: https://phabricator.meteor.com/D837 --- tools/bundler.js | 7 +- tools/commands.js | 18 +++ tools/help.txt | 27 ++++ tools/run-app.js | 15 +- tools/server/boot.js | 4 + tools/server/shell.js | 354 ++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 tools/server/shell.js diff --git a/tools/bundler.js b/tools/bundler.js index ea2ec845ee..0aa2997db9 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1630,6 +1630,8 @@ _.extend(ServerTarget.prototype, { // Server bootstrap builder.write('boot.js', { file: path.join(__dirname, 'server', 'boot.js') }); + builder.write('shell.js', + { file: path.join(__dirname, 'server', 'shell.js') }); // Script that fetches the dev_bundle and runs the server bootstrap var archToPlatform = { @@ -1755,8 +1757,9 @@ var writeSiteArchive = function (targets, outputPath, options) { // Affordances for standalone use if (targets.server) { // add program.json as the first argument after "node main.js" to the boot script. - var stub = new Buffer(exports._mainJsContents, 'utf8'); - builder.write('main.js', { data: stub }); + builder.write('main.js', { + data: new Buffer(exports._mainJsContents, 'utf8') + }); builder.write('README', { data: new Buffer( "This is a Meteor application bundle. It has only one external dependency:\n" + diff --git a/tools/commands.js b/tools/commands.js index f9bc9481a7..a4efcafe14 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -372,6 +372,24 @@ main.registerCommand(_.extend( return doRunCommand(options); }); +/////////////////////////////////////////////////////////////////////////////// +// shell +/////////////////////////////////////////////////////////////////////////////// + +main.registerCommand({ + name: 'shell', + catalogRefresh: new catalog.Refresh.Never() +}, function (options) { + if (!options.appDir) { + Console.stderr.write( + "The 'meteor shell' command must be run in a Meteor app directory." + ); + } else { + require('./server/shell.js').connect(options.appDir); + throw new main.WaitForExit; + } +}); + /////////////////////////////////////////////////////////////////////////////// // create /////////////////////////////////////////////////////////////////////////////// diff --git a/tools/help.txt b/tools/help.txt index 24fb3e85f2..9f48446d5e 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -82,6 +82,7 @@ Options: for incoming connections from debugging clients, such as node-inspector (default 5858). + >>> create Create a new project. Usage: meteor create [--release ] @@ -273,6 +274,32 @@ Options: Defaults to localhost:3000. Can include a URL scheme (for example, --server=https://example.com:443). + +>>> shell +Launch a Node REPL for interactively evaluating server-side code. + +Usage: meteor shell + +When `meteor shell` is executed in an application directory where a server +is already running, it connects to the server and starts an interactive +shell for evaluating server-side code. + +Multiple shells can be attached to the same server. If no server is +currently available, `meteor shell` will keep trying to connect until it +succeeds. + +Exiting the shell does not terminate the server. If the server restarts +because a change was made in server code, or a fatal exception was +encountered, the shell will restart along with the server. This behavior +can be simulated by typing `.reload` in the shell. + +The shell supports tab completion for global variables like `Meteor`, +`Mongo`, and `Package`. Try typing `Meteor.is` and then pressing tab. + +The shell maintains a persistent history across sessions. Previously-run +commands can be accessed by pressing the up arrow. + + >>> mongo Connect to the Mongo database for the specified site. Usage: meteor mongo [--url] [site] diff --git a/tools/run-app.js b/tools/run-app.js index 20ba304dce..d9dce0201e 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -52,6 +52,7 @@ var getNodeOptionsFromEnvironment = function () { var AppProcess = function (options) { var self = this; + self.appDir = options.appDir; self.bundlePath = options.bundlePath; self.port = options.port; self.listenHost = options.listenHost; @@ -96,6 +97,7 @@ _.extend(AppProcess.prototype, { // This is the child process telling us that it's ready to // receive connections. self.onListen && self.onListen(); + } else { runLog.logAppOutput(line); } @@ -133,6 +135,13 @@ _.extend(AppProcess.prototype, { // exception and the whole app dies. // http://stackoverflow.com/questions/2893458/uncatchable-errors-in-node-js self.proc.stdin.on('error', function () {}); + + // When the parent process exits (i.e. the server is shutting down and + // not merely restarting), make sure to disconnect any still-connected + // shell clients. + require("./cleanup.js").onExit(function() { + require("./server/shell.js").unlinkSocketFile(self.appDir); + }); }, _maybeCallOnExit: function (code, signal) { @@ -193,6 +202,8 @@ _.extend(AppProcess.prototype, { env.HTTP_FORWARDED_COUNT = "" + ((parseInt(process.env['HTTP_FORWARDED_COUNT']) || 0) + 1); + env.ENABLE_METEOR_SHELL = 'true'; + return env; }, @@ -207,13 +218,14 @@ _.extend(AppProcess.prototype, { if (! self.program) { // Old-style bundle var opts = _.clone(self.nodeOptions); + if (self.debugPort) { require("./inspector.js").start(self.debugPort); opts.push("--debug-brk=" + self.debugPort); } - opts.push(path.join(self.bundlePath, 'main.js')); opts.push( + path.join(self.bundlePath, 'main.js'), '--parent-pid', process.env.METEOR_BAD_PARENT_PID_FOR_TEST ? "foobar" : process.pid ); @@ -561,6 +573,7 @@ _.extend(AppRunner.prototype, { // Run the program options.beforeRun && options.beforeRun(); var appProcess = new AppProcess({ + appDir: self.appDir, bundlePath: bundlePath, port: self.port, listenHost: self.listenHost, diff --git a/tools/server/boot.js b/tools/server/boot.js index 16d0e3a01c..3ff572f350 100644 --- a/tools/server/boot.js +++ b/tools/server/boot.js @@ -77,6 +77,10 @@ sourcemap_support.install({ handleUncaughtExceptions: false }); +// Only enabled by default in development. +if (process.env.ENABLE_METEOR_SHELL) { + require('./shell.js').listen(); +} Fiber(function () { _.each(serverJson.load, function (fileInfo) { diff --git a/tools/server/shell.js b/tools/server/shell.js new file mode 100644 index 0000000000..f09ccea68d --- /dev/null +++ b/tools/server/shell.js @@ -0,0 +1,354 @@ +var assert = require("assert"); +var path = require("path"); +var stream = require("stream"); +var fs = require("fs"); +var net = require("net"); +var tty = require("tty"); +var vm = require("vm"); +var Fiber = require("fibers"); +var eachline = require("eachline"); +var chalk = require("chalk"); +var EOL = require("os").EOL; +var _ = require("underscore"); +var EXITING_MESSAGE = "Shell exiting..."; + +// Invoked by the server process to listen for incoming connections from +// shell clients. Each connection gets its own REPL instance. +exports.listen = function listen() { + var socketFile = getSocketFile(); + fs.unlink(socketFile, function() { + net.createServer(onConnection).listen(socketFile); + }); +}; + +function onConnection(socket) { + var dataSoFar = ""; + + // Let connecting clients configure certain REPL options by sending a + // JSON object over the socket. For example, only the client knows + // whether it's running a TTY or an Emacs subshell or some other kind of + // terminal, so the client must decide the value of options.terminal. + socket.on("data", function onData(buffer) { + // Just in case the options JSON comes in fragments. + dataSoFar += buffer.toString("utf8"); + + try { + var options = JSON.parse(dataSoFar); + } finally { + if (! _.isObject(options)) { + return; // Silence any parsing exceptions. + } + } + + socket.removeListener("data", onData); + + // Immutable options. + _.extend(options, { + input: socket, + output: socket, + eval: evalCommand + }); + + // Overridable options. + _.defaults(options, { + prompt: "> ", + terminal: true, + useColors: true, + useGlobal: true, + ignoreUndefined: true, + }); + + startREPL(options); + }); +} + +// The child process calls this function when it receives the SHELLSTART +// command from the parent process (via stdin). +function startREPL(options) { + if (! options.output.columns) { + // The REPL's tab completion logic assumes process.stdout is a TTY, + // and while that isn't technically true here, we can get tab + // completion to behave correctly if we fake the .columns property. + options.output.columns = getTerminalWidth(); + } + + var repl = require("repl").start(options); + + // History persists across shell sessions! + initializeHistory(repl); + + // Use the same `require` function and `module` object visible to the + // shell.js module. + repl.context.require = require; + repl.context.module = module; + repl.context.repl = repl; + + // Some improvements to the existing help messages. + repl.commands[".break"].help = + "Terminate current command input and display new prompt"; + repl.commands[".exit"].help = "Disconnect from server and leave shell"; + repl.commands[".help"].help = "Show this help information"; + + // When the REPL exits, signal the attached client to exit by sending it + // the special EXITING_MESSAGE. + repl.on("exit", function() { + options.output.write(EXITING_MESSAGE + "\n"); + options.output.end(); + }); + + // When the server process exits, end the output stream but do not + // signal the attached client to exit. + process.on("exit", function() { + options.output.end(); + }); + + // This Meteor-specific shell command rebuilds the application as if a + // change was made to server code. + repl.defineCommand("reload", { + help: "Restart the server and the shell", + action: function() { + process.exit(0); + } + }); +} + +function getSocketFile(appDir) { + return path.join(appDir || getAppDir(), ".meteor", "local", "shell.sock"); +} +exports.getSocketFile = getSocketFile; + +// Unlinking the socket file causes all attached shell clients to +// disconnect and exit. +exports.unlinkSocketFile = function(appDir) { + var socketFile = getSocketFile(appDir); + try { + fs.unlinkSync(socketFile); + } finally { + // Replace the socket file with a regular file so that any connected + // shell clients will fail to connect with the ENOTSOCK error. + fs.writeFileSync(socketFile, "not a socket\n"); + return; // Silence any exception from fs.unlinkSync. + } +}; + +function getHistoryFile(appDir) { + return path.join( + appDir || getAppDir(), + ".meteor", "local", "shell-history" + ); +} + +function getAppDir() { + for (var dir = __dirname, nextDir; + path.basename(dir) !== ".meteor"; + dir = nextDir) { + nextDir = path.dirname(dir); + if (dir === nextDir) { + throw new Error("Not a Meteor app directory"); + } + } + return path.dirname(dir); +} + +function getTerminalWidth() { + try { + // Inspired by https://github.com/TooTallNate/ttys/blob/master/index.js + var fd = fs.openSync("/dev/tty", "r"); + assert.ok(tty.isatty(fd)); + var ws = new tty.WriteStream(fd); + ws.end(); + return ws.columns; + } catch (fancyApproachWasTooFancy) { + return 80; + } +} + +// Shell commands need to be executed in fibers in case they call into +// code that yields. +function evalCommand(command, context, filename, callback) { + Fiber(function() { + try { + var result = vm.runInThisContext(command, filename); + } catch (error) { + if (process.domain) { + process.domain.emit("error", error); + process.domain.exit(); + } else { + callback(error); + } + return; + } + callback(null, result); + }).run(); +} + +// This function allows a persistent history of shell commands to be saved +// to and loaded from .meteor/local/shell-history. +function initializeHistory(repl) { + var rli = repl.rli; + var historyFile = getHistoryFile(); + var historyFd = fs.openSync(historyFile, "a+"); + var historyLines = fs.readFileSync(historyFile, "utf8").split(EOL); + var seenLines = Object.create(null); + + if (! rli.history) { + rli.history = []; + rli.historyIndex = -1; + } + + while (rli.history && historyLines.length > 0) { + var line = historyLines.pop(); + if (line && /\S/.test(line) && ! seenLines[line]) { + rli.history.push(line); + seenLines[line] = true; + } + } + + rli.addListener("line", function(line) { + if (historyFd >= 0 && /\S/.test(line)) { + fs.writeSync(historyFd, line + "\n"); + } + }); + + repl.on("exit", function() { + fs.closeSync(historyFd); + historyFd = -1; + }); +} + +// Invoked by the process running `meteor shell` to attempt to connect to +// the server via the socket file. +exports.connect = function(appDir) { + var socketFile = getSocketFile(appDir); + var exitOnClose = false; + var firstTimeConnecting = true; + var connected = false; + reconnect.count = 0; + + // We have to attach a "data" event even if we do nothing with the data + // in order to put the stream in "flowing mode." + function onData(buffer) {} + + function reconnect(delay) { + // Display the "Server unavailable" warning only on the third attempt + // to reconnect, so it doesn't get shown for successful reconnects. + if (++reconnect.count === 3) { + console.error(chalk.yellow( + "Server unavailable (waiting to reconnect)" + )); + } + + if (!reconnect.timer) { + reconnect.timer = setTimeout(function() { + delete reconnect.timer; + connect(); + }, delay || 100); + } + } + + function connect() { + if (connected) { + return; + } + + var sock = net.connect(socketFile); + + process.stdin.on("data", onData); + sock.pipe(process.stdout); + sock.on("connect", onConnect); + sock.on("close", onClose); + sock.on("error", onError); + + function onConnect() { + firstTimeConnecting = false; + reconnect.count = 0; + connected = true; + + sock.write(JSON.stringify({ + terminal: !process.env.EMACS + })); + + process.stderr.write(shellBanner()); + process.stdin.pipe(sock); + process.stdin.setRawMode(true); + } + + eachline(sock, "utf8", function(line) { + exitOnClose = line.indexOf(EXITING_MESSAGE) >= 0; + }); + + function onClose() { + tearDown(); + + // If we received the special EXITING_MESSAGE just before the socket + // closed, then exit the shell instead of reconnecting. + if (exitOnClose) { + process.exit(0); + } else { + reconnect(); + } + } + + function onError(err) { + tearDown(); + + if (err.errno === "ENOENT" || + err.errno === "ECONNREFUSED") { + // If the shell.sock file is missing or looks like a socket but is + // not accepting connections, keep trying to connect. + reconnect(); + + } else if (err.errno === "ENOTSOCK") { + // When the server shuts down completely, it replaces the + // shell.sock file with a regular file to force connected shell + // clients to disconnect and exit. If this shell client is + // connecting for the first time, however, assume the user intends + // to start the server again soon, and wait to reconnect. + if (firstTimeConnecting) { + reconnect(); + } else { + process.exit(0); + } + } + } + + function tearDown() { + connected = false; + process.stdin.unpipe(sock); + process.stdin.removeListener("data", onData); + process.stdin.setRawMode(false); + sock.unpipe(process.stdout); + sock.removeListener("connect", onConnect); + sock.removeListener("close", onClose); + sock.removeListener("error", onError); + sock.end(); + } + } + + connect(); +}; + +function shellBanner() { + var bannerLines = [ + "", + "Welcome to the server-side interactive shell!" + ]; + + if (! process.env.EMACS) { + // Tab completion sadly does not work in Emacs. + bannerLines.push( + "", + "Tab compeletion is enabled for global variables." + ); + } + + bannerLines.push( + "", + "Type .reload to restart the server and the shell.", + "Type .exit to teminate the server and the shell.", + "Type .help for additional help.", + EOL + ); + + return chalk.green(bannerLines.join(EOL)); +}