Progress bars and output formatting

scripts/dev-bundle-package.json

@@ -3,6 +3,8 @@
   "version": "0.0.0",
   "dependencies": {
     "fibers": "1.0.1",
+    "progress": "1.1.8",
+    "chalk": "0.5.1",
     "underscore": "1.5.2",
     "source-map-support": "0.2.5",
     "semver": "2.2.1"

tools/buildmessage.js

@@ -1,7 +1,10 @@
+var Fiber = require('fibers');
+var Future = require('fibers/future');
 var _ = require('underscore');
 var files = require('./files.js');
 var parseStack = require('./parse-stack.js');
 var fiberHelpers = require('./fiber-helpers.js');
+var Progress = require('./progress.js').Progress;
 
 var debugBuild = !!process.env.METEOR_DEBUG_BUILD;
 
@@ -154,9 +157,46 @@ _.extend(MessageSet.prototype, {
   }
 });
 
+// XXX: This is now a little bit silly... ideas:
+// Can we just have one hierarchical state?
+// Can we combined job & messageSet
+// Can we infer nesting level?
 var currentMessageSet = new fiberHelpers.EnvironmentVariable;
 var currentJob = new fiberHelpers.EnvironmentVariable;
 var currentNestingLevel = new fiberHelpers.EnvironmentVariable(0);
+var currentProgress = new fiberHelpers.EnvironmentVariable;
+
+var rootProgress = new Progress();
+
+var getRootProgress = function () {
+  return rootProgress;
+};
+
+var reportProgress = function (state) {
+  var progress = currentProgress.get();
+  if (progress) {
+    progress.reportProgress(state);
+  }
+};
+
+var reportProgressDone = function () {
+  var progress = currentProgress.get();
+  if (progress) {
+    progress.reportProgressDone();
+  }
+};
+
+var getCurrentProgressTracker = function () {
+  var progress = currentProgress.get();
+  return progress ? progress : rootProgress;
+};
+
+
+var addChildTracker = function (title) {
+  var options = {};
+  options.title = title;
+  return getCurrentProgressTracker().addChildTask(options);
+};
 
 // Create a new MessageSet, run `f` with that as the current
 // MessageSet for the purpose of accumulating and recovering from
@@ -169,24 +209,51 @@ var currentNestingLevel = new fiberHelpers.EnvironmentVariable(0);
 // `options`.
 var capture = function (options, f) {
   var messageSet = new MessageSet;
-  currentMessageSet.withValue(messageSet, function () {
+  var parentMessageSet = currentMessageSet.get();
-    var job = null;
-    if (typeof options === "object") {
-      job = new Job(options);
-      messageSet.jobs.push(job);
-    } else {
-      f = options; // options not actually provided
-    }
 
-    currentJob.withValue(job, function () {
+  {
-      var nestingLevel = currentNestingLevel.get();
+    var parentProgress = currentProgress.get();
-      currentNestingLevel.withValue(nestingLevel + 1, function () {
+    if (!parentProgress) {
-        debugBuild && console.log("START CAPTURE", nestingLevel, options.title);
+      parentProgress = rootProgress;
-        try {
+    }
-          f();
+    var childOptions = {};
-        } finally {
+    if (typeof options === "object") {
-          debugBuild && console.log("END CAPTURE", nestingLevel, options.title);
+      if (options.title) {
-        }
+        childOptions.title = options.title;
+      }
+    }
+    var progress = parentProgress.addChildTask(childOptions);
+  }
+
+  currentProgress.withValue(progress, function () {
+    currentMessageSet.withValue(messageSet, function () {
+      var job = null;
+      if (typeof options === "object") {
+        job = new Job(options);
+        messageSet.jobs.push(job);
+      } else {
+        f = options; // options not actually provided
+      }
+
+      currentJob.withValue(job, function () {
+        var nestingLevel = currentNestingLevel.get();
+        currentNestingLevel.withValue(nestingLevel + 1, function () {
+          var start;
+          if (debugBuild) {
+            start = Date.now();
+            console.log("START CAPTURE", nestingLevel, options.title, "took " + (end - start));
+          }
+          try {
+            f();
+          } finally {
+            progress.reportProgressDone();
+
+            if (debugBuild) {
+              var end = Date.now();
+              console.log("END CAPTURE", nestingLevel, options.title, "took " + (end - start));
+            }
+          }
+        });
       });
     });
   });
@@ -211,24 +278,57 @@ var enterJob = function (options, f) {
     options = {};
   }
 
-  if (! currentMessageSet.get()) {
+  var progress;
-    return f();
+  {
+    var parentProgress = currentProgress.get();
+    if (!parentProgress) {
+      parentProgress = rootProgress;
+    }
+    var progressOptions = {};
+    // XXX: Just pass all the options?
+    if (typeof options === "object") {
+      if (options.title) {
+        progressOptions.title = options.title;
+      }
+      if (options.forkJoin) {
+        progressOptions.forkJoin = options.forkJoin;
+      }
+    }
+    progress = parentProgress.addChildTask(progressOptions);
   }
 
-  var job = new Job(options);
+  currentProgress.withValue(progress, function () {
-  var originalJob = currentJob.get();
+    if (!currentMessageSet.get()) {
-  originalJob && originalJob.children.push(job);
-  currentMessageSet.get().jobs.push(job);
-
-  return currentJob.withValue(job, function () {
-    var nestingLevel = currentNestingLevel.get();
-    return currentNestingLevel.withValue(nestingLevel + 1, function () {
-      debugBuild && console.log("START", nestingLevel, options.title);
       try {
         return f();
       } finally {
-        debugBuild && console.log("DONE", nestingLevel, options.title);
+        progress.reportProgressDone();
       }
+    }
+
+    var job = new Job(options);
+    var originalJob = currentJob.get();
+    originalJob && originalJob.children.push(job);
+    currentMessageSet.get().jobs.push(job);
+
+    return currentJob.withValue(job, function () {
+      var nestingLevel = currentNestingLevel.get();
+      return currentNestingLevel.withValue(nestingLevel + 1, function () {
+        var start;
+        if (debugBuild) {
+          start = Date.now();
+          console.log("START", nestingLevel, options.title);
+        }
+        try {
+          return f();
+        } finally {
+          progress.reportProgressDone();
+          if (debugBuild) {
+            var end = Date.now();
+            console.log("DONE", nestingLevel, options.title, "took " + (end - start));
+          }
+        }
+      });
     });
   });
 };
@@ -366,6 +466,74 @@ var assertInCapture = function () {
     throw new Error("Expected to be in a buildmessage capture");
 };
 
+// Like _.each, but runs each operation in a separate job
+var forkJoin = function (options, iterable, fn) {
+  if (!_.isFunction(fn)) {
+    fn = iterable;
+    iterable = options;
+    options = {};
+  }
+
+  var futures = [];
+  var results = [];
+  // XXX: We could check whether the sub-jobs set estimates, and if not
+  // assume they each take the same amount of time and auto-report their completion
+  var errors = [];
+  var firstError = null;
+
+  options.forkJoin = true;
+
+  enterJob(options, function () {
+    var job = currentJob.get();
+    var messageSet = currentMessageSet.get();
+    var progress = currentProgress.get();
+
+    _.each(iterable, function (/*arguments*/) {
+      var fut = new Future();
+      var fnArguments = arguments;
+      Fiber(function () {
+        currentProgress.withValue(progress, function () {
+          currentMessageSet.withValue(messageSet, function () {
+            currentJob.withValue(job, function () {
+              try {
+                var result = enterJob({title: (options.title || '') + ' child' }, function () {
+                  return fn.apply(null, fnArguments);
+                });
+                fut['return'](result);
+              } catch (e) {
+                fut['throw'](e);
+              }
+            });
+          });
+        });
+      }).run();
+      futures.push(fut);
+    });
+
+    _.each(futures, function (future) {
+      try {
+        var result = future.wait();
+        results.push(result);
+        errors.push(null);
+      } catch (e) {
+        results.push(null);
+        errors.push(e);
+
+        if (firstError === null) {
+          firstError = e;
+        }
+      }
+    });
+  });
+
+  if (firstError) {
+    throw firstError;
+  }
+
+  return results;
+};
+
+
 var buildmessage = exports;
 _.extend(exports, {
   capture: capture,
@@ -375,5 +543,11 @@ _.extend(exports, {
   exception: exception,
   jobHasMessages: jobHasMessages,
   assertInJob: assertInJob,
-  assertInCapture: assertInCapture
+  assertInCapture: assertInCapture,
+  forkJoin: forkJoin,
+  getRootProgress: getRootProgress,
+  reportProgress: reportProgress,
+  reportProgressDone: reportProgressDone,
+  getCurrentProgressTracker: getCurrentProgressTracker,
+  addChildTracker: addChildTracker
 });

tools/catalog.js

@@ -138,16 +138,11 @@ _.extend(OfficialCatalog.prototype, {
     self._refreshFiber = Fiber.current;
     self._currentRefreshIsLoud = !options.silent;
 
-    var patience = new utils.Patience({
+    var thrownError = null;
-      messageAfterMs: 2000,
+
-      message: function () {
+    buildmessage.enterJob({
-        if (self._currentRefreshIsLoud) {
+      title: 'Refreshing package metadata.'
-          console.log("Refreshing package metadata. This may take a moment.");
+    }, function () {
-        }
-      }
-    });
-    try {
-      var thrownError = null;
       try {
         self._refresh();
         // Force the complete catalog (which is layered on top of our data) to
@@ -156,9 +151,7 @@ _.extend(OfficialCatalog.prototype, {
       } catch (e) {
         thrownError = e;
       }
-    } finally {
+    });
-      patience.stop();
-    }
 
     while (self._refreshFutures.length) {
       var fut = self._refreshFutures.pop();
@@ -897,9 +890,9 @@ _.extend(CompleteCatalog.prototype, {
     };
 
     // Load the package sources for packages and their tests into packageSources.
-    _.each(self.effectiveLocalPackages, function (x) {
+    buildmessage.forkJoin({ 'title': 'Initializing packages'}, self.effectiveLocalPackages, function (x) {
       initSourceFromDir(x);
-     });
+    });
 
     // Remove all packages from the catalog that have the same name as
     // a local package, along with all of their versions and builds.

tools/commands-cordova.js

@@ -21,6 +21,7 @@ var compiler = require('./compiler.js');
 var unipackage = require('./unipackage.js');
 var tropohouse = require('./tropohouse.js');
 var httpHelpers = require('./http-helpers.js');
+var Console = require('./console.js').Console;
 
 // XXX hard-coded the use of default tropohouse
 var tropo = tropohouse.default;
@@ -45,7 +46,7 @@ var setVerboseness = cordova.setVerboseness = function (v) {
 };
 var verboseLog = cordova.verboseLog = function (/* args */) {
   if (verboseness)
-    process.stderr.write('%% ' + util.format.apply(null, arguments) + '\n');
+    Console.stderr.write('%% ' + util.format.apply(null, arguments) + '\n');
 };
 
 
@@ -306,7 +307,7 @@ cordova.ensureCordovaProject = function (localPath, appName) {
       if (err instanceof main.ExitWithCode) {
         process.exit(err.code);
       }
-      process.stderr.write("Error creating Cordova project: " +
+      Console.stderr.write("Error creating Cordova project: " +
         err.message + "\n" + err.stack + "\n");
     }
   }
@@ -382,7 +383,7 @@ var installPlugin = function (cordovaPath, name, version, settings) {
     additionalArgs.push(variable + '=' + JSON.stringify(value));
   });
 
-  process.stdout.write('  installing ' + pluginInstallCommand + '\n');
+  Console.stdout.write('  installing ' + pluginInstallCommand + '\n');
   var execRes = execFileSyncOrThrow(localCordova,
      ['plugin', 'add', pluginInstallCommand].concat(additionalArgs),
      { cwd: cordovaPath });
@@ -528,7 +529,7 @@ var ensureCordovaPlugins = function (localPath, options) {
       files.rm_recursive(path.join(cordovaPath, 'platforms'));
       cordova.ensureCordovaPlatforms(localPath);
     };
-    process.stdout.write("Initializing Cordova plugins...\n");
+    Console.stdout.write("Initializing Cordova plugins...\n");
     uninstallAllPlugins();
 
     // Now install all of the plugins.
@@ -922,7 +923,7 @@ main.registerCommand({
       cordova.checkIsValidPlatform(platform);
     });
   } catch (err) {
-    process.stderr.write(err.message + "\n");
+    Console.stderr.write(err.message + "\n");
     return 1;
   }
 
@@ -950,7 +951,7 @@ main.registerCommand({
   }
 
   _.each(platforms, function (platform) {
-    process.stdout.write("added platform " + platform + "\n");
+    Console.stdout.write("added platform " + platform + "\n");
   });
 });
 
@@ -966,9 +967,9 @@ main.registerCommand({
 
   _.each(platforms, function (platform) {
     if (_.contains(currentPlatforms, platform)) {
-      process.stdout.write("removed platform " + platform + "\n");
+      Console.stdout.write("removed platform " + platform + "\n");
     } else {
-      process.stdout.write(platform + " is not in this project\n");
+      Console.stdout.write(platform + " is not in this project\n");
     }
   });
   project.removeCordovaPlatforms(platforms);
@@ -989,11 +990,11 @@ main.registerCommand({
   requiresApp: true
 }, function () {
   var platforms = project.getCordovaPlatforms();
-  process.stdout.write(platforms.join("\n"));
+  Console.stdout.write(platforms.join("\n"));
 
   // print nothing at all if no platforms
   if (platforms.length) {
-    process.stdout.write("\n");
+    Console.stdout.write("\n");
   }
 });
 

tools/commands.js

@@ -24,6 +24,7 @@ var unipackage = require('./unipackage.js');
 var cordova = require('./commands-cordova.js');
 var commandsPackages = require('./commands-packages.js');
 var execFileSync = require('./utils.js').execFileSync;
+var Console = require('./console.js').Console;
 
 // The architecture used by Galaxy servers; it's the architecture used
 // by 'meteor deploy'.
@@ -103,7 +104,7 @@ main.registerCommand({
   if (release.current === null) {
     if (! options.appDir)
       throw new Error("missing release, but not in an app?");
-    process.stderr.write(
+    Console.stderr.write(
 "This project was created with a checkout of Meteor, rather than an\n" +
 "official release, and doesn't have a release number associated with\n" +
 "it. You can set its release with 'meteor update'.\n");
@@ -111,7 +112,7 @@ main.registerCommand({
   }
 
   if (release.current.isCheckout()) {
-    process.stderr.write("Unreleased (running from a checkout)\n");
+    Console.stderr.write("Unreleased (running from a checkout)\n");
     return 1;
   }
 
@@ -124,15 +125,15 @@ main.registerCommand({
   requiresRelease: false
 }, function (options) {
   if (files.inCheckout()) {
-    process.stderr.write("checkout\n");
+    Console.stderr.write("checkout\n");
     return 1;
   } else if (release.current === null) {
     // .meteor/release says "none" but not in a checkout.
-    process.stderr.write("none\n");
+    Console.stderr.write("none\n");
     return 1;
   } else {
-    process.stdout.write(release.current.name + "\n");
+    Console.stdout.write(release.current.name + "\n");
-    process.stdout.write(files.getToolsVersion() + "\n");
+    Console.stdout.write(files.getToolsVersion() + "\n");
     return 0;
   }
 });
@@ -186,7 +187,7 @@ main.registerCommand({
     project.getVersions();  // #StructuredProjectInitialization
   });
   if (messages.hasMessages()) {
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
@@ -197,10 +198,10 @@ main.registerCommand({
     var parsedHostPort = parseHostPort(options.port);
   } catch (err) {
     if (options.verbose) {
-      process.stderr.write('Error while parsing --port option: '
+      Console.stderr.write('Error while parsing --port option: '
                            + err.stack + '\n');
     } else {
-      process.stderr.write(err.message + '\n');
+      Console.stderr.write(err.message + '\n');
     }
     return 1;
   }
@@ -227,7 +228,7 @@ main.registerCommand({
     cordova.verboseLog('A run on a device requested');
     // ... and if you didn't specify your ip address as host, print a warning
     if (parsedMobileHostPort.host === DEFAULT_BUILD_HOST)
-      process.stderr.write(
+      Console.stderr.write(
         "WARN: You are testing your app on a remote device but your host option\n" +
         "WARN: is set to 'localhost'. Perhaps you want to change it to your local\n" +
         "WARN: network's IP address with the -p or --mobile-port option?\n");
@@ -263,10 +264,10 @@ main.registerCommand({
       runners = runners.concat(cordova.buildPlatformRunners(localPath, options.args, options));
     } catch (err) {
       if (options.verbose) {
-        process.stderr.write('Error while running for mobile platforms ' +
+        Console.stderr.write('Error while running for mobile platforms ' +
                              err.stack + '\n');
       } else {
-        process.stderr.write(err.message + '\n');
+        Console.stderr.write(err.message + '\n');
       }
       return 1;
     }
@@ -276,7 +277,7 @@ main.registerCommand({
   if (options['app-port']) {
     var appPortMatch = options['app-port'].match(/^(?:(.+):)?([0-9]+)?$/);
     if (!appPortMatch) {
-      process.stderr.write(
+      Console.stderr.write(
 "run: --app-port must be a number or be of the form 'host:port' where\n" +
 "port is a number. Try 'meteor help run' for help.\n");
       return 1;
@@ -344,12 +345,12 @@ main.registerCommand({
 
     // No package examples exist yet.
     if (options.list && options.example) {
-      process.stderr.write("No package examples exist at this time.\n\n");
+      Console.stderr.write("No package examples exist at this time.\n\n");
       throw new main.ShowUsage;
     }
 
     if (!packageName) {
-      process.stderr.write("Please specify the name of the package. \n");
+      Console.stderr.write("Please specify the name of the package. \n");
       throw new main.ShowUsage;
     }
 
@@ -362,7 +363,7 @@ main.registerCommand({
     var inYourApp = options.appDir ? " in your app" : "";
 
     if (fs.existsSync(packageDir)) {
-      process.stderr.write(packageName + ": Already exists" + inYourApp + "\n");
+      Console.stderr.write(packageName + ": Already exists" + inYourApp + "\n");
       return 1;
     }
 
@@ -402,11 +403,11 @@ main.registerCommand({
       ignore: [/^local$/]
     });
    } catch (err) {
-     process.stderr.write("Could not create package: " + err.message + "\n");
+     Console.stderr.write("Could not create package: " + err.message + "\n");
      return 1;
    }
 
-    process.stdout.write(packageName + ": created" + inYourApp + "\n");
+    Console.stdout.write(packageName + ": created" + inYourApp + "\n");
     return 0;
   }
 
@@ -434,11 +435,11 @@ main.registerCommand({
   });
 
   if (options.list) {
-    process.stdout.write("Available examples:\n");
+    Console.stdout.write("Available examples:\n");
     _.each(examples, function (e) {
-      process.stdout.write("  " + e + "\n");
+      Console.stdout.write("  " + e + "\n");
     });
-    process.stdout.write("\n" +
+    Console.stdout.write("\n" +
 "Create a project from an example with 'meteor create --example <name>'.\n");
     return 0;
   };
@@ -452,12 +453,12 @@ main.registerCommand({
     throw new main.ShowUsage;
 
   if (fs.existsSync(appPath)) {
-    process.stderr.write(appPath + ": Already exists\n");
+    Console.stderr.write(appPath + ": Already exists\n");
     return 1;
   }
 
   if (files.findAppDir(appPath)) {
-    process.stderr.write(
+    Console.stderr.write(
       "You can't create a Meteor project inside another Meteor project.\n");
     return 1;
   }
@@ -468,8 +469,8 @@ main.registerCommand({
 
   if (options.example) {
     if (examples.indexOf(options.example) === -1) {
-      process.stderr.write(options.example + ": no such example\n\n");
+      Console.stderr.write(options.example + ": no such example\n\n");
-      process.stderr.write("List available applications with 'meteor create --list'.\n");
+      Console.stderr.write("List available applications with 'meteor create --list'.\n");
       return 1;
     } else {
       files.cp_r(path.join(exampleDir, options.example), appPath, {
@@ -509,7 +510,7 @@ main.registerCommand({
     project.appendFinishedUpgrader(upgrader);
   });
 
-  var messages = buildmessage.capture(function () {
+  var messages = buildmessage.capture({ title: "Updating dependencies" }, function () {
     // Run the constraint solver. Override the assumption that using '--release'
     // means we shouldn't update .meteor/versions.
     project._ensureDepsUpToDate({alwaysRecord: true});
@@ -517,16 +518,19 @@ main.registerCommand({
 
 
   if (messages.hasMessages()) {
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
-  process.stdout.write(appPath + ": created");
+  {
-  if (options.example && options.example !== appPath)
+    var message = appPath + ": created";
-    process.stderr.write(" (from '" + options.example + "' template)");
+    if (options.example && options.example !== appPath)
-  process.stdout.write(".\n\n");
+      message += (" (from '" + options.example + "' template)");
+    message += ".\n";
+    Console.info(message);
+  }
 
-  process.stdout.write(
+  Console.stdout.write(
     "To run your new app:\n" +
       "   cd " + appPath + "\n" +
       "   meteor\n");
@@ -588,8 +592,8 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
   ["os.osx.x86_64", "os.linux.x86_64", "os.linux.x86_32"];
   if (options.architecture &&
       _.indexOf(VALID_ARCHITECTURES, options.architecture) === -1) {
-    process.stderr.write("Invalid architecture: " + options.architecture + "\n");
+    Console.stderr.write("Invalid architecture: " + options.architecture + "\n");
-    process.stderr.write(
+    Console.stderr.write(
       "Please use one of the following: " + VALID_ARCHITECTURES + "\n");
     return 1;
   }
@@ -603,11 +607,10 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
   var appName = path.basename(options.appDir);
 
   if (! _.isEmpty(mobilePlatforms)) {
-
     try {
       var parsedHostPort = parseHostPort(options.port);
     } catch (err) {
-      process.stderr.write(err.message);
+      Console.stderr.write(err.message);
       return 1;
     }
 
@@ -638,8 +641,8 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
     loader = project.getPackageLoader();
   });
   if (messages.hasMessages()) {
-    process.stderr.write("Errors prevented bundling your app:\n");
+    Console.stderr.write("Errors prevented bundling your app:\n");
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
@@ -647,7 +650,7 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
     stats.recordPackages("sdk.bundle");
   });
   if (statsMessages.hasMessages()) {
-    process.stdout.write("Error recording package list:\n" +
+    Console.stdout.write("Error recording package list:\n" +
                          statsMessages.formatMessages());
     // ... but continue;
   }
@@ -667,8 +670,8 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
     }
   });
   if (bundleResult.errors) {
-    process.stderr.write("Errors prevented bundling:\n");
+    Console.stderr.write("Errors prevented bundling:\n");
-    process.stderr.write(bundleResult.errors.formatMessages());
+    Console.stderr.write(bundleResult.errors.formatMessages());
     return 1;
   }
 
@@ -679,7 +682,7 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
       files.createTarball(path.join(buildDir, 'bundle'), outputTar);
     } catch (err) {
       console.log(JSON.stringify(err));
-      process.stderr.write("Couldn't create tarball\n");
+      Console.stderr.write("Couldn't create tarball\n");
     }
   }
 
@@ -699,7 +702,7 @@ main.registerCommand(_.extend({ name: 'build' }, buildCommands),
 main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
     function (options) {
   cordova.setVerboseness(options.verbose);
-  process.stdout.write("WARNING: 'bundle' has been deprecated. " +
+  Console.stdout.write("WARNING: 'bundle' has been deprecated. " +
                        "Use 'build' instead.\n");
   // XXX if they pass a file that doesn't end in .tar.gz or .tgz, add
   // the former for them
@@ -718,8 +721,8 @@ main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
   ["os.osx.x86_64", "os.linux.x86_64", "os.linux.x86_32"];
   if (options.architecture &&
       _.indexOf(VALID_ARCHITECTURES, options.architecture) === -1) {
-    process.stderr.write("Invalid architecture: " + options.architecture + "\n");
+    Console.stderr.write("Invalid architecture: " + options.architecture + "\n");
-    process.stderr.write(
+    Console.stderr.write(
       "Please use one of the following: " + VALID_ARCHITECTURES + "\n");
     return 1;
   }
@@ -735,8 +738,8 @@ main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
     loader = project.getPackageLoader();
   });
   if (messages.hasMessages()) {
-    process.stderr.write("Errors prevented bundling your app:\n");
+    Console.stderr.write("Errors prevented bundling your app:\n");
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
@@ -744,7 +747,7 @@ main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
     stats.recordPackages("sdk.bundle");
   });
   if (statsMessages.hasMessages()) {
-    process.stdout.write("Error recording package list:\n" +
+    Console.stdout.write("Error recording package list:\n" +
                          statsMessages.formatMessages());
     // ... but continue;
   }
@@ -763,8 +766,8 @@ main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
     }
   });
   if (bundleResult.errors) {
-    process.stderr.write("Errors prevented bundling:\n");
+    Console.stderr.write("Errors prevented bundling:\n");
-    process.stderr.write(bundleResult.errors.formatMessages());
+    Console.stderr.write(bundleResult.errors.formatMessages());
     return 1;
   }
 
@@ -773,7 +776,7 @@ main.registerCommand(_.extend({ name: 'bundle', hidden: true }, buildCommands),
       files.createTarball(path.join(buildDir, 'bundle'), outputPath);
     } catch (err) {
       console.log(JSON.stringify(err));
-      process.stderr.write("Couldn't create tarball\n");
+      Console.stderr.write("Couldn't create tarball\n");
     }
   }
   files.rm_recursive(buildDir);
@@ -806,7 +809,7 @@ main.registerCommand({
     // specified?
 
     if (! mongoPort) {
-      process.stdout.write(
+      Console.stdout.write(
 "mongo: Meteor isn't running a local MongoDB server.\n" +
 "\n" +
 "This command only works while Meteor is running your application\n" +
@@ -862,7 +865,7 @@ main.registerCommand({
   requiresApp: true
 }, function (options) {
   if (options.args.length !== 0) {
-    process.stderr.write(
+    Console.stderr.write(
 "meteor reset only affects the locally stored database.\n" +
 "\n" +
 "To reset a deployed application use\n" +
@@ -879,7 +882,7 @@ main.registerCommand({
     require(path.join(__dirname, 'run-mongo.js')).findMongoPort;
   var isRunning = !! findMongoPort(options.appDir);
   if (isRunning) {
-    process.stderr.write(
+    Console.stderr.write(
 "reset: Meteor is running.\n" +
 "\n" +
 "This command does not work while Meteor is running your application.\n" +
@@ -890,7 +893,7 @@ main.registerCommand({
   var localDir = path.join(options.appDir, '.meteor', 'local');
   files.rm_recursive(localDir);
 
-  process.stdout.write("Project reset.\n");
+  Console.stdout.write("Project reset.\n");
 });
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -945,19 +948,19 @@ main.registerCommand({
   // issues are concurrency-related, or possibly some weird order-of-execution
   // of interaction that we are failing to understand. This seems to fix it in a
   // clear and understandable fashion.)
-  var messages = buildmessage.capture(function () {
+  var messages = buildmessage.capture({ title: "Resolving versions" }, function () {
     project.getVersions();  // #StructuredProjectInitialization
   });
   if (messages.hasMessages()) {
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
   if (options.password) {
     if (useGalaxy) {
-      process.stderr.write("Galaxy does not support --password.\n");
+      Console.stderr.write("Galaxy does not support --password.\n");
     } else {
-      process.stderr.write(
+      Console.stderr.write(
 "Setting passwords on apps is no longer supported. Now there are\n" +
 "user accounts and your apps are associated with your account so that\n" +
 "only you (and people you designate) can access them. See the\n" +
@@ -969,14 +972,14 @@ main.registerCommand({
   var starball = options.star;
   if (starball && ! useGalaxy) {
     // XXX it would be nice to support this for non-Galaxy deploys too
-    process.stderr.write(
+    Console.stderr.write(
 "--star: only supported when deploying to Galaxy.\n");
     return 1;
   }
 
   var loggedIn = auth.isLoggedIn();
   if (! loggedIn) {
-    process.stderr.write(
+    Console.stderr.write(
 "To instantly deploy your app on a free testing server, just enter your\n" +
 "email address!\n" +
 "\n");
@@ -988,9 +991,9 @@ main.registerCommand({
   // Override architecture iff applicable.
   var buildArch = DEPLOY_ARCH;
   if (options['override-architecture-with-local']) {
-    process.stdout.write(
+    Console.stdout.write(
       "\n => WARNING: OVERRIDING DEPLOY ARCHITECTURE WITH LOCAL ARCHITECTURE\n");
-    process.stdout.write(
+    Console.stdout.write(
       " => If your app contains binary code, it may break terribly and you will be sad.\n\n");
     buildArch = archinfo.host();
   }
@@ -1081,13 +1084,13 @@ main.registerCommand({
 }, function (options) {
 
   if (options.add && options.remove) {
-    process.stderr.write(
+    Console.stderr.write(
       "Sorry, you can only add or remove one user at a time.\n");
     return 1;
   }
 
   if ((options.add || options.remove) && options.list) {
-    process.stderr.write(
+    Console.stderr.write(
 "Sorry, you can't change the users at the same time as you're listing them.\n");
     return 1;
   }
@@ -1097,7 +1100,7 @@ main.registerCommand({
   var site = qualifySitename(options.args[0]);
 
   if (hostedWithGalaxy(site)) {
-    process.stderr.write(
+    Console.stderr.write(
 "Sites hosted on Galaxy do not have an authorized user list.\n" +
 "Instead, go to your Galaxy dashboard to change the authorized users\n" +
 "of your Galaxy.\n");
@@ -1105,7 +1108,7 @@ main.registerCommand({
   }
 
   if (! auth.isLoggedIn()) {
-    process.stderr.write(
+    Console.stderr.write(
       "You must be logged in for that. Try 'meteor login'.\n");
     return 1;
   }
@@ -1132,7 +1135,7 @@ main.registerCommand({
   var site = qualifySitename(options.args[0]);
 
   if (! auth.isLoggedIn()) {
-    process.stderr.write(
+    Console.stderr.write(
 "You must be logged in to claim sites. Use 'meteor login' to log in.\n" +
 "If you don't have a Meteor developer account yet, create one by clicking\n" +
 "'Sign in' and then 'Create account' at www.meteor.com.\n\n");
@@ -1140,7 +1143,7 @@ main.registerCommand({
   }
 
   if (hostedWithGalaxy(site)) {
-    process.stderr.write(
+    Console.stderr.write(
       "Sorry, you can't claim sites that are hosted on Galaxy.\n");
     return 1;
   }
@@ -1194,7 +1197,7 @@ main.registerCommand({
   try {
     var parsedHostPort = parseHostPort(options.port);
   } catch (err) {
-    process.stderr.write(err.message);
+    Console.stderr.write(err.message);
     return 1;
   }
 
@@ -1213,7 +1216,7 @@ main.registerCommand({
     localPackages = packages.localPackages;
     options.localPackageNames = packages.localPackages;
   } catch (err) {
-    process.stderr.write('\n' + err.message);
+    Console.stderr.write('\n' + err.message);
     return 1;
   }
 
@@ -1276,7 +1279,7 @@ main.registerCommand({
       runners = runners.concat(cordova.buildPlatformRunners(
         localPath, mobilePlatforms, options));
     } catch (err) {
-      process.stderr.write(err.message + '\n');
+      Console.stderr.write(err.message + '\n');
       return 1;
     }
   }
@@ -1378,7 +1381,7 @@ var getPackagesForTest = function (packages) {
     });
 
     if (messages.hasMessages()) {
-      process.stderr.write("\n" + messages.formatMessages());
+      Console.stderr.write("\n" + messages.formatMessages());
       return 1;
     }
   }
@@ -1416,7 +1419,7 @@ var runTestAppForPackages = function (testPackages, testRunnerAppDir, options) {
     });
   });
   if (messages.hasMessages()) {
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
   project.forceEditPackages(tests, 'add');
@@ -1429,7 +1432,7 @@ var runTestAppForPackages = function (testPackages, testRunnerAppDir, options) {
   });
 
   if (messages.hasMessages()) {
-    process.stderr.write(messages.formatMessages());
+    Console.stderr.write(messages.formatMessages());
     return 1;
   }
 
@@ -1519,7 +1522,7 @@ main.registerCommand({
   if (count)
     console.log("Built " + count + " packages.");
   if (messages.hasMessages()) {
-    process.stderr.write("\n" + messages.formatMessages());
+    Console.stderr.write("\n" + messages.formatMessages());
     return 1;
   }
 });
@@ -1571,18 +1574,18 @@ main.registerCommand({
 var loggedInAccountsConnectionOrPrompt = function (action) {
   var token = auth.getSessionToken(config.getAccountsDomain());
   if (! token) {
-    process.stderr.write("You must be logged in to " + action + ".\n");
+    Console.stderr.write("You must be logged in to " + action + ".\n");
     auth.doUsernamePasswordLogin({ retry: true });
-    process.stdout.write("\n");
+    Console.stdout.write("\n");
   }
 
   token = auth.getSessionToken(config.getAccountsDomain());
   var conn = auth.loggedInAccountsConnection(token);
   if (conn === null) {
     // Server rejected our token.
-    process.stderr.write("You must be logged in to " + action + ".\n");
+    Console.stderr.write("You must be logged in to " + action + ".\n");
     auth.doUsernamePasswordLogin({ retry: true });
-    process.stdout.write("\n");
+    Console.stdout.write("\n");
     token = auth.getSessionToken(config.getAccountsDomain());
     conn = auth.loggedInAccountsConnection(token);
   }
@@ -1599,9 +1602,9 @@ main.registerCommand({
 
   var token = auth.getSessionToken(config.getAccountsDomain());
   if (! token) {
-    process.stderr.write("You must be logged in to list your organizations.\n");
+    Console.stderr.write("You must be logged in to list your organizations.\n");
     auth.doUsernamePasswordLogin({ retry: true });
-    process.stdout.write("\n");
+    Console.stdout.write("\n");
   }
 
   var url = config.getAccountsApiUrl() + "/organizations";
@@ -1614,13 +1617,13 @@ main.registerCommand({
     });
     var body = JSON.parse(result.body);
   } catch (err) {
-    process.stderr.write("Error listing organizations.\n");
+    Console.stderr.write("Error listing organizations.\n");
     return 1;
   }
 
   if (result.response.statusCode === 401 &&
       body && body.error === "invalid_credential") {
-    process.stderr.write("You must be logged in to list your organizations.\n");
+    Console.stderr.write("You must be logged in to list your organizations.\n");
     // XXX It would be nice to do a username/password prompt here like
     // we do for the other orgs commands.
     return 1;
@@ -1628,14 +1631,14 @@ main.registerCommand({
 
   if (result.response.statusCode !== 200 ||
       ! body || ! body.organizations) {
-    process.stderr.write("Error listing organizations.\n");
+    Console.stderr.write("Error listing organizations.\n");
     return 1;
   }
 
   if (body.organizations.length === 0) {
-    process.stdout.write("You are not a member of any organizations.\n");
+    Console.stdout.write("You are not a member of any organizations.\n");
   } else {
-    process.stdout.write(_.pluck(body.organizations, "name").join("\n") + "\n");
+    Console.stdout.write(_.pluck(body.organizations, "name").join("\n") + "\n");
   }
   return 0;
 });
@@ -1652,7 +1655,7 @@ main.registerCommand({
 }, function (options) {
 
   if (options.add && options.remove) {
-    process.stderr.write(
+    Console.stderr.write(
       "Sorry, you can only add or remove one member at a time.\n");
     throw new main.ShowUsage;
   }
@@ -1671,13 +1674,13 @@ main.registerCommand({
         options.add ? "addOrganizationMember": "removeOrganizationMember",
         options.args[0], username);
     } catch (err) {
-      process.stderr.write("Error " +
+      Console.stderr.write("Error " +
                            (options.add ? "adding" : "removing") +
                            " member: " + err.reason + "\n");
       return 1;
     }
 
-    process.stdout.write(username + " " +
+    Console.stdout.write(username + " " +
                          (options.add ? "added to" : "removed from") +
                          " organization " + options.args[0] + ".\n");
   } else {
@@ -1685,14 +1688,14 @@ main.registerCommand({
     try {
       var result = conn.call("showOrganization", options.args[0]);
     } catch (err) {
-      process.stderr.write("Error showing organization: " +
+      Console.stderr.write("Error showing organization: " +
                            err.reason + "\n");
       return 1;
     }
 
     var members = _.pluck(result, "username");
 
-    process.stdout.write(members.join("\n") + "\n");
+    Console.stdout.write(members.join("\n") + "\n");
   }
 
   return 0;
@@ -1739,7 +1742,7 @@ main.registerCommand({
     } catch (e) {
       if (!(e instanceof SyntaxError))
         throw e;
-      process.stderr.write("Bad regular expression: " + options.args[0] + "\n");
+      Console.stderr.write("Bad regular expression: " + options.args[0] + "\n");
       return 1;
     }
   }
@@ -1770,7 +1773,7 @@ main.registerCommand({
 }, function (options) {
   auth.pollForRegistrationCompletion();
   if (! auth.isLoggedIn()) {
-    process.stderr.write(
+    Console.stderr.write(
       "You must be logged in for that. Try 'meteor login'.\n");
     return 1;
   }
@@ -1803,10 +1806,10 @@ main.registerCommand({
     return 'none';
   };
 
-  process.stdout.write(p('email') + " " + p('port') + " " + p('changed') +
+  Console.stdout.write(p('email') + " " + p('port') + " " + p('changed') +
                        " " + p('args') + "\n");
   if (options.url)
-    process.stdout.write('url\n');
+    Console.stdout.write('url\n');
   if (options['delete'])
-    process.stdout.write('delete\n');
+    Console.stdout.write('delete\n');
 });

tools/compiler.js

@@ -797,7 +797,7 @@ compiler.compile = function (packageSource, options) {
         versions: buildTimeDeps.pluginDependencies[info.name],
         catalog: packageSource.catalog
       });
-      loader.downloadMissingPackages({serverArch: archinfo.host()});
+      loader.downloadMissingPackages({serverArch: archinfo.host() });
 
       var buildResult = bundler.buildJsImage({
         name: info.name,

tools/console.js

@@ -0,0 +1,301 @@
+///
+/// utility functions for formatting output to the screen
+///
+
+var _ = require('underscore');
+var Fiber = require('fibers');
+var Future = require('fibers/future');
+var ProgressBar = require('progress');
+var buildmessage = require('./buildmessage.js');
+// XXX: Are we happy with chalk (and its sub-dependencies)?
+var chalk = require('chalk');
+
+PROGRESS_DEBUG = !!process.env.METEOR_PROGRESS_DEBUG;
+USE_PRETTY = (process.env.METEOR_PRETTY_OUTPUT != '0');
+
+var Console = function (options) {
+  var self = this;
+
+  options = options || {};
+
+  self._progressBar = null;
+  self._progressBarText = null;
+  self._watching = null;
+
+  self._lastStatusPoll = 0;
+
+  // Legacy helpers
+  self.stdout = {};
+  self.stderr = {};
+  self.stdout.write = function (msg) {
+    self._legacyWrite(LEVEL_INFO, msg);
+  };
+  self.stderr.write = function (msg) {
+    self._legacyWrite(LEVEL_WARN, msg);
+  };
+
+  self._stream =process.stdout;
+};
+
+
+PROGRESS_BAR_WIDTH = 20;
+PROGRESS_BAR_FORMAT = '[:bar] :percent :etas';
+STATUS_POSITION = PROGRESS_BAR_WIDTH + 15;
+STATUS_MAX_LENGTH = 40;
+
+
+// Message to show when we don't know what we're doing
+// XXX: ? FALLBACK_STATUS = 'Pondering';
+FALLBACK_STATUS = '';
+
+// This function returns a future which resolves after a timeout. This
+// demonstrates manually resolving futures.
+function sleep(ms) {
+  var future = new Future;
+  setTimeout(function() {
+    future.return();
+  }, ms);
+  return future.wait();
+};
+
+LEVEL_CODE_ERROR = 4;
+LEVEL_CODE_WARN = 3;
+LEVEL_CODE_INFO = 2;
+LEVEL_CODE_DEBUG = 1;
+
+LEVEL_ERROR = { code: LEVEL_CODE_ERROR };
+LEVEL_WARN = { code: LEVEL_CODE_WARN };
+LEVEL_INFO = { code: LEVEL_CODE_INFO };
+LEVEL_DEBUG = { code: LEVEL_CODE_DEBUG };
+
+_.extend(Console.prototype, {
+  hideProgressBar: function () {
+    var self = this;
+
+    if (!self._progressBar) {
+      return;
+    }
+    self._progressBar.terminate();
+  },
+
+  _renderProgressBar: function () {
+    var self = this;
+    if (self._progressBar) {
+      self._progressBar.render();
+      if (self._progressBarText) {
+        var text = self._progressBarText;
+        if (text > STATUS_MAX_LENGTH) {
+          text = text.substring(0, STATUS_MAX_LENGTH - 3) + "...";
+        } else {
+          while (text.length < STATUS_MAX_LENGTH) {
+            text = text + ' ';
+          }
+        }
+        self._stream.cursorTo(STATUS_POSITION);
+        self._stream.write(chalk.bold(text));
+      }
+    }
+  },
+
+  _statusPoll: function () {
+    var self = this;
+
+    self._lastStatusPoll = Date.now();
+
+    var rootProgress = buildmessage.getRootProgress();
+    if (PROGRESS_DEBUG) {
+      rootProgress.dump(process.stdout, {skipDone: true});
+    }
+    var current = (rootProgress ? rootProgress.getCurrentProgress() : null);
+    if (self._watching === current) {
+      return;
+    }
+
+    self._watching = current;
+    var title = (current != null ? current._title : null) || FALLBACK_STATUS;
+    if (title != self._progressBarText) {
+      self._progressBarText = title;
+      self._renderProgressBar();
+    }
+
+    self._watchProgress();
+  },
+
+  statusPollMaybe: function () {
+    var self = this;
+    var now = Date.now();
+
+    if ((now - self._lastStatusPoll) < 50) {
+      return;
+    }
+    self._statusPoll();
+  },
+
+  enableStatusPoll: function () {
+    var self = this;
+    Fiber(function () {
+      while (true) {
+        sleep(10);
+
+        self._statusPoll();
+      }
+    }).run();
+  },
+
+  info: function(/*arguments*/) {
+    var self = this;
+
+    var message = self._format(arguments);
+    self._print(LEVEL_INFO, message);
+  },
+
+  warn: function(/*arguments*/) {
+    var self = this;
+
+    var message = self._format(arguments);
+    self._print(LEVEL_WARN, message);
+  },
+
+  error: function(/*arguments*/) {
+    var self = this;
+
+    var message = self._format(arguments);
+    self._print(LEVEL_ERROR, message);
+  },
+
+  _legacyWrite: function (level, message) {
+    var self = this;
+    if(message.substr(-1) == '\n') {
+      message = message.substr(0, message.length - 1);
+    }
+    self._print(level, message);
+  },
+
+  _print: function(level, message) {
+    var self = this;
+
+    var progressBar = self._progressBar;
+    if (progressBar) {
+      //progressBar.terminate();
+      self._stream.clearLine();
+      self._stream.cursorTo(0);
+    }
+
+    var dest = process.stdout;
+    var style = null;
+
+    if (level && USE_PRETTY) {
+      switch (level.code) {
+        case LEVEL_CODE_ERROR:
+          dest = process.stderr;
+          style = chalk.bold.red;
+          break;
+        case LEVEL_CODE_WARN:
+          dest = process.stderr;
+          style = chalk.red;
+          break;
+        //case LEVEL_CODE_INFO:
+        //  style = chalk.blue;
+        //  break;
+      }
+    }
+
+    if (style) {
+      dest.write(style(message + '\n'));
+    } else {
+      dest.write(message + '\n');
+    }
+
+    if (progressBar) {
+      self._renderProgressBar();
+    }
+  },
+
+  _format: function (logArguments) {
+    var self = this;
+
+    var message = '';
+    var format = logArguments[0];
+    message = format;
+    return message;
+  },
+
+  printMessages: function (messages) {
+    var self = this;
+
+    if (messages.hasMessages()) {
+      self._print(null, "\n" + messages.formatMessages());
+    }
+  },
+
+  showProgressBar: function () {
+    var self = this;
+
+    if (self._progressBar) {
+      return;
+    }
+
+    if (!self._stream.isTTY || !USE_PRETTY) return;
+
+    var options = {
+      complete: '=',
+      incomplete: ' ',
+      width: PROGRESS_BAR_WIDTH,
+      total: 100,
+      clear: true,
+      stream: self._stream
+    };
+
+    var progressBar = new ProgressBar(PROGRESS_BAR_FORMAT, options);
+    progressBar.start = new Date;
+
+    self._progressBar = progressBar;
+  },
+
+  _watchProgress: function () {
+    var self = this;
+
+    var progress = self._watching;
+    if (!progress) return;
+
+    progress.addWatcher(function (state) {
+      //console.log(state);
+      if (progress != self._watching) {
+        // No longer active
+        //console.log("NOT WATCHING");
+        return;
+      }
+
+      var progressBar = self._progressBar;
+      if (!progressBar) {
+        //console.log("NOT PROGRESS BAR");
+        // Progress bar disabled
+        return;
+      }
+
+      var fraction;
+      if (state.done) {
+        fraction = 1.0;
+      } else {
+        var current = state.current;
+        var end = state.end;
+        if (end === undefined || end == 0 || current == 0) {
+          // Arbitrary end-point
+          fraction = progressBar.curr / 100;
+        } else {
+          fraction = current / end;
+        }
+      }
+
+      if (!isNaN(fraction) && fraction >= 0) {
+        progressBar.curr = Math.floor(fraction * progressBar.total);
+        self._renderProgressBar();
+      }
+    });
+  }
+
+});
+
+Console.warning = Console.warn;
+
+exports.Console = new Console;

tools/deploy.js

@@ -15,6 +15,7 @@ var utils = require('./utils.js');
 var _ = require('underscore');
 var Future = require('fibers/future');
 var stats = require('./stats.js');
+var Console = require('./console.js').Console;
 
 // Make a synchronous RPC to the "classic" MDG deploy API. The deploy
 // API has the following contract:
@@ -65,6 +66,7 @@ var deployRpc = function (options) {
   if (options.headers.cookie)
     throw new Error("sorry, can't combine cookie headers yet");
 
+  var progress = buildmessage.addChildTracker("Uploading");
   try {
     var result = httpHelpers.request(_.extend(options, {
       url: config.getDeployUrl() + '/' + options.operation +
@@ -72,13 +74,16 @@ var deployRpc = function (options) {
       method: options.method || 'GET',
       bodyStream: options.bodyStream,
       useAuthHeader: true,
-      encoding: 'utf8' // Hack, but good enough for the deploy server..
+      encoding: 'utf8', // Hack, but good enough for the deploy server..
+      progress: progress
     }));
   } catch (e) {
     return {
       statusCode: null,
       errorMessage: "Connection error (" + e.message + ")"
     };
+  } finally {
+    progress.reportProgressDone();
   }
 
   var response = result.response;
@@ -254,7 +259,7 @@ var authedRpc = function (options) {
 // password-protected app, instruct them to claim it with 'meteor
 // claim'.
 var printLegacyPasswordMessage = function (site) {
-    process.stderr.write(
+  Console.stderr.write(
 "\nThis site was deployed with an old version of Meteor that used\n" +
 "site passwords instead of user accounts. Now we have a much better\n" +
 "system, Meteor developer accounts.\n\n" +
@@ -267,7 +272,7 @@ var printLegacyPasswordMessage = function (site) {
 // --add' or switch accounts.
 var printUnauthorizedMessage = function () {
   var username = auth.loggedInUsername();
-  process.stderr.write(
+  Console.stderr.write(
 "Sorry, that site belongs to a different user.\n" +
 (username ? "You are currently logged in as " + username  + ".\n" : "") +
 "\nEither have the site owner use 'meteor authorized --add' to add you\n" +
@@ -288,7 +293,7 @@ var canonicalizeSite = function (site) {
   // characters (url.parse will do something very strange if a component is
   // larger than 63, which is the maximum legal length).
   if (site.length > 63) {
-    process.stdout.write(
+    Console.stdout.write(
 "The maximum hostname length currently supported is 63 characters.\n" +
 site + " is too long.\n" +
 "Please try again with a shorter URL for your site.\n");
@@ -302,14 +307,14 @@ site + " is too long.\n" +
   var parsed = require('url').parse(url);
 
   if (! parsed.hostname) {
-    process.stdout.write(
+    Console.stdout.write(
 "Please specify a domain to connect to, such as www.example.com or\n" +
 "http://www.example.com/\n");
     return false;
   }
 
   if (parsed.pathname != '/' || parsed.hash || parsed.query) {
-    process.stdout.write(
+    Console.stdout.write(
 "Sorry, Meteor does not yet support specific path URLs, such as\n" +
 "http://www.example.com/blog .  Please specify the root of a domain.\n");
     return false;
@@ -362,14 +367,14 @@ var bundleAndDeploy = function (options) {
   });
 
   if (preflight.errorMessage) {
-    process.stderr.write("\nError deploying application: " +
+    Console.stderr.write("\nError deploying application: " +
                          preflight.errorMessage + "\n");
     return 1;
   }
 
   if (preflight.protection === "password") {
     printLegacyPasswordMessage(site);
-    process.stderr.write("If it's not your site, please try a different name!\n");
+    Console.stderr.write("If it's not your site, please try a different name!\n");
     return 1;
 
   } else if (preflight.protection === "account" &&
@@ -381,7 +386,7 @@ var bundleAndDeploy = function (options) {
   var buildDir = path.join(options.appDir, '.meteor', 'local', 'build_tar');
   var bundlePath = path.join(buildDir, 'bundle');
 
-  process.stdout.write('Deploying to ' + site + '. Bundling...\n');
+  Console.stdout.write('Deploying to ' + site + '. Bundling...\n');
 
   var settings = null;
   var messages = buildmessage.capture({
@@ -396,11 +401,11 @@ var bundleAndDeploy = function (options) {
     var bundler = require('./bundler.js');
 
     if (options.recordPackageUsage) {
-      var statsMessages = buildmessage.capture(function () {
+      var statsMessages = buildmessage.capture({ title: 'Reporting statistics' }, function () {
         stats.recordPackages("sdk.deploy", site);
       });
       if (statsMessages.hasMessages()) {
-        process.stdout.write("Error recording package list:\n" +
+        Console.stdout.write("Error recording package list:\n" +
                              statsMessages.formatMessages());
         // ... but continue;
       }
@@ -416,25 +421,29 @@ var bundleAndDeploy = function (options) {
   }
 
   if (messages.hasMessages()) {
-    process.stdout.write("\nErrors prevented deploying:\n");
+    Console.stdout.write("\nErrors prevented deploying:\n");
-    process.stdout.write(messages.formatMessages());
+    Console.stdout.write(messages.formatMessages());
     return 1;
   }
 
-  process.stdout.write('Uploading...\n');
+  Console.stdout.write('Uploading...\n');
 
-  var result = authedRpc({
+  var result;
-    method: 'POST',
+  buildmessage.enterJob({ title: "Uploading" }, function () {
-    operation: 'deploy',
+    result = authedRpc({
-    site: site,
+      method: 'POST',
-    qs: settings !== null ? { settings: settings } : {},
+      operation: 'deploy',
-    bodyStream: files.createTarGzStream(path.join(buildDir, 'bundle')),
+      site: site,
-    expectPayload: ['url'],
+      qs: settings !== null ? {settings: settings} : {},
-    preflightPassword: preflight.preflightPassword
+      bodyStream: files.createTarGzStream(path.join(buildDir, 'bundle')),
+      expectPayload: ['url'],
+      preflightPassword: preflight.preflightPassword
+    });
   });
 
+
   if (result.errorMessage) {
-    process.stderr.write("\nError deploying application: " +
+    Console.stderr.write("\nError deploying application: " +
                          result.errorMessage + "\n");
     return 1;
   }
@@ -442,7 +451,7 @@ var bundleAndDeploy = function (options) {
   var deployedAt = require('url').parse(result.payload.url);
   var hostname = deployedAt.hostname;
 
-  process.stdout.write('Now serving at ' + hostname + '\n');
+  Console.stdout.write('Now serving at ' + hostname + '\n');
   files.rm_recursive(buildDir);
 
   if (! hostname.match(/meteor\.com$/)) {
@@ -451,11 +460,11 @@ var bundleAndDeploy = function (options) {
       if (err || cnames[0] !== 'origin.meteor.com') {
         dns.resolve(hostname, 'A', function (err, addresses) {
           if (err || addresses[0] !== '107.22.210.133') {
-            process.stdout.write('-------------\n');
+            Console.stdout.write('-------------\n');
-            process.stdout.write("You've deployed to a custom domain.\n");
+            Console.stdout.write("You've deployed to a custom domain.\n");
-            process.stdout.write("Please be sure to CNAME your hostname to origin.meteor.com,\n");
+            Console.stdout.write("Please be sure to CNAME your hostname to origin.meteor.com,\n");
-            process.stdout.write("or set an A record to 107.22.210.133.\n");
+            Console.stdout.write("or set an A record to 107.22.210.133.\n");
-            process.stdout.write('-------------\n');
+            Console.stdout.write('-------------\n');
           }
         });
         }
@@ -478,12 +487,12 @@ var deleteApp = function (site) {
   });
 
   if (result.errorMessage) {
-    process.stderr.write("Couldn't delete application: " +
+    Console.stderr.write("Couldn't delete application: " +
                          result.errorMessage + "\n");
     return 1;
   }
 
-  process.stdout.write("Deleted.\n");
+  Console.stdout.write("Deleted.\n");
   return 0;
 };
 
@@ -504,7 +513,7 @@ var checkAuthThenSendRpc = function (site, operation, what) {
   });
 
   if (preflight.errorMessage) {
-    process.stderr.write("Couldn't " + what + ": " +
+    Console.stderr.write("Couldn't " + what + ": " +
                          preflight.errorMessage + "\n");
     return null;
   }
@@ -529,7 +538,7 @@ var checkAuthThenSendRpc = function (site, operation, what) {
       } else {
         // Shouldn't ever get here because we set the retry flag on the
         // login, but just in case.
-        process.stderr.write(
+        Console.stderr.write(
 "\nYou must be logged in to " + what + " for this app. Use 'meteor login'\n" +
 "to log in.\n\n" +
 "If you don't have a Meteor developer account yet, you can quickly\n" +
@@ -537,7 +546,7 @@ var checkAuthThenSendRpc = function (site, operation, what) {
         return null;
       }
     } else { // User is logged in but not authorized for this app
-      process.stderr.write("\n");
+      Console.stderr.write("\n");
       printUnauthorizedMessage();
       return null;
     }
@@ -553,7 +562,7 @@ var checkAuthThenSendRpc = function (site, operation, what) {
   });
 
   if (result.errorMessage) {
-    process.stderr.write("Couldn't " + what + ": " +
+    Console.stderr.write("Couldn't " + what + ": " +
                          result.errorMessage + "\n");
     return null;
   }
@@ -589,7 +598,7 @@ var logs = function (site) {
   if (result === null) {
     return 1;
   } else {
-    process.stdout.write(result.message);
+    Console.stdout.write(result.message);
     auth.maybePrintRegistrationLink({ leadingNewline: true });
     return 0;
   }
@@ -606,33 +615,33 @@ var listAuthorized = function (site) {
     expectPayload: []
   });
   if (result.errorMessage) {
-    process.stderr.write("Couldn't get authorized users list: " +
+    Console.stderr.write("Couldn't get authorized users list: " +
                          result.errorMessage + "\n");
     return 1;
   }
   var info = result.payload;
 
   if (! _.has(info, 'protection')) {
-    process.stdout.write("<anyone>\n");
+    Console.stdout.write("<anyone>\n");
     return 0;
   }
 
   if (info.protection === "password") {
-    process.stdout.write("<password>\n");
+    Console.stdout.write("<password>\n");
     return 0;
   }
 
   if (info.protection === "account") {
     if (! _.has(info, 'authorized')) {
-      process.stderr.write("Couldn't get authorized users list: " +
+      Console.stderr.write("Couldn't get authorized users list: " +
                            "You are not authorized\n");
       return 1;
     }
 
-    process.stdout.write((auth.loggedInUsername() || "<you>") + "\n");
+    Console.stdout.write((auth.loggedInUsername() || "<you>") + "\n");
     _.each(info.authorized, function (username) {
       if (username)
-        process.stdout.write(username + "\n");
+        Console.stdout.write(username + "\n");
     });
     return 0;
   }
@@ -654,12 +663,12 @@ var changeAuthorized = function (site, action, username) {
   });
 
   if (result.errorMessage) {
-    process.stderr.write("Couldn't change authorized users: " +
+    Console.stderr.write("Couldn't change authorized users: " +
                          result.errorMessage + "\n");
     return 1;
   }
 
-  process.stdout.write(site + ": " +
+  Console.stdout.write(site + ": " +
                        (action === "add" ? "added " : "removed ")
                        + username + "\n");
   return 0;
@@ -680,7 +689,7 @@ var claim = function (site) {
   });
 
   if (infoResult.statusCode === 404) {
-    process.stderr.write(
+    Console.stderr.write(
 "There isn't a site deployed at that address. Use 'meteor deploy' if\n" +
 "you'd like to deploy your app here.\n");
     return 1;
@@ -688,15 +697,15 @@ var claim = function (site) {
 
   if (infoResult.payload && infoResult.payload.protection === "account") {
     if (infoResult.payload.authorized)
-      process.stderr.write("That site already belongs to you.\n");
+      Console.stderr.write("That site already belongs to you.\n");
     else
-      process.stderr.write("Sorry, that site belongs to someone else.\n");
+      Console.stderr.write("Sorry, that site belongs to someone else.\n");
     return 1;
   }
 
   if (infoResult.payload &&
       infoResult.payload.protection === "password") {
-    process.stdout.write(
+    Console.stdout.write(
 "To claim this site and transfer it to your account, enter the\n" +
 "site password one last time.\n\n");
   }
@@ -712,18 +721,18 @@ var claim = function (site) {
     auth.pollForRegistrationCompletion();
     if (! auth.loggedInUsername() &&
         auth.registrationUrl()) {
-      process.stderr.write(
+      Console.stderr.write(
 "You need to set a password on your Meteor developer account before\n" +
 "you can claim sites. You can do that here in under a minute:\n\n" +
 auth.registrationUrl() + "\n\n");
     } else {
-      process.stderr.write("Couldn't claim site: " +
+      Console.stderr.write("Couldn't claim site: " +
                            result.errorMessage + "\n");
     }
     return 1;
   }
 
-  process.stdout.write(
+  Console.stdout.write(
 site + ": " + "successfully transferred to your account.\n" +
 "\n" +
 "Show authorized users with:\n" +
@@ -747,7 +756,7 @@ var listSites = function () {
   });
 
   if (result.errorMessage) {
-    process.stderr.write("Couldn't list sites: " +
+    Console.stderr.write("Couldn't list sites: " +
                          result.errorMessage + "\n");
     return 1;
   }
@@ -755,11 +764,11 @@ var listSites = function () {
   if (! result.payload ||
       ! result.payload.sites ||
       ! result.payload.sites.length) {
-    process.stdout.write("You don't have any sites yet.\n");
+    Console.stdout.write("You don't have any sites yet.\n");
   } else {
     result.payload.sites.sort();
     _.each(result.payload.sites, function (site) {
-      process.stdout.write(site + "\n");
+      Console.stdout.write(site + "\n");
     });
   }
   return 0;

tools/http-helpers.js

@@ -13,6 +13,52 @@ var auth = require('./auth.js');
 var config = require('./config.js');
 var release = require('./release.js');
 
+
+// Helper that tracks bytes written to a writable
+var WritableWithProgress = function (writable, listener) {
+  var self = this;
+  self._inner = writable;
+  self._listener = listener;
+};
+
+_.extend(WritableWithProgress.prototype, {
+  write: function (chunk, encoding, callback) {
+    var self = this;
+    self._listener(chunk.length, false);
+    return self._inner.write(chunk, encoding);
+  },
+
+  end: function (chunk, encoding, callback) {
+    var self = this;
+    self._listener(chunk ? chunk.length : 0, true);
+    return self._inner.end(chunk, encoding);
+  },
+
+  _progress: function (n, done) {
+    var self = this;
+
+    var state = self._state;
+    state.current += n;
+    if (done) {
+      state.current.done = true;
+    }
+    self.progress.reportProgress(state);
+  },
+
+  on: function (name, callback) {
+    return this._inner.on(name, callback);
+  },
+
+  once: function () {
+    return this._inner.once.apply(this._inner, arguments);
+  },
+
+  emit: function () {
+    return this._inner.emit.apply(this._inner, arguments);
+  }
+});
+
+
 // Compose a User-Agent header.
 var getUserAgent = function () {
   var version;
@@ -85,6 +131,16 @@ _.extend(exports, {
       delete options.bodyStream;
     }
 
+    var progress = null;
+    if (_.has(options, 'progress')) {
+      progress = options.progress;
+      delete options.progress;
+
+      if (callback) {
+        throw new Error("Not safe to use progress with callback");
+      }
+    }
+
     options.headers = _.extend({
       'User-Agent': getUserAgent()
     }, options.headers || {});
@@ -161,13 +217,84 @@ _.extend(exports, {
     var request = require('request');
     var req = request(options, callback);
 
-    if (bodyStream)
+    var bodyStreamLength = 0;
-      bodyStream.pipe(req);
+    if (bodyStream) {
+      // XXX
+      bodyStreamLength += 4000000;
+    }
+    // XXX
+    var responseLength = 128 * 1024;
 
-    if (fut)
+    var totalProgress = { current: 0, end: bodyStreamLength + responseLength, done: false };
-      return fut.wait();
+
-    else
+    if (bodyStream) {
+      var dest = req;
+      if (progress) {
+        dest = new WritableWithProgress(dest, function (n, done) {
+          if (!totalProgress.done) {
+            totalProgress.current += n;
+            progress.reportProgress(totalProgress);
+          }
+        });
+      }
+      bodyStream.pipe(dest);
+    }
+
+    if (progress) {
+      httpHelpers._addProgressEvents(req);
+      req.on('progress', function (state) {
+        if (!totalProgress.done) {
+          totalProgress.current = bodyStreamLength + state.current;
+          totalProgress.end = bodyStreamLength + state.end;
+          totalProgress.done = state.done;
+
+          progress.reportProgress(totalProgress);
+        }
+      });
+    }
+
+    if (fut) {
+      try {
+        return fut.wait();
+      } finally {
+        if (progress) {
+          progress.reportProgressDone();
+        }
+      }
+    } else {
       return req;
+    }
+  },
+
+  // Adds progress callbacks to a request
+  // Based on request-progress
+  _addProgressEvents: function (request) {
+    var state;
+
+    var emitProgress = function () {
+      request.emit('progress', state);
+    };
+
+    request
+      .on('response', function (response) {
+        state = {};
+        state.end = undefined;
+        state.done = false;
+        state.current = 0;
+        var contentLength = response.headers['content-length'];
+        if (contentLength) {
+          state.end = Number(contentLength);
+        }
+        emitProgress();
+      })
+      .on('data', function (data) {
+        state.current += data.length;
+        emitProgress();
+      })
+      .on('end', function (data) {
+        state.done = true;
+        emitProgress();
+      });
   },
 
   // A synchronous wrapper around request(...) that returns the response "body"

tools/main.js

@@ -4,6 +4,7 @@ if (showRequireProfile)
 
 var _ = require('underscore');
 var Fiber = require('fibers');
+var Console = require('./console.js').Console;
 var files = require('./files.js');
 var path = require('path');
 var warehouse = require('./warehouse.js');
@@ -646,7 +647,7 @@ Fiber(function () {
     // One side effect of this: we really really expect them to all build, and
     // we're fine with dying if they don't (there's no worries about needing to
     // springboard).
-    var messages = buildmessage.capture(function () {
+    var messages = buildmessage.capture({ title: "Initializing local packages" }, function () {
       catalog.uniload.initialize({
         localPackageDirs: [path.join(files.getCurrentToolsDir(), 'packages')]
       });
@@ -671,7 +672,7 @@ Fiber(function () {
   // build anything (except maybe, if running from a checkout, packages
   // that we need to uniload, which really ought to build) so it's OK
   // to die on errors.
-  var messages = buildmessage.capture(function () {
+  var messages = buildmessage.capture({ title: "Initializing server catalog" }, function () {
     catalog.official.initialize({
       offline: !!process.env.METEOR_OFFLINE_CATALOG
     });
@@ -814,7 +815,7 @@ Fiber(function () {
 
     try {
       var rel;
-      var messages = buildmessage.capture(function () {
+      var messages = buildmessage.capture({ title: "Loading release" }, function () {
         rel = release.load(releaseName);
       });
       if (messages.hasMessages()) {
@@ -918,7 +919,7 @@ Fiber(function () {
       files.getCurrentToolsDir(), 'packages'));
   }
 
-  var messages = buildmessage.capture(function () {
+  var messages = buildmessage.capture({ title: "Initializing catalog" }, function () {
     catalog.complete.initialize({
       localPackageDirs: localPackageDirs
     });
@@ -1227,10 +1228,15 @@ commandName + ": You're not in a Meteor project directory.\n" +
   if (showRequireProfile)
     require('./profile-require.js').printReport();
 
+  Console.enableStatusPoll();
+  Console.showProgressBar();
+
   // Run the command!
   try {
     var ret = command.func(options);
   } catch (e) {
+    Console.hideProgressBar();
+
     if (e === main.ShowUsage || e === main.WaitForExit ||
         e === main.SpringboardToLatestRelease ||
         e === main.SpringboardToSpecificReleaseg ||
@@ -1278,6 +1284,8 @@ commandName + ": You're not in a Meteor project directory.\n" +
     }
   }
 
+  Console.hideProgressBar();
+
   // Exit. (We will not get here if the command threw an exception
   // such as main.WaitForExit).
   if (ret === undefined)

tools/package-client.js

@@ -195,6 +195,14 @@ var writePackageDataToDisk = function (syncToken, data, options) {
 //  - useShortPages: Boolean. Request short pages of ~3 records from the
 //    server, instead of ~100 that it would send otherwise
 exports.updateServerPackageData = function (cachedServerData, options) {
+  var results;
+  buildmessage.capture({ title: 'Updating package catalog' }, function () {
+    results = _updateServerPackageData(cachedServerData, options);
+  });
+  return results;
+};
+
+_updateServerPackageData = function (cachedServerData, options) {
   var self = this;
   options = options || {};
   cachedServerData = cachedServerData || emptyCachedServerDataJson();
@@ -202,6 +210,10 @@ exports.updateServerPackageData = function (cachedServerData, options) {
   var done = false;
   var ret = {resetData: false};
 
+  var start = undefined;
+  var state = { current: 0, end: 10, done: false};
+  buildmessage.reportProgress(state);
+
   try {
     var conn = openPackageServerConnection(options.packageServerUrl);
   } catch (err) {
@@ -210,8 +222,22 @@ exports.updateServerPackageData = function (cachedServerData, options) {
     return ret;
   }
 
+  // Provide some progress indication for connection
+  // XXX though it is just a hack
+  state.current = 1;
+  buildmessage.reportProgress(state);
+
   var getSomeData = function () {
     var syncToken = cachedServerData.syncToken;
+
+    if (!start) {
+      start = syncToken.packages;
+      state.end = Date.now() - start;
+    }
+    // XXX: Is packages the best progress indicator?
+    state.current = syncToken.packages - start;
+    buildmessage.reportProgress(state);
+
     var remoteData;
     try {
       remoteData = loadRemotePackageData(conn, syncToken, {
@@ -272,6 +298,9 @@ exports.updateServerPackageData = function (cachedServerData, options) {
     conn.close();
   }
 
+  state.done = true;
+  buildmessage.reportProgress(state);
+
   ret.data = cachedServerData;
   return ret;
 };

tools/progress.js

@@ -0,0 +1,278 @@
+///
+/// utility functions for computing progress of complex tasks
+///
+/// State callback here is an object with these keys:
+///   done: bool, true if done
+///   current: number, the current progress value
+///   end: number, optional, the value of current where we expect to be done
+///
+
+var _ = require('underscore');
+var Future = require('fibers/future');
+var console = require('./console.js');
+
+var Progress = function (options) {
+  var self = this;
+
+  options = options || {};
+
+  self._lastState = null;
+  self._parent = options.parent;
+  self._watchers = options.watchers || [];
+
+  self._title = options.title;
+
+  //if (!self._title && self._parent) {
+  //  throw new Error("No title passed");
+  //}
+  self._forkJoin = options.forkJoin;
+
+  // XXX: Rationalize this; we probably don't need _completedChildren
+  // XXX: or _activeChildTasks (?)
+  self._completedChildren = { current: 0, end: 0};
+  self._activeChildTasks = [];
+  self._allTasks = [];
+
+  self._selfState = { current: 0, end: 0, done: false };
+  if (options.estimate) {
+    self._selfState.end = options.estimate;
+  }
+  self._state = _.clone(self._selfState);
+
+  self._isDone = false;
+
+  self._selfActive = false;
+};
+
+_.extend(Progress.prototype, {
+  toString: function() {
+    var self = this;
+    return "Progress [state=" + JSON.stringify(self._state) + "]";
+  },
+
+  reportProgressDone: function () {
+    var self = this;
+
+    var state = _.clone(self._selfState);
+    state.done = true;
+    if (state.current === 0) {
+      state.current = 1;
+    }
+    if (!state.end || state.end < state.current) {
+      state.end = state.current;
+    }
+    self.reportProgress(state);
+  },
+
+  // Tries to determine which is the 'current' job in the tree
+  // This is very heuristical... we use some hints, like:
+  // don't descend into fork-join jobs; we know these execute concurrently,
+  // so we assume the top-level task has the title
+  // i.e. "Downloading packages", not "downloading supercool-1.0"
+  getCurrentProgress: function () {
+    var self = this;
+
+    var isRoot = !self._parent;
+
+    if (self._isDone) {
+      return null;
+    }
+
+    if (self._selfActive && !isRoot) {
+      return self;
+    }
+
+    if (self._forkJoin) {
+      // Don't descend into fork-join tasks
+      return self;
+    }
+
+    if (self._allTasks.length) {
+      var active = _.map(self._allTasks, function (task) {
+        return task.getCurrentProgress();
+      });
+      active = _.filter(active, function (s) {
+        return !!s;
+      });
+      if (active.length == 1) {
+        return active[0];
+      }
+      return self;
+    }
+
+    //if (self._activeChildTasks.length) {
+    //  var titles = _.map(self._activeChildTasks, function (task) {
+    //    return task.getCurrent();
+    //  });
+    //  titles = _.filter(titles, function (s) { return !!s; });
+    //  if (titles.length == 1) {
+    //    return titles[0];
+    //  }
+    //  //if (titles.length > 1) {
+    //  //  console.log("Multiple titles: " + titles);
+    //  //}
+    //  return self._title;
+    //}
+
+    return null;
+  },
+
+  // Creates a subtask that must be completed as part of this (bigger) task
+  addChildTask: function (options) {
+    var self = this;
+    options = options || {};
+    var options = _.extend({ parent: self }, options);
+    var child = new Progress(options);
+    self._activeChildTasks.push(child);
+    self._allTasks.push(child);
+    self._reportChildState(child, child._state);
+    return child;
+  },
+
+  // Dumps the tree, for debug
+  dump: function (stream, options, prefix) {
+    var self = this;
+
+    options = options || {};
+    if (options.skipDone && self._isDone) {
+      return;
+    }
+
+    if (prefix) {
+      stream.write(prefix);
+    }
+    var end = self._state.end;
+    if (!end) {
+      end = '?';
+    }
+    stream.write("Task [" + self._title + "] " + self._state.current + "/" + end
+      + (self._isDone ? " done" : "")
+      + (self._selfActive ? " active" : "") +"\n");
+    if (self._allTasks.length) {
+      _.each(self._allTasks, function (child) {
+        child.dump(stream, options, (prefix || '') + '  ');
+      });
+    }
+  },
+
+  // Receives a state report indicating progress of self
+  reportProgress: function (state) {
+    var self = this;
+
+    self._selfState = state;
+    self._selfActive = !state.done;
+
+    self._updateTotalState();
+
+    console.Console.statusPollMaybe();
+
+    self._notifyState();
+  },
+
+  // Subscribes a watcher to changes
+  addWatcher: function (watcher) {
+    var self = this;
+
+    self._watchers.push(watcher);
+  },
+
+  // Notifies watchers & parents
+  _notifyState: function () {
+    var self = this;
+
+    if (self._parent) {
+      self._parent._reportChildState(self, self._state);
+    }
+
+    if (self._watchers.length) {
+      _.each(self._watchers, function (watcher) {
+        watcher(self._state);
+      });
+    }
+  },
+
+  // Recomputes state, incorporating children's states
+  _updateTotalState: function () {
+    var self = this;
+
+    var state = _.clone(self._selfState);
+
+    //state.current += self._completedChildren.current;
+    //if (state.end !== undefined) {
+    //  state.end += self._completedChildren.end;
+    //}
+
+    //var allChildrenDone = true;
+    //_.each(self._activeChildTasks, function (child) {
+    //  var childState = child._state;
+    //  state.current += childState.current;
+    //  if (!state.done) {
+    //    allChildrenDone = false;
+    //  }
+    //
+    //  if (state.done) {
+    //    if (state.end !== undefined) {
+    //      state.end += childState.current;
+    //    }
+    //  } else if (state.end !== undefined) {
+    //    if (childState.end !== undefined) {
+    //      state.end += childState.end;
+    //    } else {
+    //      state.end = undefined;
+    //    }
+    //  }
+    //});
+    //if (!allChildrenDone) {
+    //  state.done = false;
+    //}
+
+    var allChildrenDone = true;
+    var state = _.clone(self._selfState);
+    _.each(self._allTasks, function (child) {
+      var childState = child._state;
+
+      if (!child._isDone) {
+        allChildrenDone = false;
+      }
+
+      state.current += childState.current;
+      if (state.end !== undefined) {
+        if (childState.done) {
+          state.end += childState.current;
+        } else if (childState.end !== undefined) {
+          state.end += childState.end;
+        } else {
+          state.end = undefined;
+        }
+      }
+    });
+    self._isDone = allChildrenDone && !self._selfActive;
+    if (!allChildrenDone) {
+      state.done = false;
+    }
+
+    if (!state.done && self._state.done) {
+      // This shouldn't happen
+      throw new Error("Progress transition from done => !done");
+    }
+
+    self._state = state;
+  },
+
+  // Called by a child when its state changes
+  _reportChildState: function (child, state) {
+    var self = this;
+
+    if (state.done) {
+      self._activeChildTasks = _.without(self._activeChildTasks, child);
+      var weight = state.current;
+      self._completedChildren.current += weight;
+      self._completedChildren.end += weight;
+    }
+
+    self._updateTotalState();
+    self._notifyState();
+  }
+});
+
+exports.Progress = Progress;

tools/project.js

@@ -12,6 +12,7 @@ var buildmessage = require('./buildmessage.js');
 var packageLoader = require('./package-loader.js');
 var PackageSource = require('./package-source.js');
 var packageVersionParser = require('./package-version-parser.js');
+var Console = require('./console.js').Console;
 
 var project = exports;
 
@@ -220,9 +221,9 @@ _.extend(Project.prototype, {
       } catch (err) {
         // XXX This error handling is bogus. Use buildmessage instead, or
         // something. See also compiler.determineBuildTimeDependencies
-        process.stdout.write(
+        Console.warn(
           "Could not resolve the specified constraints for this project:\n"
-           + (err.constraintSolverError ? err : err.stack) + "\n");
+           + (err.constraintSolverError ? err : err.stack));
         process.exit(1);
       }
 
@@ -236,8 +237,8 @@ _.extend(Project.prototype, {
       });
 
       if (!setV.success) {
-        process.stdout.write(
+        Console.warn(
-          "Could not install all the requested packages.\n");
+          "Could not install all the requested packages.");
         process.exit(1);
       }
 
@@ -363,9 +364,9 @@ _.extend(Project.prototype, {
            options.onDiskPackages[packageName] !== version)) {
         // XXX maybe we shouldn't be letting the constraint solver choose
         // things that don't have the right arches?
-        process.stderr.write("Package " + packageName +
+        Console.warn("Package " + packageName +
                              " has no compatible build for version " +
-                             version + "\n");
+                             version);
         failed = true;
         return;
       }
@@ -396,7 +397,7 @@ _.extend(Project.prototype, {
     if ((!self.muted && !_.isEmpty(versions))
         || options.alwaysShow) {
       _.each(messageLog, function (msg) {
-        process.stdout.write(msg + "\n");
+        Console.info(msg);
       });
 
       // Pay special attention to non-backwards-compatible changes.
@@ -448,11 +449,11 @@ _.extend(Project.prototype, {
       });
 
       if (!_.isEmpty(incompatibleUpdates)) {
-        process.stderr.write(
+        Console.warn(
           "\nThe following packages have been updated to new versions that are not " +
-            "backwards compatible:\n");
+            "backwards compatible:");
-        process.stderr.write(utils.formatList(incompatibleUpdates));
+        Console.warn(utils.formatList(incompatibleUpdates));
-        process.stderr.write("\n");
+        Console.warn("\n");
       };
     }
     return 0;

tools/tropohouse.js

@@ -152,11 +152,24 @@ _.extend(exports.Tropohouse.prototype, {
   downloadBuildToTempDir: function (versionInfo, buildRecord) {
     var self = this;
     var targetDirectory = files.mkdtemp();
-    var packageTarball = httpHelpers.getUrl({
+
-      url: buildRecord.build.url,
+    var url = buildRecord.build.url;
-      encoding: null
+
-    });
+    var progress = buildmessage.addChildTracker("Download build");
-    files.extractTarGz(packageTarball, targetDirectory);
+    try {
+      buildmessage.capture({}, function () {
+        var packageTarball = httpHelpers.getUrl({
+          url: url,
+          encoding: null,
+          progress: progress,
+          wait: false
+        });
+        files.extractTarGz(packageTarball, targetDirectory);
+      });
+    } finally {
+      progress.reportProgressDone();
+    }
+
     return targetDirectory;
   },
 
@@ -239,21 +252,9 @@ _.extend(exports.Tropohouse.prototype, {
       throw e;
     }
 
-    // XXX replace with a real progress bar in downloadMissingPackages
+    buildmessage.enterJob({
-    var header = "  downloading " + packageName + " at version " + version + " ...";
+      title: "  downloading " + packageName + " at version " + version + " ...",
-    if (!options.silent) {
+    }, function() {
-        var animationFrame = 0;
-        var spinner = ['..-', '..\\', '..|', '../'];
-
-        var printUpdate = function () {
-          process.stderr.write(header + spinner[animationFrame] + "\r");
-          animationFrame = (animationFrame + 1) % spinner.length;
-        };
-        printUpdate();
-        var dlTimer = setInterval(printUpdate, 200);
-    }
-
-    try {
       var buildTempDirs = [];
       // If there's already a package in the tropohouse, start with it.
       if (packageLinkTarget) {
@@ -263,8 +264,7 @@ _.extend(exports.Tropohouse.prototype, {
       // XXX how does concurrency work here?  we could just get errors if we try
       // to rename over the other thing?  but that's the same as in warehouse?
       _.each(buildsToDownload, function (build) {
-        buildTempDirs.push(self.downloadBuildToTempDir(
+        buildTempDirs.push(self.downloadBuildToTempDir({packageName: packageName, version: version}, build));
-          {packageName: packageName, version: version}, build));
       });
 
       // We need to turn our builds into a single unipackage.
@@ -290,12 +290,7 @@ _.extend(exports.Tropohouse.prototype, {
       if (packageLinkTarget) {
         files.rm_recursive(self.packagePath(packageName, packageLinkTarget));
       }
-    } finally {
+    });
-      if (!options.silent) {
-        clearInterval(dlTimer);
-        process.stderr.write(header + " done\n");
-      }
-    }
 
     return;
   },
@@ -316,7 +311,8 @@ _.extend(exports.Tropohouse.prototype, {
     options = options || {};
     var serverArch = options.serverArch || archinfo.host();
     var downloadedPackages = {};
-    _.each(versionMap, function (version, name) {
+    buildmessage.forkJoin({ title: 'Downloading packages'},
+      versionMap, function (version, name) {
       try {
         self.maybeDownloadPackageForArchitectures({
           packageName: name,