meteor/tools/isopackets.js

var _ = require('underscore');
var bundler = require('./bundler.js');
var Builder = require('./builder.js');
var buildmessage = require('./buildmessage.js');
var files = require('./files.js');
var compiler = require('./compiler.js');
var config = require('./config.js');
var watch = require('./watch.js');
var Console = require('./console.js').Console;
var isopackCacheModule = require('./isopack-cache.js');
var packageMapModule = require('./package-map.js');
var fiberHelpers = require('./fiber-helpers.js');

// TL;DR: Isopacket is a set of isopacks. Isopackets are used only inside
// meteor-tool.

// An isopacket is a predefined set of isopackages which the meteor command-line
// tool can load into its process. This is how we use the DDP client and many
// other packages inside the tool. The isopackets are listed below in the
// ISOPACKETS constant.
//
// All packages that are in isopackets and all of their transitive dependencies
// must be part of the core Meteor git checkout (not loaded from troposphere).
//
// The requested packages will be loaded together with all of their
// dependencies. If you request to load the same isopacket more than once, you
// will efficiently get the same pre-loaded isopacket. On the other hand, two
// different loaded isopackets contain distinct copies of all of their packages
// copy of all of the packages. The return value is an object that maps package
// name to package exports (that is, it is the Package object from inside the
// sandbox created for the newly loaded packages).
//
// For built releases, all of the isopackets are pre-compiled (as JsImages,
// similar to a plugin or a server program) into the tool.
//
// When run from a checkout, all isopackets are re-compiled early in the startup
// process if any of their sources have changed.
//
// Example usage:
//   var DDP = require('./isopackets.js').load('ddp')['ddp-client'].DDP;
//   var reverse = DDP.connect('reverse.meteor.com');
//   Console.info(reverse.call('reverse', 'hello world'));


// All of the defined isopackets. Whenever they are being built, they will be
// built in the order listed here (which is mostly relevant for js-analyze).
var ISOPACKETS = {
  // Note: when running from a checkout, js-analyze must always be the
  // the first to be rebuilt, because it might need to be loaded as part
  // of building other isopackets.
  'js-analyze': ['js-analyze'],
  'ddp': ['ddp-client'],
  'mongo': ['npm-mongo'],
  'ejson': ['ejson'],
  'minifiers': ['minifiers'],
  'constraint-solver': ['constraint-solver'],
  'cordova-support': ['boilerplate-generator', 'logging', 'webapp-hashing',
                      'xmlbuilder'],
  'logging': ['logging']
};

// Caches isopackets in memory (each isopacket only needs to be loaded
// once).  This is a map from isopacket name to either:
//
//  - The 'Package' dictionary, if the isopacket has already been loaded
//    into memory
//  - null, if the isopacket hasn't been loaded into memory but its on-disk
//    instance is known to be ready
//
// The subtlety here is that when running from a checkout, we don't want to
// accidentally load an isopacket before ensuring that it doesn't need to be
// rebuilt. But we do want to be able to load the js-analyze isopacket as part
// of building other isopackets in ensureIsopacketsLoadable.
var loadedIsopackets = {};

// The main entry point: loads and returns an isopacket from cache or from
// disk. Does not do a build step: ensureIsopacketsLoadable must be called
// first!
var load = function (isopacketName) {
  return fiberHelpers.noYieldsAllowed(function () {
    if (_.has(loadedIsopackets, isopacketName)) {
      if (loadedIsopackets[isopacketName]) {
        return loadedIsopackets[isopacketName];
      }

      // This is the case where the isopacket is up to date on disk but not
      // loaded.
      var isopacket = loadIsopacketFromDisk(isopacketName);
      loadedIsopackets[isopacketName] = isopacket;
      return isopacket;
    }

    if (_.has(ISOPACKETS, isopacketName)) {
      throw Error("Can't load isopacket before it has been verified: "
                  + isopacketName);
    }

    throw Error("Unknown isopacket: " + isopacketName);
  });
};

var isopacketPath = function (isopacketName) {
  return files.pathJoin(config.getIsopacketRoot(), isopacketName);
};

// ensureIsopacketsLoadable is called at startup and ensures that all isopackets
// exist on disk as up-to-date loadable programs.
var calledEnsure = false;
var ensureIsopacketsLoadable = function () {
  if (calledEnsure) {
    throw Error("can't ensureIsopacketsLoadable twice!");
  }
  calledEnsure = true;

  // If we're not running from checkout, then there's nothing to build and we
  // can declare that all isopackets are loadable.
  if (! files.inCheckout()) {
    _.each(ISOPACKETS, function (packages, name) {
      loadedIsopackets[name] = null;
    });
    return;
  }

  // We make this object lazily later.
  var isopacketBuildContext = null;

  var failedPackageBuild = false;
  // Look at each isopacket. Check to see if it's on disk and up to date. If
  // not, build it. We rebuild them in the order listed in ISOPACKETS, which
  // ensures that we deal with js-analyze first.
  var messages = Console.withProgressDisplayVisible(function () {
    return buildmessage.capture(function () {
      _.each(ISOPACKETS, function (packages, isopacketName) {
        if (failedPackageBuild)
          return;

        var isopacketRoot = isopacketPath(isopacketName);
        var existingBuildinfo = files.readJSONOrNull(
          files.pathJoin(isopacketRoot, 'isopacket-buildinfo.json'));
        var needRebuild = ! existingBuildinfo;
        if (! needRebuild && existingBuildinfo.builtBy !== compiler.BUILT_BY) {
          needRebuild = true;
        }
        if (! needRebuild) {
          var watchSet = watch.WatchSet.fromJSON(existingBuildinfo.watchSet);
          if (! watch.isUpToDate(watchSet)) {
            needRebuild = true;
          }
        }
        if (! needRebuild) {
          // Great, it's loadable without a rebuild.
          loadedIsopackets[isopacketName] = null;
          return;
        }

        // We're going to need to build! Make a catalog and loader if we haven't
        // yet.
        if (! isopacketBuildContext) {
          isopacketBuildContext = makeIsopacketBuildContext();
        }

        buildmessage.enterJob({
          title: "bundling " + isopacketName + " packages for the tool"
        }, function () {
          // Build the packages into the in-memory IsopackCache.
          isopacketBuildContext.isopackCache.buildLocalPackages(packages);
          if (buildmessage.jobHasMessages())
            return;

          // Now bundle them into a program.
          var built = bundler.buildJsImage({
            name: "isopacket-" + isopacketName,
            packageMap: isopacketBuildContext.packageMap,
            isopackCache: isopacketBuildContext.isopackCache,
            use: packages
          });
          if (buildmessage.jobHasMessages())
            return;

          var builder = new Builder({outputPath: isopacketRoot});
          builder.writeJson('isopacket-buildinfo.json', {
            builtBy: compiler.BUILT_BY,
            watchSet: built.watchSet.toJSON()
          });
          built.image.write(builder);
          builder.complete();
          // It's loadable now.
          loadedIsopackets[isopacketName] = null;
        });
      });
    });
  });

  // This is a build step ... but it's one that only happens in development, so
  // it can just crash the app instead of being handled nicely.
  if (messages.hasMessages()) {
    Console.error("Errors prevented isopacket build:");
    Console.printMessages(messages);
    throw new Error("isopacket build failed?");
  }
};

// Returns a new all-local-packages catalog to be used for building isopackets.
var newIsopacketBuildingCatalog = function () {
  if (! files.inCheckout())
    throw Error("No need to build isopackets unless in checkout!");

  var catalogLocal = require('./catalog-local.js');
  var isopacketCatalog = new catalogLocal.LocalCatalog;
  var messages = buildmessage.capture(
    { title: "scanning local core packages" },
    function () {
      // When running from a checkout, isopacket building does use local
      // packages, but *ONLY THOSE FROM THE CHECKOUT*: not app packages or
      // $PACKAGE_DIRS packages.  One side effect of this: we really really
      // expect them to all build, and we're fine with dying if they don't
      // (there's no worries about needing to springboard).
      isopacketCatalog.initialize({
        localPackageSearchDirs: [files.pathJoin(
          files.getCurrentToolsDir(), 'packages')],
        buildingIsopackets: true
      });
    });
  if (messages.hasMessages()) {
    Console.arrowError("Errors while scanning core packages:");
    Console.printMessages(messages);
    throw new Error("isopacket scan failed?");
  }
  return isopacketCatalog;
};

var makeIsopacketBuildContext = function () {
  var context = {};
  var catalog = newIsopacketBuildingCatalog();
  var versions = {};
  _.each(catalog.getAllPackageNames(), function (packageName) {
    versions[packageName] = catalog.getLatestVersion(packageName).version;
  });
  context.packageMap = new packageMapModule.PackageMap(versions, {
    localCatalog: catalog
  });
  // Make an isopack cache that doesn't save isopacks to disk and has no
  // access to versioned packages.
  context.isopackCache = new isopackCacheModule.IsopackCache({
    packageMap: context.packageMap,
    includeCordovaUnibuild: false,
    // When linking JS files, don't include the padding spaces and line number
    // comments. Since isopackets are loaded as part of possibly very short
    // 'meteor' tool command invocations, we care more about startup time than
    // legibility, and the difference is actually observable (eg 25% speedup
    // loading constraint-solver).
    noLineNumbers: true
  });
  return context;
};

// Loads a built isopacket from disk. Always loads (the cache is in 'load', not
// this function). Does not run a build process; it must already be built.
var loadIsopacketFromDisk = function (isopacketName) {
  var image = bundler.readJsImage(
    files.pathJoin(isopacketPath(isopacketName), 'program.json'));

  // An incredibly minimalist version of the environment from
  // tools/server/boot.js.  Kind of a hack.
  var env = {
    __meteor_bootstrap__: { startupHooks: [] },
    __meteor_runtime_config__: { meteorRelease: "ISOPACKET" }
  };

  var ret;
  var messages = buildmessage.capture({
    title: "loading isopacket `" + isopacketName + "`"
  }, function () {
    ret = image.load(env);
  });

  // This is a build step ... but it's one that only happens in development, so
  // it can just crash the app instead of being handled nicely.
  if (messages.hasMessages()) {
    Console.error("Errors prevented isopacket load:");
    Console.printMessages(messages);
    throw new Error("isopacket load failed?");
  }

  // Run any user startup hooks.
  while (env.__meteor_bootstrap__.startupHooks.length) {
    var hook = env.__meteor_bootstrap__.startupHooks.shift();
    hook();
  }
  // Setting this to null tells Meteor.startup to call hooks immediately.
  env.__meteor_bootstrap__.startupHooks = null;

  return ret;
};

var isopackets = exports;
_.extend(exports, {
  load: load,
  ensureIsopacketsLoadable: ensureIsopacketsLoadable,
  ISOPACKETS: ISOPACKETS,
  makeIsopacketBuildContext: makeIsopacketBuildContext
});