meteor/tools/tool-env/isopackets.js

var assert = require('assert');
var _ = require('underscore');

var bundler = require('../isobuild/bundler.js');
import Builder from '../isobuild/builder.js';
var compiler = require('../isobuild/compiler.js');
var isopackCacheModule = require('../isobuild/isopack-cache.js');

var buildmessage = require('../utils/buildmessage.js');
var files = require('../fs/files.js');
var config = require('../meteor-services/config.js');
var watch = require('../fs/watch.js');
var Console = require('../console/console.js').Console;
var fiberHelpers = require('../utils/fiber-helpers.js');
var packageMapModule = require('../packaging/package-map.js');
var archinfo = require('../utils/archinfo.js');
var Profile = require('./profile.js').Profile;

// TL;DR: Isopacket is a set of isopacks. Isopackets are used only inside
// meteor-tool.

// An isopacket is a predefined set of isopackages which the meteor command-line
// tool can load into its process. This is how we use the DDP client and many
// other packages inside the tool. The isopackets are listed below in the
// ISOPACKETS constant.
//
// All packages that are in isopackets and all of their transitive dependencies
// must be part of the core Meteor git checkout (not loaded from troposphere).
//
// The requested packages will be loaded together with all of their
// dependencies. If you request to load the same isopacket more than once, you
// will efficiently get the same pre-loaded isopacket. On the other hand, two
// different loaded isopackets contain distinct copies of all of their packages
// copy of all of the packages. The return value is an object that maps package
// name to package exports (that is, it is the Package object from inside the
// sandbox created for the newly loaded packages).
//
// For built releases, all of the isopackets are pre-compiled (as JsImages,
// similar to a plugin or a server program) into the tool.
//
// When run from a checkout, all isopackets are re-compiled early in the startup
// process if any of their sources have changed.
//
// Example usage:
//   var DDP = require('./isopackets.js').loadIsopackage('ddp-client').DDP;
//   var reverse = DDP.connect('reverse.meteor.com');
//   Console.info(reverse.call('reverse', 'hello world'));


// All of the defined isopackets. Whenever they are being built, they will be
// built in the order listed here.
export const ISOPACKETS = {
  // These packages used to be divided up into distinct isopackets, but
  // that resulted in extremely wasteful duplication of transitive
  // dependencies, so now we have only one isopacket that combines all the
  // dependencies of every former isopacket.
  combined: [
    // ddp
    'ddp-client',
    // mongo
    'npm-mongo',
    // ejson
    'ejson',
    // constraint-solver
    'constraint-solver',
    // cordova-support
    'boilerplate-generator',
    'webapp-hashing',
    'xmlbuilder',
    // cordova-support, logging
    'logging',
    // support for childProcess.sendMessage(topic, payload)
    'inter-process-messaging',
  ]
};

// Caches isopackets in memory (each isopacket only needs to be loaded
// once).  This is a map from isopacket name to either:
//
//  - The 'Package' dictionary, if the isopacket has already been loaded
//    into memory
//  - null, if the isopacket hasn't been loaded into memory but its on-disk
//    instance is known to be ready
//
// The subtlety here is that when running from a checkout, we don't want to
// accidentally load an isopacket before ensuring that it doesn't need to be
// rebuilt. We used to need to load a "js-analyze" isopacket as part
// of building other isopackets in ensureIsopacketsLoadable which made this
// more important, though we've simplified it now by moving that code into
// the tool itself.
var loadedIsopackets = {};

// The main entry point: loads the specified isopacket ("combined" by
// default) from cache or from disk, and returns the requested package
// dependency, complaining if the package does not exist. Note that
// ensureIsopacketsLoadable must be called first, as this function does
// not trigger any building.
export function loadIsopackage(packageName, isopacketName = "combined") {
  // Small but necessary hack: because archinfo.host() calls execFileSync,
  // it yields the first time we call it, which is a problem for the
  // fiberHelpers.noYieldsAllowed block below. Calling it here ensures the
  // result is cached, so no yielding occurs later.
  assert.strictEqual(archinfo.host().split(".", 1)[0], "os");

  const isopacket = fiberHelpers.noYieldsAllowed(function () {
    if (_.has(loadedIsopackets, isopacketName)) {
      if (loadedIsopackets[isopacketName]) {
        return loadedIsopackets[isopacketName];
      }

      // This is the case where the isopacket is up to date on disk but not
      // loaded.
      return loadedIsopackets[isopacketName] =
        loadIsopacketFromDisk(isopacketName);
    }

    if (_.has(ISOPACKETS, isopacketName)) {
      throw Error("Can't load isopacket before it has been verified: "
                  + isopacketName);
    }

    throw Error("Unknown isopacket: " + isopacketName);
  });

  if (! _.has(isopacket, packageName)) {
    throw new Error("Unknown isopacket dependency: " + packageName);
  }

  return isopacket[packageName];
}

var isopacketPath = function (isopacketName) {
  return files.pathJoin(config.getIsopacketRoot(), isopacketName);
};

// ensureIsopacketsLoadable is called at startup and ensures that all isopackets
// exist on disk as up-to-date loadable programs.
var calledEnsure = false;
export function ensureIsopacketsLoadable() {
  if (calledEnsure) {
    throw Error("can't ensureIsopacketsLoadable twice!");
  }
  calledEnsure = true;

  // If we're not running from checkout, then there's nothing to build and we
  // can declare that all isopackets are loadable.
  if (! files.inCheckout()) {
    _.each(ISOPACKETS, function (packages, name) {
      loadedIsopackets[name] = null;
    });
    return;
  }

  // We make this object lazily later.
  var isopacketBuildContext = null;

  var failedPackageBuild = false;
  // Look at each isopacket. Check to see if it's on disk and up to date. If
  // not, build it. We rebuild them in the order listed in ISOPACKETS.
  var messages = Console.withProgressDisplayVisible(function () {
    return buildmessage.capture(function () {
      _.each(ISOPACKETS, function (packages, isopacketName) {
        if (failedPackageBuild) {
          return;
        }

        var isopacketRoot = isopacketPath(isopacketName);
        var existingBuildinfo = files.readJSONOrNull(
          files.pathJoin(isopacketRoot, 'isopacket-buildinfo.json'));
        var needRebuild = ! existingBuildinfo;
        if (! needRebuild && existingBuildinfo.builtBy !== compiler.BUILT_BY) {
          needRebuild = true;
        }
        if (! needRebuild) {
          var watchSet = watch.WatchSet.fromJSON(existingBuildinfo.watchSet);
          if (! watch.isUpToDate(watchSet)) {
            needRebuild = true;
          }
        }
        if (! needRebuild) {
          // Great, it's loadable without a rebuild.
          loadedIsopackets[isopacketName] = null;
          return;
        }

        // We're going to need to build! Make a catalog and loader if we haven't
        // yet.
        if (! isopacketBuildContext) {
          isopacketBuildContext = makeIsopacketBuildContext();
        }

        buildmessage.enterJob({
          title: "bundling " + isopacketName + " packages for the tool"
        }, function () {
          // Build the packages into the in-memory IsopackCache.
          isopacketBuildContext.isopackCache.buildLocalPackages(packages);
          if (buildmessage.jobHasMessages()) {
            return;
          }

          // Now bundle them into a program.
          var built = bundler.buildJsImage({
            name: "isopacket-" + isopacketName,
            packageMap: isopacketBuildContext.packageMap,
            isopackCache: isopacketBuildContext.isopackCache,
            use: packages
          });
          if (buildmessage.jobHasMessages()) {
            return;
          }

          var builder = new Builder({outputPath: isopacketRoot});
          builder.writeJson('isopacket-buildinfo.json', {
            builtBy: compiler.BUILT_BY,
            watchSet: built.watchSet.toJSON()
          });
          built.image.write(builder);
          builder.complete();
          // It's loadable now.
          loadedIsopackets[isopacketName] = null;
        });
      });
    });
  });

  // This is a build step ... but it's one that only happens in development, so
  // it can just crash the app instead of being handled nicely.
  if (messages.hasMessages()) {
    Console.error("Errors prevented isopacket build:");
    Console.printMessages(messages);
    throw new Error("isopacket build failed?");
  }
}

// Returns a new all-local-packages catalog to be used for building isopackets.
var newIsopacketBuildingCatalog = function () {
  if (! files.inCheckout()) {
    throw Error("No need to build isopackets unless in checkout!");
  }

  var catalogLocal = require('../packaging/catalog/catalog-local.js');
  var isopacketCatalog = new catalogLocal.LocalCatalog;
  var messages = buildmessage.capture(
    { title: "scanning local core packages" },
    function () {
      const packagesDir =
        files.pathJoin(files.getCurrentToolsDir(), 'packages');

      // When running from a checkout, isopacket building does use local
      // packages, but *ONLY THOSE FROM THE CHECKOUT*: not app packages or
      // $PACKAGE_DIRS packages.  One side effect of this: we really really
      // expect them to all build, and we're fine with dying if they don't
      // (there's no worries about needing to springboard).
      isopacketCatalog.initialize({
        localPackageSearchDirs: [
          packagesDir,
          files.pathJoin(packagesDir, "non-core", "*", "packages"),
        ],
        buildingIsopackets: true
      });
    });
  if (messages.hasMessages()) {
    Console.arrowError("Errors while scanning core packages:");
    Console.printMessages(messages);
    throw new Error("isopacket scan failed?");
  }
  return isopacketCatalog;
};

export function makeIsopacketBuildContext() {
  var context = {};
  var catalog = newIsopacketBuildingCatalog();
  var versions = {};
  _.each(catalog.getAllPackageNames(), function (packageName) {
    versions[packageName] = catalog.getLatestVersion(packageName).version;
  });
  context.packageMap = new packageMapModule.PackageMap(versions, {
    localCatalog: catalog
  });
  // Make an isopack cache that doesn't save isopacks to disk and has no
  // access to versioned packages.
  context.isopackCache = new isopackCacheModule.IsopackCache({
    packageMap: context.packageMap,
    includeCordovaUnibuild: false,
    // When linking JS files, don't include the padding spaces and line number
    // comments. Since isopackets are loaded as part of possibly very short
    // 'meteor' tool command invocations, we care more about startup time than
    // legibility, and the difference is actually observable (eg 25% speedup
    // loading constraint-solver).
    noLineNumbers: true
  });
  return context;
}

// Loads a built isopacket from disk. Always loads (the cache is in 'load', not
// this function). Does not run a build process; it must already be built.
var loadIsopacketFromDisk = function (isopacketName) {
  var image = bundler.readJsImage(
    files.pathJoin(isopacketPath(isopacketName), 'program.json'));

  // An incredibly minimalist version of the environment from
  // tools/server/boot.js.  Kind of a hack.
  var env = {
    __meteor_bootstrap__: { startupHooks: [] },
    __meteor_runtime_config__: { meteorRelease: "ISOPACKET" }
  };
  env.Profile = Profile;

  var ret;
  var messages = buildmessage.capture({
    title: "loading isopacket `" + isopacketName + "`"
  }, function () {
    ret = image.load(env);
  });

  // This is a build step ... but it's one that only happens in development, so
  // it can just crash the app instead of being handled nicely.
  if (messages.hasMessages()) {
    Console.error("Errors prevented isopacket load:");
    Console.printMessages(messages);
    throw new Error("isopacket load failed?");
  }

  // Run any user startup hooks.
  while (env.__meteor_bootstrap__.startupHooks.length) {
    var hook = env.__meteor_bootstrap__.startupHooks.shift();
    hook();
  }
  // Setting this to null tells Meteor.startup to call hooks immediately.
  env.__meteor_bootstrap__.startupHooks = null;

  return ret;
};