From 4000dfb993a97c6741869b78ab64a9182aad659d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 6 Nov 2014 12:34:02 -0800 Subject: [PATCH] from-checkout builds isopackets on startup Nothing actually LOADS from isopackets yet. --- tools/config.js | 8 +++ tools/files.js | 13 +++++ tools/main.js | 4 +- tools/uniload.js | 130 +++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 143 insertions(+), 12 deletions(-) diff --git a/tools/config.js b/tools/config.js index 22fbbeed32..232c42268d 100644 --- a/tools/config.js +++ b/tools/config.js @@ -236,6 +236,14 @@ _.extend(exports, { self.getLocalPackageCacheFilename()); }, + getIsopacketRoot: function () { + if (files.inCheckout()) { + return path.join(files.getCurrentToolsDir(), '.meteor', 'isopackets'); + } else { + return path.join(files.getCurrentToolsDir(), 'isopackets'); + } + }, + getBannersShownFilename: function() { return path.join(tropohouse.default.root, "package-metadata", "v1.1", "banners-shown.json"); diff --git a/tools/files.js b/tools/files.js index ea87be9f18..b2bac65a64 100644 --- a/tools/files.js +++ b/tools/files.js @@ -928,6 +928,19 @@ exports.getLinesOrEmpty = function (file) { } }; +// Returns null if the file does not exist, otherwise returns the parsed JSON in +// the file. Throws on errors other than ENOENT (including JSON parse failure). +exports.readJSONOrNull = function (file) { + try { + var raw = fs.readFileSync(file, 'utf8'); + } catch (e) { + if (e && e.code === 'ENOENT') + return null; + throw e; + } + return JSON.parse(raw); +}; + // Trims whitespace & other filler characters of a line in a project file. exports.trimLine = function (line) { var match = line.match(/^([^#]*)#/); diff --git a/tools/main.js b/tools/main.js index 8611447beb..ee907f3010 100644 --- a/tools/main.js +++ b/tools/main.js @@ -676,7 +676,7 @@ Fiber(function () { project.project.setRootDir(appDir); } - // XXX compare this to the previous block's usesWarehouse... + // XXX get rid of catalog.uniload eventually if (files.inCheckout()) { // When running from a checkout, uniload does use local packages, but *ONLY // THOSE FROM THE CHECKOUT*: not app packages or $PACKAGE_DIRS packages. @@ -699,10 +699,12 @@ Fiber(function () { // BuiltUniloadCatalog really shouldn't need to build anything: it's just a // bunch of precompiled isopacks! catalog.uniload.initialize({ + // XXX if you delete this call, delete files.getUniloadDir too uniloadDir: files.getUniloadDir() }); } + require('./uniload.js').ensureIsopacketsLoadable(); // Initialize the server catalog. Among other things, this is where we get // release information (used by springboarding). We do not at this point talk diff --git a/tools/uniload.js b/tools/uniload.js index 00b287f360..60d48b8945 100644 --- a/tools/uniload.js +++ b/tools/uniload.js @@ -1,10 +1,15 @@ var _ = require('underscore'); +var path = require('path'); var bundler = require('./bundler.js'); +var Builder = require('./builder.js'); var buildmessage = require('./buildmessage.js'); var release = require('./release.js'); var packageLoader = require("./package-loader.js"); var files = require('./files.js'); var catalog = require('./catalog.js'); +var compiler = require('./compiler.js'); +var config = require('./config.js'); +var watch = require('./watch.js'); // These are the only packages that may be directly loaded via this package. Add // more to the list if you need to uniload more things! (You don't have to @@ -29,6 +34,10 @@ var ROOT_PACKAGES = [ // XXX document ISOPACKETS var ISOPACKETS = { + // Note: when running from a checkout, js-analyze must always be the + // the first to be rebuilt, because it might need to be loaded as part + // of building other isopackets. + 'js-analyze': ['js-analyze'], 'ddp': ['ddp'], 'mongo': ['mongo'], 'ejson': ['ejson'], @@ -38,25 +47,123 @@ var ISOPACKETS = { 'constraint-solver': ['constraint-solver'], 'cordova-support': ['boilerplate-generator', 'logging', 'webapp-hashing', 'xmlbuilder'], - // Note: when running from a checkout, js-analyze must always be the - // the first to be rebuilt, because it might need to be loaded as part - // of building other isopackets. - 'js-analyze': ['js-analyze'], 'logging': ['logging'] }; +// Caches isopackets in memory (each isopacket only needs to be loaded +// once). This is a map from isopacket name to either: +// +// - The 'Package' dictionary, if the isopacket has already been loaded +// into memory +// - null, if the isopacket hasn't been loaded into memory but its on-disk +// instance is known to be ready +// +// The subtlety here is that when running from a checkout, we don't want to +// accidentally load an isopacket before ensuring that it doesn't need to be +// rebuilt. But we do want to be able to load the js-analyze isopacket as part +// of building other isopackets in ensureIsopacketsLoadable. var loadedIsopackets = {}; var loadIsopacket = function (isopacketName) { - if (_.has(loadedIsopackets, isopacketName)) - return loadedIsopackets[isopacketName]; - if (!_.has(ISOPACKETS, isopacketName)) - throw Error("Unknown isopacket: " + isopacketName); - var isopacket = load({packages: ISOPACKETS[isopacketName]}); - loadedIsopackets[isopacketName] = isopacket; - return isopacket; + if (_.has(loadedIsopackets, isopacketName)) { + if (loadedIsopackets[isopacketName]) { + return loadedIsopackets[isopacketName]; + } + // This is the case where the isopacket is up to date on disk but not + // loaded. + var isopacket = load({packages: ISOPACKETS[isopacketName]}); + loadedIsopackets[isopacketName] = isopacket; + return isopacket; + } + + if (_.has(ISOPACKETS, isopacketName)) { + throw Error("Can't load isopacket before it has been verified: " + + isopacketName); + } + + throw Error("Unknown isopacket: " + isopacketName); }; +var calledEnsure = false; +var ensureIsopacketsLoadable = function () { + if (calledEnsure) { + throw Error("can't ensureIsopacketsLoadable twice!"); + } + calledEnsure = true; + + // If we're not running from checkout, then there's nothing to build and we + // can declare that all isopackets are loadable. + if (!files.inCheckout()) { + _.each(ISOPACKETS, function (packages, name) { + loadedIsopackets[name] = null; + }); + return; + } + + // Build all the packages that we can load with uniload. We only want to + // load local packages. + var localPackageLoader = new packageLoader.PackageLoader({ + versions: null, + // XXX get rid of catalog.uniload + catalog: catalog.uniload, + constraintSolverOpts: { ignoreProjectDeps: true } + }); + + var messages = buildmessage.capture(function () { + _.each(ISOPACKETS, function (packages, isopacketName) { + var isopacketRoot = path.join(config.getIsopacketRoot(), isopacketName); + var existingBuildinfo = files.readJSONOrNull( + path.join(isopacketRoot, 'isopacket-buildinfo.json')); + var needRebuild = !existingBuildinfo; + if (!needRebuild && existingBuildinfo.builtBy !== compiler.BUILT_BY) { + needRebuild = true; + } + if (!needRebuild) { + var watchSet = watch.WatchSet.fromJSON(existingBuildinfo.watchSet); + if (!watch.isUpToDate(watchSet)) { + needRebuild = true; + } + } + if (!needRebuild) { + // Great, it's loadable without a rebuild. + loadedIsopackets[isopacketName] = null; + return; + } + + buildmessage.enterJob({ + title: "Compiling " + isopacketName + " packages for the tool" + }, function () { + var built = bundler.buildJsImage({ + name: "isopacket-" + isopacketName, + packageLoader: localPackageLoader, + use: packages, + catalog: catalog.uniload, + ignoreProjectDeps: true + }); + + if (buildmessage.jobHasMessages()) + return; + var builder = new Builder({outputPath: isopacketRoot}); + builder.writeJson('isopacket-buildinfo.json', { + builtBy: compiler.BUILT_BY, + watchSet: built.watchSet.toJSON() + }); + built.image.write(builder); + builder.complete(); + // It's loadable now. + loadedIsopackets[isopacketName] = null; + }); + }); + }); + + // This is a build step ... but it's one that only happens in development, so + // it can just crash the app instead of being handled nicely. + if (messages.hasMessages()) { + process.stderr.write("Errors prevented tool build:\n"); + process.stderr.write(messages.formatMessages()); + throw new Error("isopacket build failed?"); + } +}; // Load isopacks into the currently running node.js process. Use // this to use isopacks (such as the DDP client) from command-line @@ -180,6 +287,7 @@ var load = function (options) { var uniload = exports; _.extend(exports, { loadIsopacket: loadIsopacket, + ensureIsopacketsLoadable: ensureIsopacketsLoadable, ROOT_PACKAGES: ROOT_PACKAGES, ISOPACKETS: ISOPACKETS });