var compiler = require('./compiler.js'); var archinfo = require('./archinfo.js'); var _ = require('underscore'); var linker = require('./linker.js'); var buildmessage = require('./buildmessage.js'); var fs = require('fs'); var path = require('path'); var Builder = require('./builder.js'); var bundler = require('./bundler.js'); var watch = require('./watch.js'); var packageLoader = require('./package-loader.js'); var catalog = require('./catalog.js'); var files = require('./files.js'); var uniload = require('./uniload.js'); var Future = require('fibers/future'); var rejectBadPath = function (p) { if (p.match(/\.\./)) throw new Error("bad path: " + p); }; /////////////////////////////////////////////////////////////////////////////// // Unibuild /////////////////////////////////////////////////////////////////////////////// // Options: // - name [required] // - arch [required] // - uses // - implies // - watchSet // - nodeModulesPath // - prelinkFiles // - packageVariables // - resources var nextBuildId = 1; var Unibuild = function (isopack, options) { var self = this; options = options || {}; self.pkg = isopack; self.arch = options.arch; self.uses = options.uses; self.implies = options.implies || []; // This WatchSet will end up having the watch items from the // SourceArch (such as package.js or .meteor/packages), plus all of // the actual source files for the unibuild (including items that we // looked at to find the source files, such as directories we // scanned). self.watchSet = options.watchSet || new watch.WatchSet(); // Each Unibuild is given a unique id when it's loaded (it is // not saved to disk). This is just a convenience to make it easier // to keep track of Unibuilds in a map; it's used by bundler // and compiler. We put some human readable info in here too to make // debugging easier. self.id = isopack.name + "." + self.pkg.name + "@" + self.arch + "#" + (nextBuildId ++); // Prelink output. // // 'prelinkFiles' is the partially linked JavaScript code (an // array of objects with keys 'source' and 'servePath', both strings -- see // prelink() in linker.js) // // 'packageVariables' are variables that are syntactically globals // in our input files and which we capture with a package-scope // closure. A list of objects with keys 'name' (required) and // 'export' (true, 'tests', or falsy). // // Both of these are saved into unibuilds on disk, and are inputs into the final // link phase, which inserts the final JavaScript resources into // 'resources'. self.prelinkFiles = options.prelinkFiles; self.packageVariables = options.packageVariables; // All of the data provided for eventual inclusion in the bundle, // other than JavaScript that still needs to be fed through the // final link stage. A list of objects with these keys: // // type: "js", "css", "head", "body", "asset" // // data: The contents of this resource, as a Buffer. For example, // for "head", the data to insert in ; for "js", the // JavaScript source code (which may be subject to further // processing such as minification); for "asset", the contents of a // static resource such as an image. // // servePath: The (absolute) path at which the resource would prefer // to be served. Interpretation varies by type. For example, always // honored for "asset", ignored for "head" and "body", sometimes // honored for CSS but ignored if we are concatenating. // // sourceMap: Allowed only for "js". If present, a string. self.resources = options.resources; // Absolute path to the node_modules directory to use at runtime to // resolve Npm.require() calls in this unibuild. null if this unibuild // does not have a node_modules. self.nodeModulesPath = options.nodeModulesPath; }; _.extend(Unibuild.prototype, { // Get the resources that this function contributes to a bundle, in // the same format as self.resources as documented above. This // includes static assets and fully linked JavaScript. // // @param bundleArch The architecture targeted by the bundle. Might // be more specific than self.arch. // // It is when you call this function that we read our dependent // packages and commit to whatever versions of them we currently // have in the library -- at least for the purpose of imports, which // is resolved at bundle time. (On the other hand, when it comes to // the extension handlers we'll use, we previously commited to those // versions at package build ('compile') time.) // // loader is the PackageLoader that should be used to resolve // the package's bundle-time dependencies. getResources: function (bundleArch, loader) { var self = this; if (! archinfo.matches(bundleArch, self.arch)) throw new Error("unibuild of arch '" + self.arch + "' does not support '" + bundleArch + "'?"); // Compute imports by merging the exports of all of the packages // we use. Note that in the case of conflicting symbols, later // packages get precedence. // // We don't get imports from unordered dependencies (since they may not be // defined yet) or from weak/debugOnly dependencies (because the meaning of // a name shouldn't be affected by the non-local decision of whether or not // an unrelated package in the target depends on something). var imports = {}; // map from symbol to supplying package name compiler.eachUsedUnibuild( self.uses, bundleArch, loader, { skipUnordered: true, skipDebugOnly: true }, function (depUnibuild) { _.each(depUnibuild.packageVariables, function (symbol) { // Slightly hacky implementation of test-only exports. if (symbol.export === true || (symbol.export === "tests" && self.pkg.isTest)) imports[symbol.name] = depUnibuild.pkg.name; }); }); // Phase 2 link var isApp = ! self.pkg.name; var files = linker.link({ imports: imports, useGlobalNamespace: isApp, // XXX report an error if there is a package called global-imports importStubServePath: isApp && '/packages/global-imports.js', prelinkFiles: self.prelinkFiles, packageVariables: self.packageVariables, includeSourceMapInstructions: archinfo.matches(self.arch, "web"), name: self.pkg.name || null }); // Add each output as a resource var jsResources = _.map(files, function (file) { return { type: "js", data: new Buffer(file.source, 'utf8'), // XXX encoding servePath: file.servePath, sourceMap: file.sourceMap }; }); return _.union(self.resources, jsResources); // union preserves order } }); /////////////////////////////////////////////////////////////////////////////// // Isopack /////////////////////////////////////////////////////////////////////////////// // Helper function. Takes an object mapping package name to version, and // ensures that all the versions have real build ids (by loading them // through a PackageLoader) rather than +local build ids (which is what // they could have if we just read them out of the catalog). // // If the optional `filter` function is provided, then we will only load // packages for which `filter(packageName, version)` returns truthy. var getLoadedPackageVersions = function (versions, catalog, filter) { buildmessage.assertInCapture(); var result = {}; var loader = new packageLoader.PackageLoader({ versions: versions, catalog: catalog }); _.each(versions, function (version, packageName) { if (! filter || filter(packageName, version)) { var isopack = loader.getPackage(packageName); result[packageName] = isopack.version; } }); return result; }; var convertIsopackFormat = function (data, versionFrom, versionTo) { var convertedData = _.clone(data); if (versionFrom === versionTo) { return convertedData; } // XXX COMPAT WITH 0.9.3 if (versionFrom === "unipackage-pre2" && versionTo === "isopack-1") { convertedData.builds = convertedData.unibuilds; delete convertedData.unibuilds; return convertedData; } else if (versionFrom === "isopack-1" && versionTo === "unipackage-pre2") { convertedData.unibuilds = convertedData.builds; convertedData.format = "unipackage-pre2"; delete convertedData.builds; return convertedData; } }; var currentFormat = "isopack-1"; // XXX document var Isopack = function () { var self = this; // These have the same meaning as in PackageSource. self.name = null; self.metadata = {}; self.version = null; self.earliestCompatibleVersion = null; self.isTest = false; self.debugOnly = false; // Unibuilds, an array of class Unibuild. self.unibuilds = []; // Plugins in this package. Map from plugin name to {arch -> JsImage}. self.plugins = {}; self.cordovaDependencies = {}; // -- Information for up-to-date checks -- // Version number of the tool that built this isopack // (compiler.BUILT_BY) or null if unknown self.builtBy = null; // If true, force the checkUpToDate to return false for this isopack. self.forceNotUpToDate = false; // The versions that we used at build time for each of our direct // dependencies. Map from package name to version string. self.buildTimeDirectDependencies = null; // The complete list of versions (including transitive dependencies) // that we used at build time to build each of our plugins. Map from // plugin name to package name to version string. Note that two // plugins might not use the same version for the same transitive // dependency. self.buildTimePluginDependencies = null; // XXX this is likely to change once we have build versions // // A WatchSet for the full transitive dependencies for all plugins in this // package, as well as this package's package.js. If any of these dependencies // change, our plugins need to be rebuilt... but also, any package that // directly uses this package needs to be rebuilt in case the change to // plugins affected compilation. self.pluginWatchSet = new watch.WatchSet(); // -- Loaded plugin state -- // True if plugins have been initialized (if _ensurePluginsInitialized has // been called) self._pluginsInitialized = false; // Source file handlers registered by plugins. Map from extension // (without a dot) to a handler function that takes a // CompileStep. Valid only when _pluginsInitialized is true. self.sourceHandlers = null; // See description in PackageSource. If this is set, then we include a copy of // our own source, in addition to any other tools that were originally in the // isopack. self.includeTool = false; // This is tools to copy from trees on disk. This is used by the // isopack-merge code in tropohouse. self.toolsOnDisk = []; }; _.extend(Isopack.prototype, { // Make a dummy (empty) package that contains nothing of interest. // XXX used? initEmpty: function (name) { var self = this; self.name = name; }, // This is primarily intended to be used by the compiler. After // calling this, call addUnibuild to add the unibuilds. initFromOptions: function (options) { var self = this; self.name = options.name; self.metadata = options.metadata; self.version = options.version; self.earliestCompatibleVersion = options.earliestCompatibleVersion; self.isTest = options.isTest; self.plugins = options.plugins; self.cordovaDependencies = options.cordovaDependencies; self.pluginWatchSet = options.pluginWatchSet; self.buildTimeDirectDependencies = options.buildTimeDirectDependencies; self.buildTimePluginDependencies = options.buildTimePluginDependencies; self.includeTool = options.includeTool; self.debugOnly = options.debugOnly; }, // Programmatically add a unibuild to this Isopack. Should only be // called as part of building up a new Isopack using // initFromOptions. 'options' are the options to the Unibuild // constructor. addUnibuild: function (options) { var self = this; self.unibuilds.push(new Unibuild(self, options)); }, // An sorted array of all the architectures included in this package. architectures: function () { var self = this; var archSet = {}; _.each(self.unibuilds, function (unibuild) { archSet[unibuild.arch] = true; }); _.each(self._toolArchitectures(), function (arch) { archSet[arch] = true; }); _.each(self.plugins, function (plugin, name) { _.each(plugin, function (plug, arch) { archSet[arch] = true; }); }); var arches = _.keys(archSet).sort(); // Ensure that our buildArchitectures string does not look like // web+os+os.osx.x86_64 // This would happen if there is an 'os' unibuild but a platform-specific // tool (eg, in meteor-tool). This would confuse catalog.getBuildsForArches // into thinking that it would work for Linux, since the 'os' means // 'works on any Node server'. if (_.any(arches, function (a) { return a.match(/^os\./); })) { arches = _.without(arches, 'os'); } return arches; }, // A sorted plus-separated string of all the architectures included in this // package. buildArchitectures: function () { var self = this; return self.architectures().join('+'); }, tarballName: function () { var self = this; return self.name + '-' + self.version + '-' + self.buildArchitectures(); }, _toolArchitectures: function () { var self = this; var toolArches = _.pluck(self.toolsOnDisk, 'arch'); self.includeTool && toolArches.push(archinfo.host()); return _.uniq(toolArches).sort(); }, // Return the unibuild of the package to use for a given target architecture // (eg, 'os.linux.x86_64' or 'web'), or throw an exception if that // packages can't be loaded under these circumstances. getUnibuildAtArch: function (arch) { var self = this; var chosenArch = archinfo.mostSpecificMatch( arch, _.pluck(self.unibuilds, 'arch')); if (! chosenArch) { buildmessage.error( (self.name || "this app") + " is not compatible with architecture '" + arch + "'", { secondary: true }); // recover by returning by no unibuilds return null; } return _.findWhere(self.unibuilds, { arch: chosenArch }); }, // Load this package's plugins into memory, if they haven't already // been loaded, and return the list of source file handlers // registered by the plugins: a map from extension (without a dot) // to a handler function that takes a CompileStep. getSourceHandlers: function () { var self = this; self._ensurePluginsInitialized(); return self.sourceHandlers; }, // If this package has plugins, initialize them (run the startup // code in them so that they register their extensions). Idempotent. _ensurePluginsInitialized: function () { var self = this; if (self._pluginsInitialized) return; /** * @global * @namespace Plugin * @summary The namespace that is exposed inside build plugin files. */ var Plugin = { // 'extension' is a file extension without the separation dot // (eg 'js', 'coffee', 'coffee.md') // // 'options' can be elided. The only known option is 'isTemplate', which // is a bit of a hack meaning "in an app, these files should be loaded // before non-templates". // // 'handler' is a function that takes a single argument, a // CompileStep (#CompileStep) /** * @summary Inside a build plugin source file specified in * [Package.registerBuildPlugin](#Package-registerBuildPlugin), * add a handler to compile files with a certain file extension. * @param {String} fileExtension The file extension that this plugin * should handle, without the first dot. * Examples: `"coffee"`, `"coffee.md"`. * @param {Function} handler A function that takes one argument, * a CompileStep object. * * Documentation for CompileStep is available [on the GitHub Wiki](https://github.com/meteor/meteor/wiki/CompileStep-API-for-Build-Plugin-Source-Handlers). * @memberOf Plugin * @locus Build Plugin */ registerSourceHandler: function (extension, options, handler) { if (!handler) { handler = options; options = {}; } if (_.has(self.sourceHandlers, extension)) { buildmessage.error("duplicate handler for '*." + extension + "'; may only have one per Plugin", { useMyCaller: true }); // recover by ignoring all but the first return; } self.sourceHandlers[extension] = { handler: handler, isTemplate: !!options.isTemplate, archMatching: options.archMatching }; } }; self.sourceHandlers = {}; _.each(self.plugins, function (pluginsByArch, name) { var arch = archinfo.mostSpecificMatch( archinfo.host(), _.keys(pluginsByArch)); if (! arch) { buildmessage.error("package `" + name + "` is built for incompatible " + "architecture"); // Recover by ignoring plugin // XXX does this recovery work? return; } var plugin = pluginsByArch[arch]; buildmessage.enterJob({ title: "Loading plugin `" + name + "` from package `" + self.name + "`" // don't necessarily have rootPath anymore // (XXX we do, if the isopack was locally built, which is // the important case for debugging. it'd be nice to get this // case right.) }, function () { plugin.load({ Plugin: Plugin }); }); }); self._pluginsInitialized = true; }, // Load a Isopack on disk. // // options: // - buildOfPath: If present, the source directory (as an absolute // path on local disk) of which we think this isopack is a // build. If it's not (it was copied from somewhere else), we // consider it not up to date (in the sense of checkUpToDate) so // that we can rebuild it and correct the absolute paths in the // dependency information. initFromPath: function (name, dir, options) { var self = this; options = _.clone(options || {}); options.firstIsopack = true; return self._loadUnibuildsFromPath(name, dir, options); }, _loadUnibuildsFromPath: function (name, dir, options) { var self = this; options = options || {}; // In the tropohouse, isopack paths are symlinks (which can be updated if // more unibuilds are merged in). For any given call to // _loadUnibuildsFromPath, let's ensure we see a consistent isopack by // realpath'ing dir. dir = fs.realpathSync(dir); var mainJson; // deal with different versions of "isopack.json", backwards compatible var isopackJsonPath = path.join(dir, "isopack.json"); if (fs.existsSync(isopackJsonPath)) { var isopackJson = JSON.parse(fs.readFileSync(isopackJsonPath)); if (isopackJson[currentFormat]) { mainJson = isopackJson[currentFormat]; } else { // This file is from the future and no longer supports this version throw new Error("Could not find isopack data with format " + currentFormat + ".\n" + "This isopack was likely built with a much newer version of Meteor."); } } else { // super old version with different file name // XXX COMPAT WITH 0.9.3 var unipackageJsonPath = path.join(dir, "unipackage.json"); if (fs.existsSync(unipackageJsonPath)) { mainJson = JSON.parse(fs.readFileSync(unipackageJsonPath)); // in the old format, builds were called unibuilds // use string to make sure this doesn't get caught in a find/replace mainJson.builds = mainJson["unibuilds"]; mainJson = convertIsopackFormat(mainJson, "unipackage-pre2", currentFormat); } if (mainJson.format !== "unipackage-pre2") { // We don't support pre-0.9.0 isopacks, but we do know enough to delete // them if we find them in .build.* somehow (rather than crash). if (mainJson.format === "unipackage-pre1") { throw new exports.OldIsopackFormatError(); } throw new Error("Unsupported isopack format: " + JSON.stringify(mainJson.format)); } } // done dealing with different versions, below here // mainJson is the data we want // isopacks didn't used to know their name, but they should. if (_.has(mainJson, 'name') && name !== mainJson.name) { throw new Error("isopack " + name + " thinks its name is " + mainJson.name); } var buildInfoPath = path.join(dir, 'buildinfo.json'); var buildInfoJson = fs.existsSync(buildInfoPath) && JSON.parse(fs.readFileSync(buildInfoPath)); if (buildInfoJson) { if (!options.firstIsopack) { throw Error("can't merge isopacks with buildinfo"); } } else { buildInfoJson = {}; } // XXX should comprehensively sanitize (eg, typecheck) everything // read from json files // Read basic buildinfo.json info self.builtBy = buildInfoJson.builtBy || null; self.buildTimeDirectDependencies = buildInfoJson.buildTimeDirectDependencies || null; self.buildTimePluginDependencies = buildInfoJson.buildTimePluginDependencies || null; if (options.buildOfPath && (buildInfoJson.source !== options.buildOfPath)) { // This catches the case where you copy a source tree that had a // .build directory and then modify a file. Without this check // you won't see a rebuild (even if you stop and restart // meteor), at least not until you modify the *original* copies // of the source files, because that is still where all of the // dependency info points. self.forceNotUpToDate = true; } // Read the watch sets for each unibuild var unibuildWatchSets = {}; _.each(buildInfoJson.buildDependencies, function (watchSetJSON, unibuildTag) { unibuildWatchSets[unibuildTag] = watch.WatchSet.fromJSON(watchSetJSON); }); // Read pluginWatchSet and pluginProviderPackageDirs. (In the // multi-sub-isopack case, these are guaranteed to be trivial // (since we check that there's no buildinfo.json), so no need to // merge.) self.pluginWatchSet = watch.WatchSet.fromJSON( buildInfoJson.pluginDependencies); self.pluginProviderPackageDirs = buildInfoJson.pluginProviderPackages || {}; // If we are loading multiple isopacks, only take this stuff from the // first one. if (options.firstIsopack) { self.name = name; self.metadata = { summary: mainJson.summary }; self.version = mainJson.version; self.earliestCompatibleVersion = mainJson.earliestCompatibleVersion; self.isTest = mainJson.isTest; self.debugOnly = !!mainJson.debugOnly; } _.each(mainJson.plugins, function (pluginMeta) { rejectBadPath(pluginMeta.path); var plugin = bundler.readJsImage(path.join(dir, pluginMeta.path)); if (!_.has(self.plugins, pluginMeta.name)) { self.plugins[pluginMeta.name] = {}; } // If we already loaded a plugin of this name/arch, just ignore this one. if (!_.has(self.plugins[pluginMeta.name], plugin.arch)) { self.plugins[pluginMeta.name][plugin.arch] = plugin; } }); self.pluginsBuilt = true; _.each(mainJson.builds, function (unibuildMeta) { // aggressively sanitize path (don't let it escape to parent // directory) rejectBadPath(unibuildMeta.path); // Skip unibuilds we already have. var alreadyHaveUnibuild = _.find(self.unibuilds, function (unibuild) { return unibuild.arch === unibuildMeta.arch; }); if (alreadyHaveUnibuild) return; var unibuildJson = JSON.parse( fs.readFileSync(path.join(dir, unibuildMeta.path))); var unibuildBasePath = path.dirname(path.join(dir, unibuildMeta.path)); if (unibuildJson.format !== "unipackage-unibuild-pre1") throw new Error("Unsupported isopack unibuild format: " + JSON.stringify(unibuildJson.format)); var nodeModulesPath = null; if (unibuildJson.node_modules) { rejectBadPath(unibuildJson.node_modules); nodeModulesPath = path.join(unibuildBasePath, unibuildJson.node_modules); } var prelinkFiles = []; var resources = []; _.each(unibuildJson.resources, function (resource) { rejectBadPath(resource.file); var data = new Buffer(resource.length); // Read the data from disk, if it is non-empty. Avoid doing IO for empty // files, because (a) unnecessary and (b) fs.readSync with length 0 // throws instead of acting like POSIX read: // https://github.com/joyent/node/issues/5685 if (resource.length > 0) { var fd = fs.openSync(path.join(unibuildBasePath, resource.file), "r"); try { var count = fs.readSync( fd, data, 0, resource.length, resource.offset); } finally { fs.closeSync(fd); } if (count !== resource.length) throw new Error("couldn't read entire resource"); } if (resource.type === "prelink") { var prelinkFile = { source: data.toString('utf8'), servePath: resource.servePath }; if (resource.sourceMap) { rejectBadPath(resource.sourceMap); prelinkFile.sourceMap = fs.readFileSync( path.join(unibuildBasePath, resource.sourceMap), 'utf8'); } prelinkFiles.push(prelinkFile); } else if (_.contains(["head", "body", "css", "js", "asset"], resource.type)) { resources.push({ type: resource.type, data: data, servePath: resource.servePath || undefined, path: resource.path || undefined }); } else throw new Error("bad resource type in isopack: " + JSON.stringify(resource.type)); }); self.cordovaDependencies = mainJson.cordovaDependencies || null; self.unibuilds.push(new Unibuild(self, { name: unibuildMeta.name, arch: unibuildMeta.arch, uses: unibuildJson.uses, implies: unibuildJson.implies, watchSet: unibuildWatchSets[unibuildMeta.path], nodeModulesPath: nodeModulesPath, prelinkFiles: prelinkFiles, packageVariables: unibuildJson.packageVariables || [], resources: resources, cordovaDependencies: unibuildJson.cordovaDependencies })); }); _.each(mainJson.tools, function (toolMeta) { toolMeta.rootDir = dir; // XXX check for overlap self.toolsOnDisk.push(toolMeta); }); return true; }, // options: // // - buildOfPath: Optional. The absolute path on local disk of the // directory that was built to produce this package. Used as part // of the dependency info to detect builds that were moved and // then modified. // - elideBuildInfo: If set, don't write a buildinfo.json file. saveToPath: function (outputDir, options) { var self = this; var outputPath = outputDir; options = options || {}; buildmessage.assertInCapture(); if (!options.catalog && !options.elideBuildInfo) throw Error("catalog required to generate buildinfo.json"); var builder = new Builder({ outputPath: outputPath }); try { var mainJson = { name: self.name, summary: self.metadata.summary, version: self.version, earliestCompatibleVersion: self.earliestCompatibleVersion, isTest: self.isTest, builds: [], plugins: [] }; if (self.debugOnly) { mainJson.debugOnly = true; } if (! _.isEmpty(self.cordovaDependencies)) { mainJson.cordovaDependencies = self.cordovaDependencies; } var buildInfoJson = null; if (!options.elideBuildInfo) { // Note: The contents of buildInfoJson (with the root directory of the // Meteor checkout naively deleted) gets its SHA taken to determine the // built package's warehouse version. So it should not contain // platform-dependent data and should contain all sources of change to // the isopack's output. See // scripts/admin/build-package-tarballs.sh. // XXX this script is no longer relevant; we use "build IDs" now instead var buildTimeDirectDeps = getLoadedPackageVersions( self.buildTimeDirectDependencies, options.catalog); var buildTimePluginDeps = {}; _.each(self.buildTimePluginDependencies, function (versions, pluginName) { buildTimePluginDeps[pluginName] = getLoadedPackageVersions( versions, options.catalog); }); buildInfoJson = { builtBy: compiler.BUILT_BY, buildDependencies: { }, pluginDependencies: self.pluginWatchSet.toJSON(), pluginProviderPackages: self.pluginProviderPackageDirs, source: options.buildOfPath || undefined, buildTimeDirectDependencies: buildTimeDirectDeps, buildTimePluginDependencies: buildTimePluginDeps, cordovaDependencies: self.cordovaDependencies }; } // XXX COMPAT WITH 0.9.3 builder.reserve("unipackage.json"); builder.reserve("isopack.json"); // Reserve this even if elideBuildInfo is set, to ensure nothing else // writes it somehow. builder.reserve("buildinfo.json"); builder.reserve("head"); builder.reserve("body"); // Map from absolute path to npm directory in the unibuild, to the // generated filename in the isopack we're writing. Multiple builds // can use the same npm modules (though maybe not any more, since tests // have been separated?), but also there can be different sets of // directories as well (eg, for a isopack merged with from multiple // isopacks with _loadUnibuildsFromPath). var npmDirectories = {}; // Pre-linker versions of Meteor expect all packages in the warehouse to // contain a file called "package.js"; they use this as part of deciding // whether or not they need to download a new package. Because packages // are downloaded by the *existing* version of the tools, we need to // include this file until we're comfortable breaking "meteor update" from // 0.6.4. (Specifically, warehouse.packageExistsInWarehouse used to check // to see if package.js exists instead of just looking for the package // directory.) // XXX Remove this once we can. builder.write("package.js", { data: new Buffer( ("// This file is included for compatibility with the Meteor " + "0.6.4 package downloader.\n"), "utf8") }); // Unibuilds _.each(self.unibuilds, function (unibuild) { // Make up a filename for this unibuild var baseUnibuildName = unibuild.arch; var unibuildDir = builder.generateFilename(baseUnibuildName, { directory: true }); var unibuildJsonFile = builder.generateFilename(baseUnibuildName + ".json"); mainJson.builds.push({ arch: unibuild.arch, path: unibuildJsonFile }); // Save unibuild dependencies. Keyed by the json path rather than thinking // too hard about how to encode pair (name, arch). if (buildInfoJson) { buildInfoJson.buildDependencies[unibuildJsonFile] = unibuild.watchSet.toJSON(); } // Figure out where the npm dependencies go. var nodeModulesPath = undefined; var needToCopyNodeModules = false; if (unibuild.nodeModulesPath) { if (_.has(npmDirectories, unibuild.nodeModulesPath)) { // We already have this npm directory from another unibuild. nodeModulesPath = npmDirectories[unibuild.nodeModulesPath]; } else { // It's important not to put node_modules at the top level of the // isopack, so that it is not visible from within plugins. nodeModulesPath = npmDirectories[unibuild.nodeModulesPath] = builder.generateFilename("npm/node_modules", {directory: true}); needToCopyNodeModules = true; } } // Construct unibuild metadata var unibuildJson = { format: "unipackage-unibuild-pre1", packageVariables: unibuild.packageVariables, uses: _.map(unibuild.uses, function (u) { return { 'package': u.package, // For cosmetic value, leave false values for these options out of // the JSON file. constraint: u.constraint || undefined, unordered: u.unordered || undefined, weak: u.weak || undefined }; }), implies: (_.isEmpty(unibuild.implies) ? undefined : unibuild.implies), node_modules: nodeModulesPath, resources: [] }; // Output 'head', 'body' resources nicely var concat = { head: [], body: [] }; var offset = { head: 0, body: 0 }; _.each(unibuild.resources, function (resource) { if (_.contains(["head", "body"], resource.type)) { if (concat[resource.type].length) { concat[resource.type].push(new Buffer("\n", "utf8")); offset[resource.type]++; } if (! (resource.data instanceof Buffer)) throw new Error("Resource data must be a Buffer"); unibuildJson.resources.push({ type: resource.type, file: path.join(unibuildDir, resource.type), length: resource.data.length, offset: offset[resource.type] }); concat[resource.type].push(resource.data); offset[resource.type] += resource.data.length; } }); _.each(concat, function (parts, type) { if (parts.length) { builder.write(path.join(unibuildDir, type), { data: Buffer.concat(concat[type], offset[type]) }); } }); // Output other resources each to their own file _.each(unibuild.resources, function (resource) { if (_.contains(["head", "body"], resource.type)) return; // already did this one unibuildJson.resources.push({ type: resource.type, file: builder.writeToGeneratedFilename( path.join(unibuildDir, resource.servePath), { data: resource.data }), length: resource.data.length, offset: 0, servePath: resource.servePath || undefined, path: resource.path || undefined }); }); // Output prelink resources _.each(unibuild.prelinkFiles, function (file) { var data = new Buffer(file.source, 'utf8'); var resource = { type: 'prelink', file: builder.writeToGeneratedFilename( path.join(unibuildDir, file.servePath), { data: data }), length: data.length, offset: 0, servePath: file.servePath || undefined }; if (file.sourceMap) { // Write the source map. resource.sourceMap = builder.writeToGeneratedFilename( path.join(unibuildDir, file.servePath + '.map'), { data: new Buffer(file.sourceMap, 'utf8') } ); } unibuildJson.resources.push(resource); }); // If unibuild has included node_modules, copy them in if (needToCopyNodeModules) { builder.copyDirectory({ from: unibuild.nodeModulesPath, to: nodeModulesPath }); } // Control file for unibuild builder.writeJson(unibuildJsonFile, unibuildJson); }); // Plugins _.each(self.plugins, function (pluginsByArch, name) { _.each(pluginsByArch, function (plugin) { var pluginDir = builder.generateFilename('plugin.' + name + '.' + plugin.arch, { directory: true }); var relPath = plugin.write(builder.enter(pluginDir)); mainJson.plugins.push({ name: name, arch: plugin.arch, path: path.join(pluginDir, relPath) }); }); }); // Tools // First, are we supposed to include our own source as a tool? if (self.includeTool) { var toolsJson = self._writeTool(builder); mainJson.tools = toolsJson; } // Next, what about other tools we may be merging from other isopacks? // XXX check for overlap _.each(self.toolsOnDisk, function (toolMeta) { toolMeta = _.clone(toolMeta); var rootDir = toolMeta.rootDir; delete toolMeta.rootDir; builder.copyDirectory({ from: path.join(rootDir, toolMeta.path), to: toolMeta.path }); if (!mainJson.tools) { mainJson.tools = []; } mainJson.tools.push(toolMeta); }); // old unipackage.json format/filename // XXX COMPAT WITH 0.9.3 builder.writeJson("unipackage.json", convertIsopackFormat(mainJson, currentFormat, "unipackage-pre2")); // write several versions of the file // add your new format here, and define some stuff // in convertIsopackFormat var formats = ["isopack-1"]; var isopackJson = {}; _.each(formats, function (format) { // new, extensible format - forwards-compatible isopackJson[format] = convertIsopackFormat(mainJson, currentFormat, format); }); // writes one file with all of the new formats, so that it is possible // to invent a new format and have old versions of meteor still read the // old format // // This looks something like: // { // isopack-1: {... data ...}, // isopack-2: {... data ...} // } builder.writeJson("isopack.json", isopackJson); if (buildInfoJson) { builder.writeJson("buildinfo.json", buildInfoJson); } builder.complete(); } catch (e) { builder.abort(); throw e; } }, _writeTool: function (builder) { var self = this; buildmessage.assertInCapture(); var pathsToCopy = files.runGitInCheckout( 'ls-tree', '-r', // recursive '--name-only', '--full-tree', 'HEAD', // The actual trees to copy! 'tools', 'examples', 'LICENSE.txt', 'meteor', 'scripts/admin/launch-meteor'); // Trim blank line and unnecessary examples. pathsToCopy = _.filter(pathsToCopy.split('\n'), function (f) { return f && !f.match(/^examples\/other/) && !f.match(/^examples\/unfinished/); }); var gitSha = files.runGitInCheckout('rev-parse', 'HEAD'); var toolPath = 'meteor-tool-' + archinfo.host(); builder = builder.enter(toolPath); var unipath = builder.reserve('isopacks', {directory: true}); builder.write('.git_version.txt', {data: new Buffer(gitSha, 'utf8')}); builder.copyDirectory({ from: files.getCurrentToolsDir(), to: '', specificFiles: pathsToCopy }); builder.copyDirectory({ from: path.join(files.getDevBundle()), to: 'dev_bundle', ignore: bundler.ignoreFiles }); // We only want to load local packages. var localPackageLoader = new packageLoader.PackageLoader({ versions: null, catalog: catalog.uniload, constraintSolverOpts: { ignoreProjectDeps: true } }); bundler.iterateOverAllUsedIsopacks( localPackageLoader, archinfo.host(), uniload.ROOT_PACKAGES, function (isopk) { // XXX assert that each name shows up once isopk.saveToPath(path.join(unipath, isopk.name), { // There's no mechanism to rebuild these packages. elideBuildInfo: true }); }); return [{ name: 'meteor', arch: archinfo.host(), path: toolPath }]; }, // Computes a hash of the versions of all the package's dependencies // (direct and plugin dependencies) and the unibuilds' and plugins' watch // sets. Options are: // - relativeTo: if provided, the watch set file paths are // relativized to this path. If not provided, we use absolute // paths. // - catalog: required // // Returns the build id as a hex string. getBuildIdentifier: function (options) { var self = this; buildmessage.assertInCapture(); if (!options.catalog) throw Error("required to specify the catalog"); // Gather all the direct dependencies (that provide plugins) and // plugin dependencies' versions and organize them into arrays. We // use arrays to avoid relying on the order of stringified object // keys. var pluginProviders = []; var pluginProviderVersions = getLoadedPackageVersions( self.buildTimeDirectDependencies, options.catalog, function (packageName, version) { // filter if (packageName !== self.name) { var catalogVersion = options.catalog.getVersion(packageName, version); // XXX This could throw if we call it on a freshly-built // isopack (as opposed to one read from disk that has real // build ids for build-time deps instead of +local) before // catalog initialization has finished. See XXX at the top of // `getPluginProviders` in compiler.js. if (! catalogVersion) { throw new Error("No catalog version for" + packageName + "version" + version + "?"); } return catalogVersion.containsPlugins; } else { return false; } } ); _.each(pluginProviderVersions, function (version, packageName) { pluginProviders.push([packageName, version]); }); _.sortBy(pluginProviders, "0"); var pluginDeps = []; // Mild hack documentation: versions for a pluginName can be null if this is // a preconstraint-solver build. (That, elsewhere, indicates to us that we // should only use local packages to build it -- and neatly avoids having to // resolve its dependencies) So, we need to check for that. // #UnbuiltConstraintSolverMustUseLocalPackages _.each( self.buildTimePluginDependencies, function (versions, pluginName) { versions = versions ?_.clone(versions): {}; var singlePluginDeps = []; delete versions[self.name]; _.each( getLoadedPackageVersions(versions, options.catalog), function (version, packageName) { if (packageName !== self.name) { singlePluginDeps.push([packageName, version]); } } ); singlePluginDeps = _.sortBy(singlePluginDeps, "0"); pluginDeps.push([pluginName, singlePluginDeps]); } ); pluginDeps = _.sortBy(pluginDeps, "0"); // Now that we have versions for all our dependencies, canonicalize // the unibuilds' and plugins' watch sets. var watchFiles = []; var watchSet = new watch.WatchSet(); watchSet.merge(self.pluginWatchSet); _.each(self.unibuilds, function (unibuild) { watchSet.merge(unibuild.watchSet); }); _.each(watchSet.files, function (hash, fileAbsPath) { var watchFilePath = fileAbsPath; if (options.relativeTo) { watchFilePath = path.relative(options.relativeTo, fileAbsPath); } watchFiles.push([watchFilePath, hash]); }); watchFiles = _.sortBy(watchFiles, "0"); // Stick all our info into one big array, stringify it, and hash it. var buildIdInfo = [ self.builtBy, pluginProviders, pluginDeps, watchFiles ]; var crypto = require('crypto'); var hasher = crypto.createHash('sha1'); hasher.update(JSON.stringify(buildIdInfo)); return hasher.digest('hex'); }, // Adds the build identifier to the isopack's `version` field. The // caller is responsible for checking whether the existing version has // a build identifier already. Options are the same as // `getBuildIdentifier`. addBuildIdentifierToVersion: function (options) { var self = this; buildmessage.assertInCapture(); self.version = self.version + "+" + self.getBuildIdentifier(options); } }); exports.Isopack = Isopack; exports.OldIsopackFormatError = function () { // This should always be caught anywhere where it can appear (ie, anywhere // that isn't definitely loading something from the tropohouse). this.toString = function () { return "old isopack format!" }; }; exports.isopackExistsAtPath = function (thePath) { // XXX COMPAT WITH 0.9.3 // Remove unipackage.json when no longer supported return fs.existsSync(path.join(thePath, 'isopack.json')) || fs.existsSync(path.join(thePath, 'unipackage.json')); };