diff --git a/tools/catalog.js b/tools/catalog.js index 9f5e4ec92e..b777d4fffc 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -135,7 +135,9 @@ _.extend(Catalog.prototype, { self.packages = []; self.versions = []; self.builds = []; - self._insertServerPackages(serverPackageData); + if (serverPackageData) { + self._insertServerPackages(serverPackageData); + } self._addLocalPackageOverrides(true /* setInitialized */); }, @@ -212,7 +214,7 @@ _.extend(Catalog.prototype, { var packageSources = {}; // name to PackageSource var versionIds = {}; // name to _id of the created Version record _.each(self.effectiveLocalPackages, function (packageDir, name) { - var packageSource = new PackageSource(packageDir); + var packageSource = new PackageSource; packageSource.initFromPackageDir(name, packageDir); packageSources[name] = packageSource; @@ -324,7 +326,7 @@ _.extend(Catalog.prototype, { var sourcePath = self.effectiveLocalPackages[name]; var buildDir = path.join(sourcePath, '.build'); if (fs.existsSync(buildDir)) { - var unipackage = new Unipackage(sourcePath); + var unipackage = new Unipackage; unipackage.initFromPath(name, buildDir, { buildOfPath: sourcePath }); if (compiler.checkUpToDate(packageSources[name], unipackage)) { return unipackage; diff --git a/tools/commands.js b/tools/commands.js index 63f2e0728e..850b162966 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1578,12 +1578,12 @@ main.registerCommand({ // still confirming that it matches the name of the directory) var packageName = path.basename(options.packageDir.toLowerCase()); - packageSource = new PackageSource(options.packageDir); + packageSource = new PackageSource; packageSource.initFromPackageDir(packageName, options.packageDir); if (buildmessage.jobHasMessages()) return; // already have errors, so skip the build - compileResult = compiler.compile(packageSource); + compileResult = compiler.compile(packageSource, { officialBuild: true }); }); if (messages.hasMessages()) { @@ -1716,9 +1716,11 @@ main.registerCommand({ return 1; } - var packageSource = new PackageSource(packageDir); + var packageSource = new PackageSource; packageSource.initFromPackageDir(options.name, packageDir); - var unipackage = compiler.compile(packageSource).unipackage; + var unipackage = compiler.compile(packageSource, { + officialBuild: true + }).unipackage; unipackage.saveToPath(path.join(packageDir, '.build')); var conn; diff --git a/tools/compiler.js b/tools/compiler.js index 260556ebee..04acb9b414 100644 --- a/tools/compiler.js +++ b/tools/compiler.js @@ -26,7 +26,7 @@ var compiler = exports; // end up as watched dependencies. (At least for now, packages only used in // target creation (eg minifiers and dev-bundle-fetcher) don't require you to // update BUILT_BY, though you will need to quit and rerun "meteor run".) -compiler.BUILT_BY = 'meteor/10'; +compiler.BUILT_BY = 'meteor/11'; // XXX where should this go? I'll make it a random utility function // for now @@ -599,16 +599,27 @@ var compileSlice = function (unipackage, inputSlice, packageLoader, // the appropriate compiler plugins. Once build has completed, any errors // detected in the package will have been emitted to buildmessage. // +// Options: +// - officialBuild: defaults to false. If false, then we will compute a +// build identifier (a hash of the package's dependency versions and +// source files) and include it as part of the unipackage's version +// string. If true, then we will use the version that is contained in +// the package's source. You should set it to true when you are +// building a package to publish as an official build with the +// package server. +// // Returns an object with keys: // - unipackage: the build Unipackage // - sources: array of source files (identified by their path on local // disk) that were used by the build (the source files you'd have to // ship to a different machine to replicate the build there) -compiler.compile = function (packageSource) { +compiler.compile = function (packageSource, options) { var sources = []; var pluginWatchSet = packageSource.pluginWatchSet.clone(); var plugins = {}; + options = _.extend({ officialBuild: false }, options); + // Determine versions of build-time dependencies var buildTimeDeps = determineBuildTimeDependencies(packageSource); @@ -681,22 +692,7 @@ compiler.compile = function (packageSource) { } } - // XXX XXX HERE HERE - // - // Unless the 'officalBuild' option was set, compute a build - // identifier by finding the versions of all of our package - // dependencies (direct and plugin dependencies) -- real versions, - // not +local version, which means a lookup in the catalog -- and - // hashing them together (in a structured way with what they go - // with, canoncialized as well as possible) with the contents of the - // merged watchsets of our slices and our plugins, with paths - // relativized somehow (that last bit may be tricky!) - // - // Then -- again, unless 'officialBuild' was set -- modify version - // by adding + to the version (it's an error if you already - // had one, I guess). - - var unipackage = new Unipackage(); + var unipackage = new Unipackage; unipackage.initFromOptions({ name: packageSource.name, metadata: packageSource.metadata, @@ -704,7 +700,6 @@ compiler.compile = function (packageSource) { earliestCompatibleVersion: packageSource.earliestCompatibleVersion, defaultSlices: packageSource.defaultSlices, testSlices: packageSource.testSlices, - packageDirectoryForBuildInfo: packageSource.packageDirectoryForBuildInfo, plugins: plugins, pluginWatchSet: pluginWatchSet, buildTimeDirectDependencies: buildTimeDeps.directDependencies, @@ -723,6 +718,27 @@ compiler.compile = function (packageSource) { sources.push.apply(sources, sliceSources); }); + // XXX what should we do if the PackageSource doesn't have a version? + // (e.g. a plugin) + if (! options.officialBuild && packageSource.version) { + // XXX I have no idea if this should be using buildmessage.enterJob + // or not. test what happens on error + buildmessage.enterJob({ + title: "compute build identifier for package `" + + packageSource.name + "`", + rootPath: packageSource.sourceRoot + }, function () { + if (packageSource.version.indexOf("+") !== -1) { + buildmessage.error("cannot compute build identifier for package `" + + packageSource.name + "` version " + + packageSource.version + "because it already " + + "has a build identifier"); + } else { + unipackage.addBuildIdentifierToVersion(); + } + }); + } + return { sources: _.uniq(sources), unipackage: unipackage @@ -773,47 +789,88 @@ compiler.getBuildOrderConstraints = function (packageSource) { // identical code). True if we have dependency info and it // says that the package is up-to-date. False if a source file or // build-time dependency has changed. -// -// 'what' identifies the build to check for up-to-dateness and is an -// object with exactly one of the following keys: -// - path: a path on disk to a unipackage -// - unipackage: a Unipackage object compiler.checkUpToDate = function (packageSource, unipackage) { if (unipackage.forceNotUpToDate) return false; // Do we think we'd generate different contents than the tool that // built this package? - if (unipackage.builtBy !== compiler.BUILT_BY) + if (unipackage.builtBy !== compiler.BUILT_BY) { + // XXX XXX XXX XXX XXX XXX XXX + // + // This branch is not currently in a state where we can build + // packages with plugins, so we explicitly do NOT want to trigger + // re-builds of packages built by different versions of meteor. + // + // Once this branch is in a state where we CAN build packages + // a-fresh, then we should change this back to "return false". + // + // XXX XXX XXX XXX XXX XXX XXX + console.log("XXX warning: considering package", + packageSource.name, "to be up to date because", + "it was built by <", compiler.BUILT_BY, + "and this makes no sense at all"); + return true; return false; + } - // XXX XXX XXX - var pluginProviderPackageDirs = unipackage.pluginProviderPackageDirs; + var buildTimeDeps = determineBuildTimeDependencies(packageSource); - /* - // XXX XXX this shouldn't work this way at all. instead it should - // just get the resolved build-time dependencies from packageSource - // and make sure they match the versions that were used for the - // build. - var packageLoader = XXX; + // Compute the unipackage's direct and plugin dependencies to + // `buildTimeDeps`, by comparing versions (including build + // identifiers). - // Are all of the packages we directly use (which can provide - // plugins which affect compilation) resolving to the same - // directory? (eg, have we updated our release version to something - // with a new version of a package?) - var packageResolutionsSame = _.all( - _pluginProviderPackageDirs, function (packageDir, name) { - return packageLoader.getLoadPathForPackage(name) === packageDir; - }); - if (! packageResolutionsSame) + if (_.keys(buildTimeDeps.directDependencies).length !== + _.keys(unipackage.buildTimeDirectDependencies).length) { return false; - */ + } - // XXX as we're checking build-time dependency freshness in the - // future, remember to not rely on - // packageSource.directBuildTimeDependencies, which may contain - // versions like 1.2.3+local, but instead get versions with real - // build ids through the catalog + var directDepsPackageLoader = new PackageLoader( + buildTimeDeps.directDependencies); + var directDepsMatch = _.all( + buildTimeDeps.directDependencies, + function (version, packageName) { + var loadedPackage = directDepsPackageLoader.getPackage(packageName); + // XXX Check that `versionWithBuildId` is the same as `version` + // except for the build id? + return (loadedPackage && + unipackage.buildTimeDirectDependencies[packageName] === + loadedPackage.version); + } + ); + if (! directDepsMatch) { + return false; + } + + if (_.keys(buildTimeDeps.pluginDependencies).length !== + _.keys(unipackage.buildTimePluginDependencies).length) { + return false; + } + + var pluginDepsMatch = _.all( + buildTimeDeps.pluginDependencies, + function (pluginDeps, pluginName) { + // For each plugin, check that the resolved build-time deps for + // that plugin match the unipackage's build time deps for it. + var packageLoaderForPlugin = new PackageLoader( + buildTimeDeps.pluginDependencies + ); + var unipackagePluginDeps = unipackage.buildTimePluginDependencies[pluginName]; + if (! unipackagePluginDeps || + _.keys(pluginDeps).length !== _.keys(unipackagePluginDeps).length) { + return false; + } + return _.all(pluginDeps, function (version, packageName) { + var loadedPackage = packageLoaderForPlugin.getPackage(packageName); + return loadedPackage && + unipackagePluginDeps[packageName] === loadedPackage.version; + }); + } + ); + + if (! pluginDepsMatch) { + return false; + } var watchSet = new watch.WatchSet(); watchSet.merge(unipackage.pluginWatchSet); @@ -821,8 +878,9 @@ compiler.checkUpToDate = function (packageSource, unipackage) { watchSet.merge(slice.watchSet); }); - if (! watch.isUpToDate(watchSet)) + if (! watch.isUpToDate(watchSet)) { return false; + } return true; }; diff --git a/tools/package-cache.js b/tools/package-cache.js index c0bff38ffe..41216f4a6b 100644 --- a/tools/package-cache.js +++ b/tools/package-cache.js @@ -70,14 +70,15 @@ _.extend(PackageCache.prototype, { delete self.softReloadCache[loadPath]; var isUpToDate; + var unipackage; if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) { // We don't even have the source to this package, so it must // be up to date. isUpToDate = true; } else { - var packageSource = new PackageSource(loadPath); + var packageSource = new PackageSource; packageSource.initFromPackageDir(name, loadPath); - var unipackage = new Unipackage(loadPath); + unipackage = new Unipackage; unipackage.initFromPath(name, entry.buildDir); isUpToDate = compiler.checkUpToDate(packageSource, entry.pkg); } @@ -94,7 +95,7 @@ _.extend(PackageCache.prototype, { // Does loadPath point directly at a unipackage (rather than a // source tree?) if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) { - var unipackage = new Unipackage(loadPath); + unipackage = new Unipackage; unipackage.initFromPath(name, loadPath); self.loadedPackages[loadPath] = { pkg: unipackage, @@ -105,13 +106,13 @@ _.extend(PackageCache.prototype, { }; // It's a source tree. Load it. - var packageSource = new PackageSource(loadPath); + var packageSource = new PackageSource; packageSource.initFromPackageDir(name, loadPath); // Does it have an up-to-date build? var buildDir = path.join(loadPath, '.build'); if (fs.existsSync(buildDir)) { - var unipackage = new Unipackage(loadPath); + unipackage = new Unipackage; unipackage.initFromPath(name, buildDir); if (compiler.checkUpToDate(packageSource, unipackage)) { self.loadedPackages[loadPath] = { pkg: unipackage, diff --git a/tools/package-source.js b/tools/package-source.js index 080c7cf5c6..e48ebbd436 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -171,7 +171,7 @@ var SourceSlice = function (pkg, options) { // PackageSource /////////////////////////////////////////////////////////////////////////////// -var PackageSource = function (packageDirectoryForBuildInfo) { +var PackageSource = function () { var self = this; // The name of the package, or null for an app pseudo-package or @@ -193,15 +193,6 @@ var PackageSource = function (packageDirectoryForBuildInfo) { // it's still nice to get it right). self.serveRoot = null; - // The package's directory. This is used only by other packages that use this - // package in their buildinfo.json (to detect that they need to be rebuilt if - // the PackageLoader resolves it to a different package); it is not used to - // read files or anything else. Notably, it should be the same if a package is - // read from a source tree or read from the .build unipackage inside that - // source tree. - // XXX can this go away now? - self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo; - // Package metadata. Keys are 'summary' and 'internal'. Currently // both of these are optional. self.metadata = {}; diff --git a/tools/unipackage.js b/tools/unipackage.js index e462e51183..21a9c1a4aa 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -8,6 +8,7 @@ var path = require('path'); var Builder = require('./builder.js'); var bundler = require('./bundler.js'); var watch = require('./watch.js'); +var PackageLoader = require('./package-loader.js'); var rejectBadPath = function (p) { if (p.match(/\.\./)) @@ -179,7 +180,7 @@ _.extend(UnipackageSlice.prototype, { /////////////////////////////////////////////////////////////////////////////// // XXX document -var Unipackage = function (packageDirectoryForBuildInfo) { +var Unipackage = function () { var self = this; // These have the same meaning as in PackageSource. @@ -190,10 +191,6 @@ var Unipackage = function (packageDirectoryForBuildInfo) { self.defaultSlices = {}; self.testSlices = {}; - // XXX this is likely to go away once we have build versions - // (also in PackageSource) - self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo; - // Build slices. Array of UnipackageSlice. self.slices = []; @@ -211,7 +208,6 @@ var Unipackage = function (packageDirectoryForBuildInfo) { // The versions that we used at build time for each of our direct // dependencies. Map from package name to version string. - // XXX save to disk self.buildTimeDirectDependencies = null; // The complete list of versions (including transitive dependencies) @@ -262,7 +258,6 @@ _.extend(Unipackage.prototype, { self.earliestCompatibleVersion = options.earliestCompatibleVersion; self.defaultSlices = options.defaultSlices; self.testSlices = options.testSlices; - self.packageDirectoryForBuildInfo = options.packageDirectoryForBuildInfo; self.plugins = options.plugins; self.pluginWatchSet = options.pluginWatchSet; self.buildTimeDirectDependencies = options.buildTimeDirectDependencies; @@ -463,6 +458,10 @@ _.extend(Unipackage.prototype, { // Read basic buildinfo.json info self.builtBy = buildInfoJson.builtBy || null; + self.buildTimeDirectDependencies = + buildInfoJson.buildTimeDirectDependencies || null; + self.buildTimePluginDependencies = + buildInfoJson.buildTimePluginDependencies || null; if (options.buildOfPath && (buildInfoJson.source !== options.buildOfPath)) { @@ -657,7 +656,9 @@ _.extend(Unipackage.prototype, { sliceDependencies: { }, pluginDependencies: self.pluginWatchSet.toJSON(), pluginProviderPackages: self.pluginProviderPackageDirs, - source: options.buildOfPath || undefined + source: options.buildOfPath || undefined, + buildTimeDirectDependencies: self._buildTimeDirectDependenciesWithBuildIds(), + buildTimePluginDependencies: self._buildTimePluginDependenciesWithBuildIds() }; builder.reserve("unipackage.json"); @@ -851,6 +852,98 @@ _.extend(Unipackage.prototype, { builder.abort(); throw e; } + }, + + _buildTimeDirectDependenciesWithBuildIds: function () { + var self = this; + var directDepsLoader = new PackageLoader({ + versions: self.buildTimeDirectDependencies + }); + var result = {}; + _.each(self.buildTimeDirectDependencies, function (version, packageName) { + var unipackage = directDepsLoader.getPackage(packageName); + result[packageName] = unipackage.version; + }); + return result; + }, + + _buildTimePluginDependenciesWithBuildIds: function () { + var self = this; + var result = {}; + _.each(self.buildTimePluginDependencies, function (deps, pluginName) { + var pluginPackageLoader = new PackageLoader({ versions: deps }); + result[pluginName] = {}; + _.each(deps, function (version, packageName) { + var unipackage = pluginPackageLoader.getPackage(packageName); + result[pluginName][packageName] = unipackage.version; + }); + }); + return result; + }, + + // Computes a hash of the versions of all the package's dependencies + // (direct and plugin dependencies) and the slices' and plugins' watch + // sets. Adds the result as a build identifier to the unipackage's + // version. The caller is responsible for checking whether the + // existing version has a build identifier already. + addBuildIdentifierToVersion: function () { + var self = this; + // Gather all the dependencies' versions and organize them into + // arrays. We use arrays to avoid relying on the order of + // stringified object keys. + var directDeps = []; + _.each( + self._buildTimeDirectDependenciesWithBuildIds(), + function (version, packageName) { + directDeps.push([packageName, version]); + } + ); + + // Sort direct dependencies by package name (which is the "0" property + // of each element in the array). + directDeps = _.sortBy(directDeps, "0"); + + var pluginDeps = []; + _.each( + self._buildTimePluginDependenciesWithBuildIds(), + function (versions, pluginName) { + var pluginDepsLoader = new PackageLoader({ versions: versions }); + var singlePluginDeps = []; + _.each(versions, function (version, packageName) { + var unipackage = pluginDepsLoader.getPackage(packageName); + singlePluginDeps.push([unipackage.name, unipackage.version]); + }); + singlePluginDeps = _.sortBy(singlePluginDeps, "0"); + pluginDeps.push([pluginName, singlePluginDeps]); + } + ); + pluginDeps = _.sortBy(pluginDeps, "0"); + + // Now that we have versions for all our dependencies, canonicalize + // the slices' and plugins' watch sets. + // XXX Do we need to relativize paths? Why? + var watchFiles = []; + var watchSet = new watch.WatchSet(); + watchSet.merge(self.pluginWatchSet); + _.each(self.slices, function (slice) { + watchSet.merge(slice.watchSet); + }); + _.each(watchSet.files, function (hash, fileAbsPath) { + watchFiles.push([fileAbsPath, hash]); + }); + watchFiles = _.sortBy(watchFiles, "0"); + + // Stick all our info into one big array, stringify it, and hash it. + var buildIdInfo = [ + directDeps, + pluginDeps, + watchFiles + ]; + var crypto = require('crypto'); + var hasher = crypto.createHash('sha1'); + hasher.update(JSON.stringify(buildIdInfo)); + var buildId = hasher.digest('hex'); + self.version = self.version + "+" + buildId; } }); diff --git a/tools/watch.js b/tools/watch.js index 04f86e2f48..aed13b978e 100644 --- a/tools/watch.js +++ b/tools/watch.js @@ -138,9 +138,9 @@ _.extend(WatchSet.prototype, { self.alwaysFire = true; return; } -// _.each(other.files, function (hash, name) { -// self.addFile(name, hash); -// }); + _.each(other.files, function (hash, name) { + self.addFile(name, hash); + }); _.each(other.directories, function (dir) { // XXX this doesn't deep-clone the directory, but I think these objects // are never mutated #WatchSetShallowClone