github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'library-refactor-buildids' into library-refactor

Browse Source
This commit is contained in:
Emily Stark
2014-03-31 09:09:43 -07:00
parent 2bab6940b0 2ca550ae6e
commit b3a8944167
7 changed files with 229 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
tools/catalog.js
View File

@@ -135,7 +135,9 @@ _.extend(Catalog.prototype, {
self.packages = [];
self.versions = [];
self.builds = [];
self._insertServerPackages(serverPackageData);
if (serverPackageData) {
self._insertServerPackages(serverPackageData);
}
self._addLocalPackageOverrides(true /* setInitialized */);
},
@@ -212,7 +214,7 @@ _.extend(Catalog.prototype, {
var packageSources = {}; // name to PackageSource
var versionIds = {}; // name to _id of the created Version record
_.each(self.effectiveLocalPackages, function (packageDir, name) {
var packageSource = new PackageSource(packageDir);
var packageSource = new PackageSource;
packageSource.initFromPackageDir(name, packageDir);
packageSources[name] = packageSource;
@@ -324,7 +326,7 @@ _.extend(Catalog.prototype, {
var sourcePath = self.effectiveLocalPackages[name];
var buildDir = path.join(sourcePath, '.build');
if (fs.existsSync(buildDir)) {
var unipackage = new Unipackage(sourcePath);
var unipackage = new Unipackage;
unipackage.initFromPath(name, buildDir, { buildOfPath: sourcePath });
if (compiler.checkUpToDate(packageSources[name], unipackage)) {
return unipackage;

10
tools/commands.js
View File

@@ -1578,12 +1578,12 @@ main.registerCommand({
// still confirming that it matches the name of the directory)
var packageName = path.basename(options.packageDir.toLowerCase());
packageSource = new PackageSource(options.packageDir);
packageSource = new PackageSource;
packageSource.initFromPackageDir(packageName, options.packageDir);
if (buildmessage.jobHasMessages())
return; // already have errors, so skip the build
compileResult = compiler.compile(packageSource);
compileResult = compiler.compile(packageSource, { officialBuild: true });
});
if (messages.hasMessages()) {
@@ -1716,9 +1716,11 @@ main.registerCommand({
return 1;
}
var packageSource = new PackageSource(packageDir);
var packageSource = new PackageSource;
packageSource.initFromPackageDir(options.name, packageDir);
var unipackage = compiler.compile(packageSource).unipackage;
var unipackage = compiler.compile(packageSource, {
officialBuild: true
}).unipackage;
unipackage.saveToPath(path.join(packageDir, '.build'));
var conn;

156
tools/compiler.js
View File

@@ -26,7 +26,7 @@ var compiler = exports;
// end up as watched dependencies. (At least for now, packages only used in
// target creation (eg minifiers and dev-bundle-fetcher) don't require you to
// update BUILT_BY, though you will need to quit and rerun "meteor run".)
compiler.BUILT_BY = 'meteor/10';
compiler.BUILT_BY = 'meteor/11';
// XXX where should this go? I'll make it a random utility function
// for now
@@ -599,16 +599,27 @@ var compileSlice = function (unipackage, inputSlice, packageLoader,
// the appropriate compiler plugins. Once build has completed, any errors
// detected in the package will have been emitted to buildmessage.
//
// Options:
// - officialBuild: defaults to false. If false, then we will compute a
// build identifier (a hash of the package's dependency versions and
// source files) and include it as part of the unipackage's version
// string. If true, then we will use the version that is contained in
// the package's source. You should set it to true when you are
// building a package to publish as an official build with the
// package server.
//
// Returns an object with keys:
// - unipackage: the build Unipackage
// - sources: array of source files (identified by their path on local
// disk) that were used by the build (the source files you'd have to
// ship to a different machine to replicate the build there)
compiler.compile = function (packageSource) {
compiler.compile = function (packageSource, options) {
var sources = [];
var pluginWatchSet = packageSource.pluginWatchSet.clone();
var plugins = {};
options = _.extend({ officialBuild: false }, options);
// Determine versions of build-time dependencies
var buildTimeDeps = determineBuildTimeDependencies(packageSource);
@@ -681,22 +692,7 @@ compiler.compile = function (packageSource) {
}
}
// XXX XXX HERE HERE
//
// Unless the 'officalBuild' option was set, compute a build
// identifier by finding the versions of all of our package
// dependencies (direct and plugin dependencies) -- real versions,
// not +local version, which means a lookup in the catalog -- and
// hashing them together (in a structured way with what they go
// with, canoncialized as well as possible) with the contents of the
// merged watchsets of our slices and our plugins, with paths
// relativized somehow (that last bit may be tricky!)
//
// Then -- again, unless 'officialBuild' was set -- modify version
// by adding +<buildid> to the version (it's an error if you already
// had one, I guess).
var unipackage = new Unipackage();
var unipackage = new Unipackage;
unipackage.initFromOptions({
name: packageSource.name,
metadata: packageSource.metadata,
@@ -704,7 +700,6 @@ compiler.compile = function (packageSource) {
earliestCompatibleVersion: packageSource.earliestCompatibleVersion,
defaultSlices: packageSource.defaultSlices,
testSlices: packageSource.testSlices,
packageDirectoryForBuildInfo: packageSource.packageDirectoryForBuildInfo,
plugins: plugins,
pluginWatchSet: pluginWatchSet,
buildTimeDirectDependencies: buildTimeDeps.directDependencies,
@@ -723,6 +718,27 @@ compiler.compile = function (packageSource) {
sources.push.apply(sources, sliceSources);
});
// XXX what should we do if the PackageSource doesn't have a version?
// (e.g. a plugin)
if (! options.officialBuild && packageSource.version) {
// XXX I have no idea if this should be using buildmessage.enterJob
// or not. test what happens on error
buildmessage.enterJob({
title: "compute build identifier for package `" +
packageSource.name + "`",
rootPath: packageSource.sourceRoot
}, function () {
if (packageSource.version.indexOf("+") !== -1) {
buildmessage.error("cannot compute build identifier for package `" +
packageSource.name + "` version " +
packageSource.version + "because it already " +
"has a build identifier");
} else {
unipackage.addBuildIdentifierToVersion();
}
});
}
return {
sources: _.uniq(sources),
unipackage: unipackage
@@ -773,47 +789,88 @@ compiler.getBuildOrderConstraints = function (packageSource) {
// identical code). True if we have dependency info and it
// says that the package is up-to-date. False if a source file or
// build-time dependency has changed.
//
// 'what' identifies the build to check for up-to-dateness and is an
// object with exactly one of the following keys:
// - path: a path on disk to a unipackage
// - unipackage: a Unipackage object
compiler.checkUpToDate = function (packageSource, unipackage) {
if (unipackage.forceNotUpToDate)
return false;
// Do we think we'd generate different contents than the tool that
// built this package?
if (unipackage.builtBy !== compiler.BUILT_BY)
if (unipackage.builtBy !== compiler.BUILT_BY) {
// XXX XXX XXX XXX XXX XXX XXX
//
// This branch is not currently in a state where we can build
// packages with plugins, so we explicitly do NOT want to trigger
// re-builds of packages built by different versions of meteor.
//
// Once this branch is in a state where we CAN build packages
// a-fresh, then we should change this back to "return false".
//
// XXX XXX XXX XXX XXX XXX XXX
console.log("XXX warning: considering package",
packageSource.name, "to be up to date because",
"it was built by <", compiler.BUILT_BY,
"and this makes no sense at all");
return true;
return false;
}
// XXX XXX XXX
var pluginProviderPackageDirs = unipackage.pluginProviderPackageDirs;
var buildTimeDeps = determineBuildTimeDependencies(packageSource);
/*
// XXX XXX this shouldn't work this way at all. instead it should
// just get the resolved build-time dependencies from packageSource
// and make sure they match the versions that were used for the
// build.
var packageLoader = XXX;
// Compute the unipackage's direct and plugin dependencies to
// `buildTimeDeps`, by comparing versions (including build
// identifiers).
// Are all of the packages we directly use (which can provide
// plugins which affect compilation) resolving to the same
// directory? (eg, have we updated our release version to something
// with a new version of a package?)
var packageResolutionsSame = _.all(
_pluginProviderPackageDirs, function (packageDir, name) {
return packageLoader.getLoadPathForPackage(name) === packageDir;
});
if (! packageResolutionsSame)
if (_.keys(buildTimeDeps.directDependencies).length !==
_.keys(unipackage.buildTimeDirectDependencies).length) {
return false;
*/
}
// XXX as we're checking build-time dependency freshness in the
// future, remember to not rely on
// packageSource.directBuildTimeDependencies, which may contain
// versions like 1.2.3+local, but instead get versions with real
// build ids through the catalog
var directDepsPackageLoader = new PackageLoader(
buildTimeDeps.directDependencies);
var directDepsMatch = _.all(
buildTimeDeps.directDependencies,
function (version, packageName) {
var loadedPackage = directDepsPackageLoader.getPackage(packageName);
// XXX Check that `versionWithBuildId` is the same as `version`
// except for the build id?
return (loadedPackage &&
unipackage.buildTimeDirectDependencies[packageName] ===
loadedPackage.version);
}
);
if (! directDepsMatch) {
return false;
}
if (_.keys(buildTimeDeps.pluginDependencies).length !==
_.keys(unipackage.buildTimePluginDependencies).length) {
return false;
}
var pluginDepsMatch = _.all(
buildTimeDeps.pluginDependencies,
function (pluginDeps, pluginName) {
// For each plugin, check that the resolved build-time deps for
// that plugin match the unipackage's build time deps for it.
var packageLoaderForPlugin = new PackageLoader(
buildTimeDeps.pluginDependencies
);
var unipackagePluginDeps = unipackage.buildTimePluginDependencies[pluginName];
if (! unipackagePluginDeps ||
_.keys(pluginDeps).length !== _.keys(unipackagePluginDeps).length) {
return false;
}
return _.all(pluginDeps, function (version, packageName) {
var loadedPackage = packageLoaderForPlugin.getPackage(packageName);
return loadedPackage &&
unipackagePluginDeps[packageName] === loadedPackage.version;
});
}
);
if (! pluginDepsMatch) {
return false;
}
var watchSet = new watch.WatchSet();
watchSet.merge(unipackage.pluginWatchSet);
@@ -821,8 +878,9 @@ compiler.checkUpToDate = function (packageSource, unipackage) {
watchSet.merge(slice.watchSet);
});
if (! watch.isUpToDate(watchSet))
if (! watch.isUpToDate(watchSet)) {
return false;
}
return true;
};

11
tools/package-cache.js
View File

@@ -70,14 +70,15 @@ _.extend(PackageCache.prototype, {
delete self.softReloadCache[loadPath];
var isUpToDate;
var unipackage;
if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) {
// We don't even have the source to this package, so it must
// be up to date.
isUpToDate = true;
} else {
var packageSource = new PackageSource(loadPath);
var packageSource = new PackageSource;
packageSource.initFromPackageDir(name, loadPath);
var unipackage = new Unipackage(loadPath);
unipackage = new Unipackage;
unipackage.initFromPath(name, entry.buildDir);
isUpToDate = compiler.checkUpToDate(packageSource, entry.pkg);
}
@@ -94,7 +95,7 @@ _.extend(PackageCache.prototype, {
// Does loadPath point directly at a unipackage (rather than a
// source tree?)
if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) {
var unipackage = new Unipackage(loadPath);
unipackage = new Unipackage;
unipackage.initFromPath(name, loadPath);
self.loadedPackages[loadPath] = {
pkg: unipackage,
@@ -105,13 +106,13 @@ _.extend(PackageCache.prototype, {
};
// It's a source tree. Load it.
var packageSource = new PackageSource(loadPath);
var packageSource = new PackageSource;
packageSource.initFromPackageDir(name, loadPath);
// Does it have an up-to-date build?
var buildDir = path.join(loadPath, '.build');
if (fs.existsSync(buildDir)) {
var unipackage = new Unipackage(loadPath);
unipackage = new Unipackage;
unipackage.initFromPath(name, buildDir);
if (compiler.checkUpToDate(packageSource, unipackage)) {
self.loadedPackages[loadPath] = { pkg: unipackage,

11
tools/package-source.js
View File

@@ -171,7 +171,7 @@ var SourceSlice = function (pkg, options) {
// PackageSource
///////////////////////////////////////////////////////////////////////////////
var PackageSource = function (packageDirectoryForBuildInfo) {
var PackageSource = function () {
var self = this;
// The name of the package, or null for an app pseudo-package or
@@ -193,15 +193,6 @@ var PackageSource = function (packageDirectoryForBuildInfo) {
// it's still nice to get it right).
self.serveRoot = null;
// The package's directory. This is used only by other packages that use this
// package in their buildinfo.json (to detect that they need to be rebuilt if
// the PackageLoader resolves it to a different package); it is not used to
// read files or anything else. Notably, it should be the same if a package is
// read from a source tree or read from the .build unipackage inside that
// source tree.
// XXX can this go away now?
self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo;
// Package metadata. Keys are 'summary' and 'internal'. Currently
// both of these are optional.
self.metadata = {};

109
tools/unipackage.js
View File

@@ -8,6 +8,7 @@ var path = require('path');
var Builder = require('./builder.js');
var bundler = require('./bundler.js');
var watch = require('./watch.js');
var PackageLoader = require('./package-loader.js');
var rejectBadPath = function (p) {
if (p.match(/\.\./))
@@ -179,7 +180,7 @@ _.extend(UnipackageSlice.prototype, {
///////////////////////////////////////////////////////////////////////////////
// XXX document
var Unipackage = function (packageDirectoryForBuildInfo) {
var Unipackage = function () {
var self = this;
// These have the same meaning as in PackageSource.
@@ -190,10 +191,6 @@ var Unipackage = function (packageDirectoryForBuildInfo) {
self.defaultSlices = {};
self.testSlices = {};
// XXX this is likely to go away once we have build versions
// (also in PackageSource)
self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo;
// Build slices. Array of UnipackageSlice.
self.slices = [];
@@ -211,7 +208,6 @@ var Unipackage = function (packageDirectoryForBuildInfo) {
// The versions that we used at build time for each of our direct
// dependencies. Map from package name to version string.
// XXX save to disk
self.buildTimeDirectDependencies = null;
// The complete list of versions (including transitive dependencies)
@@ -262,7 +258,6 @@ _.extend(Unipackage.prototype, {
self.earliestCompatibleVersion = options.earliestCompatibleVersion;
self.defaultSlices = options.defaultSlices;
self.testSlices = options.testSlices;
self.packageDirectoryForBuildInfo = options.packageDirectoryForBuildInfo;
self.plugins = options.plugins;
self.pluginWatchSet = options.pluginWatchSet;
self.buildTimeDirectDependencies = options.buildTimeDirectDependencies;
@@ -463,6 +458,10 @@ _.extend(Unipackage.prototype, {
// Read basic buildinfo.json info
self.builtBy = buildInfoJson.builtBy || null;
self.buildTimeDirectDependencies =
buildInfoJson.buildTimeDirectDependencies || null;
self.buildTimePluginDependencies =
buildInfoJson.buildTimePluginDependencies || null;
if (options.buildOfPath &&
(buildInfoJson.source !== options.buildOfPath)) {
@@ -657,7 +656,9 @@ _.extend(Unipackage.prototype, {
sliceDependencies: { },
pluginDependencies: self.pluginWatchSet.toJSON(),
pluginProviderPackages: self.pluginProviderPackageDirs,
source: options.buildOfPath || undefined
source: options.buildOfPath || undefined,
buildTimeDirectDependencies: self._buildTimeDirectDependenciesWithBuildIds(),
buildTimePluginDependencies: self._buildTimePluginDependenciesWithBuildIds()
};
builder.reserve("unipackage.json");
@@ -851,6 +852,98 @@ _.extend(Unipackage.prototype, {
builder.abort();
throw e;
}
},
_buildTimeDirectDependenciesWithBuildIds: function () {
var self = this;
var directDepsLoader = new PackageLoader({
versions: self.buildTimeDirectDependencies
});
var result = {};
_.each(self.buildTimeDirectDependencies, function (version, packageName) {
var unipackage = directDepsLoader.getPackage(packageName);
result[packageName] = unipackage.version;
});
return result;
},
_buildTimePluginDependenciesWithBuildIds: function () {
var self = this;
var result = {};
_.each(self.buildTimePluginDependencies, function (deps, pluginName) {
var pluginPackageLoader = new PackageLoader({ versions: deps });
result[pluginName] = {};
_.each(deps, function (version, packageName) {
var unipackage = pluginPackageLoader.getPackage(packageName);
result[pluginName][packageName] = unipackage.version;
});
});
return result;
},
// Computes a hash of the versions of all the package's dependencies
// (direct and plugin dependencies) and the slices' and plugins' watch
// sets. Adds the result as a build identifier to the unipackage's
// version. The caller is responsible for checking whether the
// existing version has a build identifier already.
addBuildIdentifierToVersion: function () {
var self = this;
// Gather all the dependencies' versions and organize them into
// arrays. We use arrays to avoid relying on the order of
// stringified object keys.
var directDeps = [];
_.each(
self._buildTimeDirectDependenciesWithBuildIds(),
function (version, packageName) {
directDeps.push([packageName, version]);
}
);
// Sort direct dependencies by package name (which is the "0" property
// of each element in the array).
directDeps = _.sortBy(directDeps, "0");
var pluginDeps = [];
_.each(
self._buildTimePluginDependenciesWithBuildIds(),
function (versions, pluginName) {
var pluginDepsLoader = new PackageLoader({ versions: versions });
var singlePluginDeps = [];
_.each(versions, function (version, packageName) {
var unipackage = pluginDepsLoader.getPackage(packageName);
singlePluginDeps.push([unipackage.name, unipackage.version]);
});
singlePluginDeps = _.sortBy(singlePluginDeps, "0");
pluginDeps.push([pluginName, singlePluginDeps]);
}
);
pluginDeps = _.sortBy(pluginDeps, "0");
// Now that we have versions for all our dependencies, canonicalize
// the slices' and plugins' watch sets.
// XXX Do we need to relativize paths? Why?
var watchFiles = [];
var watchSet = new watch.WatchSet();
watchSet.merge(self.pluginWatchSet);
_.each(self.slices, function (slice) {
watchSet.merge(slice.watchSet);
});
_.each(watchSet.files, function (hash, fileAbsPath) {
watchFiles.push([fileAbsPath, hash]);
});
watchFiles = _.sortBy(watchFiles, "0");
// Stick all our info into one big array, stringify it, and hash it.
var buildIdInfo = [
directDeps,
pluginDeps,
watchFiles
];
var crypto = require('crypto');
var hasher = crypto.createHash('sha1');
hasher.update(JSON.stringify(buildIdInfo));
var buildId = hasher.digest('hex');
self.version = self.version + "+" + buildId;
}
});

6
tools/watch.js
View File

@@ -138,9 +138,9 @@ _.extend(WatchSet.prototype, {
self.alwaysFire = true;
return;
}
// _.each(other.files, function (hash, name) {
// self.addFile(name, hash);
// });
_.each(other.files, function (hash, name) {
self.addFile(name, hash);
});
_.each(other.directories, function (dir) {
// XXX this doesn't deep-clone the directory, but I think these objects
// are never mutated #WatchSetShallowClone