github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

WIP: Break Library into PackageLoader, PackageCache, Catalog

Browse Source
This commit is contained in:
Geoff Schmidt
2014-03-11 18:10:28 -07:00
parent 3f8beb0935
commit 5f0d1714eb
10 changed files with 489 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
tools/bundler.js
View File

@@ -394,15 +394,15 @@ _.extend(File.prototype, {
///////////////////////////////////////////////////////////////////////////////
// options:
// - library: package library to use for resolving package dependenices
// - packageLoader: PackageLoader to use for resolving package dependenices
// - arch: the architecture to build
//
// see subclasses for additional options
var Target = function (options) {
var self = this;
// Package library to use for resolving package dependenices.
self.library = options.library;
// PackageLoader to use for resolving package dependenices.
self.packageLoader = options.packageLoader;
// Something like "browser.w3c" or "os" or "os.osx.x86_64"
self.arch = options.arch;
@@ -498,19 +498,19 @@ _.extend(Target.prototype, {
// strings) whose test slices should be included
_determineLoadOrder: function (options) {
var self = this;
var library = self.library;
var packageLoader = self.packageLoader;
// Find the roots
var rootSlices =
_.flatten([
_.map(options.packages || [], function (p) {
if (typeof p === "string")
return library.getSlices(p, self.arch);
return packageLoader.getSlices(p, self.arch);
else
return p.getDefaultSlices(self.arch);
}),
_.map(options.test || [], function (p) {
var pkg = (typeof p === "string" ? library.get(p) : p);
var pkg = (typeof p === "string" ? packageLoader.getPackage(p) : p);
return pkg.getTestSlices(self.arch);
})
]);
@@ -528,7 +528,8 @@ _.extend(Target.prototype, {
if (_.has(getsUsed, slice.id))
return;
getsUsed[slice.id] = slice;
slice.eachUsedSlice(self.arch, {skipWeak: true}, addToGetsUsed);
slice.eachUsedSlice(self.arch, packageLoader,
{skipWeak: true}, addToGetsUsed);
};
_.each(rootSlices, addToGetsUsed);
@@ -559,7 +560,8 @@ _.extend(Target.prototype, {
// those edge. Because we did follow those edges in Phase 1, any unordered
// slices were at some point in `needed` and will not be left out).
slice.eachUsedSlice(
self.arch, {skipUnordered: true}, function (usedSlice, useOptions) {
self.arch, packageLoader, {skipUnordered: true},
function (usedSlice, useOptions) {
// If this is a weak dependency, and nothing else in the target had a
// strong dependency on it, then ignore this edge.
if (useOptions.weak && ! _.has(getsUsed, usedSlice.id))
@@ -602,7 +604,7 @@ _.extend(Target.prototype, {
var isApp = ! slice.pkg.name;
// Emit the resources
var resources = slice.getResources(self.arch);
var resources = slice.getResources(self.arch, self.packageLoader);
// First, find all the assets, so that we can associate them with each js
// resource (for os slices).
@@ -699,8 +701,8 @@ _.extend(Target.prototype, {
// Depend on the source files that produced these resources.
self.watchSet.merge(slice.watchSet);
// Remember the library resolution of all packages used in these
// resources.
// Remember the versions of all of the build-time dependencies
// that were used in these resources.
// XXX assumes that this merges cleanly
_.extend(self.pluginProviderPackageDirs,
slice.pkg.pluginProviderPackageDirs)
@@ -1363,7 +1365,7 @@ var ServerTarget = function (options) {
self.clientTarget = options.clientTarget;
self.releaseName = options.releaseName;
self.library = options.library;
self.packageLoader = options.packageLoader;
if (! archinfo.matches(self.arch, "os"))
throw new Error("ServerTarget targeting something that isn't a server?");
@@ -1636,6 +1638,10 @@ var writeSiteArchive = function (targets, outputPath, options) {
* untarred bundle) should go. This directory will be created if it
* doesn't exist, and removed first if it does exist.
*
* - packageLoader: Required. The PackageLoader used to retrieve any
* packages needed by the app or its dependencies at the appropriate
* versions.
*
* - nodeModulesMode: what to do about the core npm modules needed by
* the server bootstrap. one of:
* - 'copy': copy from a prebuilt local installation. used by
@@ -1682,12 +1688,12 @@ exports.bundle = function (options) {
var appDir = options.appDir;
var outputPath = options.outputPath;
var nodeModulesMode = options.nodeModulesMode || 'copy';
var packageLoader = options.packageLoader;
var buildOptions = options.buildOptions || {};
if (! release.usingRightReleaseForApp(appDir))
throw new Error("running wrong release for app?");
var library = release.current.library;
var releaseName =
release.current.isCheckout() ? "none" : release.current.name;
var builtBy = "Meteor" + (release.current.name ?
@@ -1704,7 +1710,7 @@ exports.bundle = function (options) {
var makeClientTarget = function (app) {
var client = new ClientTarget({
library: library,
packageLoader: packageLoader,
arch: "browser"
});
@@ -1720,7 +1726,7 @@ exports.bundle = function (options) {
var makeBlankClientTarget = function () {
var client = new ClientTarget({
library: library,
packageLoader: packageLoader,
arch: "browser"
});
client.make({
@@ -1733,7 +1739,7 @@ exports.bundle = function (options) {
var makeServerTarget = function (app, clientTarget) {
var targetOptions = {
library: library,
packageLoader: packageLoader,
arch: buildOptions.arch || archinfo.host(),
releaseName: releaseName
};
@@ -1761,7 +1767,8 @@ exports.bundle = function (options) {
if (includeDefaultTargets) {
// Create a Package object that represents the app
var app = library.getForApp(appDir, ignoreFiles);
var app = packageLoader.getPackageCache().loadAppAtPath(appDir,
ignoreFiles);
// Client
var client = makeClientTarget(app);
@@ -1871,11 +1878,13 @@ exports.bundle = function (options) {
_.each(programs, function (p) {
// Read this directory as a package and create a target from
// it
library.override(p.name, p.path);
var pkg = packageLoader.getPackageCache().
loadPackageAtPath(p.name, p.loadPath);
var target;
switch (p.type) {
case "server":
target = makeServerTarget(p.name);
target = makeServerTarget(pkg);
break;
case "traditional":
var clientTarget;
@@ -1900,10 +1909,10 @@ exports.bundle = function (options) {
// We don't check whether targets[p.client] is actually a
// ClientTarget. If you want to be clever, go ahead.
target = makeServerTarget(p.name, clientTarget);
target = makeServerTarget(pkg, clientTarget);
break;
case "client":
target = makeClientTarget(p.name);
target = makeClientTarget(pkg);
break;
default:
buildmessage.error(
@@ -1912,7 +1921,6 @@ exports.bundle = function (options) {
// recover by ignoring target
return;
};
library.removeOverride(p.name);
targets[p.name] = target;
});
@@ -1921,10 +1929,6 @@ exports.bundle = function (options) {
if (! (controlProgram in targets))
controlProgram = undefined;
// Make sure notice when somebody adds a package to the app packages dir
// that may override a warehouse package.
library.watchLocalPackageDirs(watchSet);
// Write to disk
starResult = writeSiteArchive(targets, outputPath, {
nodeModulesMode: nodeModulesMode,
@@ -1960,7 +1964,7 @@ exports.bundle = function (options) {
// letting exceptions escape?
//
// options:
// - library: required. the Library for resolving package dependencies
// - packageLoader: required. the PackageLoader for resolving dependencies
// - name: required. a name for this image (cosmetic, but will appear
// in, eg, error messages) -- technically speaking, this is the name
// of the package created to contain the sources and package
@@ -1987,7 +1991,7 @@ exports.buildJsImage = function (options) {
if (! options.name)
throw new Error("Must provide a name");
var pkg = new packages.Package(options.library);
var pkg = new packages.Package;
pkg.initFromOptions(options.name, {
sliceName: "plugin",
@@ -1998,10 +2002,10 @@ exports.buildJsImage = function (options) {
npmDependencies: options.npmDependencies,
npmDir: options.npmDir
});
pkg.build();
pkg.build(options.packageLoader);
var target = new JsImageTarget({
library: options.library,
packageLoader: options.packageLoader,
// This function does not yet support cross-compilation (neither does
// initFromOptions). That's OK for now since we're only trying to support
// cross-bundling, not cross-package-building, and this function is only

89
tools/catalog.js
View File

@@ -11,8 +11,6 @@ var catalog = exports;
// we know about (including packages on the package server that we
// haven't actually download yet).
//
// 'library' is a library to use for loading packages.
//
// Options:
// - localPackageDirs: paths on local disk, that contain
// subdirectories, that each contain a package that should override
@@ -22,11 +20,10 @@ var catalog = exports;
// server. Directories that don't exist (or paths that aren't
// directories) will be silently ignored.
catalog.Catalog = function (library, options) {
catalog.Catalog = function (options) {
var self = this;
options = options || {};
self.library = library;
self.loaded = false; // #CatalogLazyLoading
// Package server data
@@ -98,8 +95,7 @@ _.extend(catalog.Catalog.prototype, {
// XXX XXX for now, get the package name from the
// directory. in a future refactor, should instead build the
// package right here and get the name from the (not yet
// added) 'name' attribute in package.js. in this future,
// Library caches packages by path rather than by name.
// added) 'name' attribute in package.js.
if (! _.has(self.effectiveLocalPackages, item))
self.effectiveLocalPackages[item] = packageDir;
}
@@ -112,10 +108,10 @@ _.extend(catalog.Catalog.prototype, {
// the collections, shadowing any versions of those packages from
// the package server.
_.each(self.effectiveLocalPackages, function (packageDir, name) {
var cache = new packageCache.PackageCache; // XXX make singleton
// Load the package
var pkg = self.library.get(name, {
packageDir: packageDir
});
var pkg = cache.loadPackageAtPath(name, packageDir);
// Hide any versions from the package server
self.versions.find({ packageName: name }).forEach(function (versionInfo) {
@@ -192,6 +188,81 @@ _.extend(catalog.Catalog.prototype, {
return _.has(self.effectiveLocalPackages, name);
},
// Register local package directories with a watchSet. We want to know if a
// package is created or deleted, which includes both its top-level source
// directory and its main package metadata file.
watchLocalPackageDirs: function (watchSet) {
var self = this;
_.each(self.localPackageDirs, function (packageDir) {
var packages = watch.readAndWatchDirectory(watchSet, {
absPath: packageDir,
include: [/\/$/]
});
_.each(packages, function (p) {
watch.readAndWatchFile(watchSet,
path.join(packageDir, p, 'package.js'));
watch.readAndWatchFile(watchSet,
path.join(packageDir, p, 'unipackage.json'));
});
});
},
// Rebuild all source packages in our search paths. If two packages
// have the same name only the one that we would load will get
// rebuilt.
//
// Returns a count of packages rebuilt.
rebuildLocalPackages: function () {
var self = this;
// We're going to need a PackageCache -- just for the purpose of
// forcing builds of the packages. We'll just create a new one and
// throw it away when we're done. That will mean that the builds
// we do won't be cached in memory, but since this is only ever
// called to implement a command-line command, that shouldn't be a
// problem.
//
// Note that if we were reusing an existing PackageCache, we'd
// want to call refresh() on it first.
var packageCache = new packageCache.PackageCache;
// Delete any that are source packages with builds.
var count = 0;
_.each(self.effectiveLocalPackages, function (loadPath, name) {
var buildDir = path.join(loadPath, '.build');
files.rm_recursive(loadPath);
});
// Now reload them, forcing a rebuild. We have to do this in two
// passes because otherwise we might end up rebuilding a package
// and then immediately deleting it.
_.each(self.effectiveLocalPackages, function (loadPath, name) {
packageCache.loadPackageAtPath(name, loadPath, { throwOnError: false });
count ++;
});
return count;
},
// Given a name and a version of a package, return a path on disk
// from which we can load it. If we don't have it on disk (we
// haven't downloaded it, or it just plain doesn't exist in the
// catalog) return null.
//
// Doesn't download packages. Downloading should be done at the time
// that .meteor/versions is updated.
getLoadPathForPackage: function (name, version) {
var self = this;
if (_.has(self.effectiveLocalPackages, name)) {
// XXX should confirm that the version on disk actually matches
// the requested version
return self.effectiveLocalPackages[name];
}
return tropohouse.packagePath(name, version);
},
// Return an array with the names of all of the packages that we
// know about, in no particular order.
getAllPackageNames: function () {

80
tools/commands.js
View File

@@ -20,6 +20,7 @@ var httpHelpers = require('./http-helpers.js');
var archinfo = require('./archinfo.js');
var tropohouse = require('./tropohouse.js');
var packages = require('./packages.js');
var packageLoader = require('./package-loader.js');
// Given a site name passed on the command line (eg, 'mysite'), return
// a fully-qualified hostname ('mysite.meteor.com').
@@ -48,18 +49,28 @@ var hostedWithGalaxy = function (site) {
};
// Get all packages available. Returns a map from the package name to
// a Package object.
// a Package object -- for the latest version of the package.
//
// If problems happen while generating the list, print appropriate
// messages to stderr and return null.
var getPackages = function () {
var result = release.current.library.list();
if (result.packages)
return result.packages;
var ret = {};
process.stderr.write("=> Errors while scanning packages:\n\n");
process.stderr.write(result.messages.formatMessages());
return null;
var catalog = release.current.catalog;
var messages = buildmessage.capture(function () {
var names = catalog.getAllPackageNames();
_.each(names, function (name) {
ret[name] = catalog.getLatestVersion(name);
});
});
if (message.hasMessages()) {
process.stderr.write("=> Errors while scanning packages:\n\n");
process.stderr.write(result.messages.formatMessages());
return null;
} else {
return ret;
}
};
var XXX_DEPLOY_ARCH = 'os.linux.x86_64';
@@ -148,8 +159,9 @@ main.registerCommand({
if (! packages)
return 1; // build failed
// XXX we rely on the fact that library.list() forces all of the
// packages to be built. #ListingPackagesImpliesBuildingThem
// XXX we rely on the fact that loading a package, even to get its
// metadata, forces it to be built if it's a source
// package. #ListingPackagesImpliesBuildingThem
}
});
@@ -527,26 +539,7 @@ main.registerCommand({
maxArgs: Infinity,
requiresApp: true
}, function (options) {
var all = getPackages();
if (! all)
return 1;
var using = {};
_.each(project.getPackages(options.appDir), function (name) {
using[name] = true;
});
_.each(options.args, function (name) {
if (! _.has(all, name)) {
process.stderr.write(name + ": no such package\n");
} else if (_.has(using, name)) {
process.stderr.write(name + ": already using\n");
} else {
project.addPackage(options.appDir, name);
var note = all[name].metadata.summary || '';
process.stderr.write(name + ": " + note + "\n");
}
});
throw new Error("XXX replace with add-package");
});
@@ -733,16 +726,7 @@ main.registerCommand({
return;
}
var list = getPackages();
if (! list)
return 1;
var names = _.keys(list);
names.sort();
var pkgs = [];
_.each(names, function (name) {
pkgs.push(list[name]);
});
process.stdout.write("\n" + library.formatList(pkgs) + "\n");
throw new Error("XXX replace with list-all or remove completely");
});
@@ -1184,7 +1168,7 @@ main.registerCommand({
// on each other.
//
// Note: testRunnerAppDir deliberately DOES NOT MATCH the app
// package search path baked into release.current.library: we are
// package search path baked into release.current.catalog: we are
// bundling the test runner app, but finding app packages from the
// current app (if any).
var testRunnerAppDir = files.mkdtemp('meteor-test-run');
@@ -1235,7 +1219,7 @@ main.registerCommand({
hidden: true
}, function (options) {
if (options.appDir) {
// The library doesn't know about other programs in your app. Let's blow
// The catalog doesn't know about other programs in your app. Let's blow
// away their .build directories if they have them, and not rebuild
// them. Sort of hacky, but eh.
var programsDir = path.join(options.appDir, 'programs');
@@ -1254,7 +1238,7 @@ main.registerCommand({
var count = null;
var messages = buildmessage.capture(function () {
count = release.current.library.rebuildAll();
count = release.current.catalog.rebuildLocalPackages();
});
if (count)
console.log("Built " + count + " packages.");
@@ -1585,9 +1569,17 @@ main.registerCommand({
return 1;
}
var pkg = new packages.Package(release.current.library, packageDir);
// #RunningTheConstraintSolverToBuildAPackage
var versions = { }; // XXX XXX actually run the constraint solver!
var loader = new packageLoader.PackageLoader({
catalog: release.current.catalog,
versions: versions,
packageCache: new PackageCache
});
var pkg = new packages.Package(packageDir);
pkg.initFromPackageDir(options.name, packageDir);
pkg.build();
pkg.build(loader);
pkg.saveAsUnipackage(path.join(packageDir, '.build'));
var conn = packageClient.loggedInPackagesConnection();

2
tools/library.js
View File

@@ -211,7 +211,7 @@ _.extend(Library.prototype, {
if (! packageDir) {
if (options.throwOnError === false)
return null;
//XXX buildmessage.error("package not available: " + name);
buildmessage.error("package not available: " + name);
// recover by returning a dummy (empty) package
var pkg = new packages.Package(self);
pkg.initEmpty(name);

161
tools/package-cache.js Normal file
View File

@@ -0,0 +1,161 @@
// XXX XXX make this a global singleton and eliminate the calls to
// 'new PackageCache' and 'getPackageCache()'
var packageCache = exports;
packageCache.PackageCache = function () {
var self = this;
// both map from package load path to:
// - pkg: cached Package object
// - packageDir: directory from which it was loaded
self.softReloadCache = {};
self.loadedPackages = {};
};
_.extend(packageCache.PackageCache, {
// Force reload of changed packages. See description at loadPackageAtPath().
//
// If soft is false, the default, the cache is totally flushed and
// all packages are reloaded unconditionally.
//
// If soft is true, then built packages without dependency info (such as those
// from the warehouse) aren't reloaded (there's no way to rebuild them, after
// all), and if we loaded a built package with dependency info, we won't
// reload it if the dependency info says that its source files are still up to
// date. The ideas is that assuming the user is "following the rules", this
// will correctly reload any changed packages while in most cases avoiding
// nearly all reloading.
refresh: function (soft) {
var self = this;
soft = soft || false;
self.softReloadCache = soft ? self.loadedPackages : {};
self.loadedPackages = {};
},
// Given a path to a package on disk, retrieve a Package
// object. Options are:
// - forceRebuild: see documentation in PackageLoader.getPackage
//
// loadPackageAtPath() caches the packages it returns, meaning if
// you call loadPackageAtPath('/foo/bar') and later /foo/bar changes
// on disk, you won't see the changes. To flush the package cache
// and force all of the packages to be reloaded the next time
// loadPackageAtPath() is called for them, see refresh().
loadPackageAtPath: function (name, loadPath, options) {
var self = this;
options = options || {};
// Packages cached from previous calls
if (! options.forceRebuild && _.has(self.loadedPackages, loadPath)) {
return self.loadedPackages[loadPath].pkg;
}
// See if we can reuse a package that we have cached from before
// the last soft refresh.
if (! options.forceRebuild && _.has(self.softReloadCache, loadPath)) {
var entry = self.softReloadCache[loadPath];
// Either we will decide that the cache is invalid, or we will "upgrade"
// this entry into loadedPackages. Either way, it's not needed in
// softReloadCache any more.
delete self.softReloadCache[loadPath];
if (entry.pkg.checkUpToDate()) {
// Cache hit
self.loadedPackages[loadedPackages] = entry;
return entry.pkg;
}
}
// Load package from disk
var pkg = new packages.Package(loadPath);
if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) {
// It's an already-built package
if (options.forceRebuild) {
throw new Error('Cannot rebuild from a unipackage directory.');
}
pkg.initFromUnipackage(name, loadPath);
self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath};
} else {
// It's a source tree. Does it have a built unipackage inside it?
var buildDir = path.join(loadPath, '.build');
if (! options.forceRebuild &&
fs.existsSync(buildDir) &&
pkg.initFromUnipackage(name, buildDir,
{ onlyIfUpToDate: true,
buildOfPath: loadPath })) {
// We already had a build and it was up to date.
self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath};
} else {
// Either we didn't have a build, or it was out of date, or the
// caller wanted us to rebuild no matter what. Build the
// package.
buildmessage.enterJob({
title: "building package `" + name + "`",
rootPath: loadPath
}, function () {
// This has to be done in the right sequence: initialize
// (which loads the dependency list but does not get() those
// packages), then put the package into the package list,
// then call build() to get() the dependencies and finish
// the build. If you called build() before putting the
// package in the package list then you'd recurse
// forever. (build() needs the dependencies because it needs
// to look at the handlers registered by any plugins in the
// packages that we use.)
pkg.initFromPackageDir(name, loadPath);
self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath};
// #RunningTheConstraintSolverToBuildAPackage
var versions = { }; // XXX XXX actually run the constraint solver!
var loader = new packageLoader.PackageLoader({
catalog: release.current.catalog,
versions: versions,
packageCache: self
});
pkg.build(loader);
if (! buildmessage.jobHasMessages()) {
// Save it, for a fast load next time
try {
files.addToGitignore(loadPath, '.build*');
pkg.saveAsUnipackage(buildDir, { buildOfPath: loadPath });
} catch (e) {
// If we can't write to this directory, we don't get to cache our
// output, but otherwise life is good.
if (!(e && (e.code === 'EACCES' || e.code === 'EPERM')))
throw e;
}
}
});
}
}
return pkg;
},
// Get a package that represents an app. (ignoreFiles is optional
// and if given, it should be an array of regexps for filenames to
// ignore when scanning for source files.)
// XXX formerly called getForApp
loadAppAtPath: function (appDir, ignoreFiles) {
var self = this;
// #RunningTheConstraintSolverToBuildAPackage
var versions = { }; // XXX XXX actually run the constraint solver!
var loader = new packageLoader.PackageLoader({
catalog: release.current.catalog,
versions: versions,
packageCache: self
});
pkg.build(loader);
var pkg = new packages.Package;
pkg.initFromAppDir(appDir, ignoreFiles || []);
pkg.build(loader);
return pkg;
}
});

84
tools/package-loader.js Normal file
View File

@@ -0,0 +1,84 @@
var packageLoader = exports;
// options:
// catalog: the Catalog used to locate the packages on disk
// versions: a map from package name to the version to use
// packageCache: the PackageCache used to load packages and cache them
// in memory
packageLoader.PackageLoader = function (options) {
var self = this;
self.catalog = options.catalog;
self.versions = options.versions;
self.packageCache = options.packageCache;
};
_.extend(packageLoader.PackageLoader, {
// Given the name of a package, return a Package object, or throw an
// error if the package wasn't included in the 'versions' passed on
// initalization or isn't available (for example, hasn't been
// downloaded yet).
//
// Options are:
// - throwOnError: if true (the default), throw an error if the
// package can't be found. (If false is passed for throwOnError,
// then return null if the package can't be found.) When called
// inside buildmessage.enterJob, however, instead of throwing an
// error it will record a build error and return a dummy (empty)
// package.
// - forceRebuild: defaults to false. If true, we will initialize the
// package from the source and ignore a built unipackage if it
// exists. This option is ignored if you pass `name` as a Package.
//
// XXX rename to throwOnNotFound
getPackage: function (name, options) {
var self = this;
options = options || {};
if (options.throwOnError === undefined) {
options.throwOnError = true;
}
if (! _.has(self.versions, name))
throw new Error("no version chosen for package?");
var loadPath = self.catalog.getLoadPathForPackage(name,
self.versions[name]);
if (! loadPath) {
if (options.throwOnError === false)
return null;
buildmessage.error("package not available: " + name);
// recover by returning a dummy (empty) package
var pkg = new packages.Package;
pkg.initEmpty(name);
return pkg;
}
return self.packageCache.loadPackageAtPath(name, loadPath, {
forceRebuild: options.forceRebuild
});
},
// Given a slice set spec -- either a package name like "ddp", or a particular
// slice within the package like "ddp/client", or a parsed object like
// {package: "ddp", slice: "client"} -- return the list of matching slices (as
// an array of Slice objects) for a given architecture.
getSlices: function (spec, arch) {
var self = this;
if (typeof spec === "string")
spec = packages.parseSpec(spec);
var pkg = self.getPackage(spec.package, { throwOnError: true });
if (spec.slice)
return [pkg.getSingleSlice(spec.slice, arch)];
else
return pkg.getDefaultSlices(arch);
},
// Return the PackageCache that backs this loader.
getPackageCache: function () {
var self = this;
return self.packageCache;
}
});

133
tools/packages.js
View File

@@ -250,7 +250,11 @@ _.extend(Slice.prototype, {
// resulting JavaScript. Also add all provided source files to the
// package dependencies. Sets fields such as dependencies, exports,
// prelinkFiles, packageVariables, and resources.
build: function () {
//
// packageLoader is the PackageLoader to use to validate that the
// slice's dependencies actually exist (for cleaner error
// messages).
build: function (packageLoader) {
var self = this;
var isApp = ! self.pkg.name;
@@ -269,11 +273,12 @@ _.extend(Slice.prototype, {
_.each(['uses', 'implies'], function (field) {
var scrubbed = [];
_.each(self[field], function (u) {
var pkg = self.pkg.library.get(u.package, { throwOnError: false });
/* if (! pkg) {
var pkg = packageLoader.getPackage(u.package,
{ throwOnError: false });
if (! pkg) {
buildmessage.error("no such package: '" + u.package + "'");
// recover by omitting this package from the field
} else */
} else
scrubbed.push(u);
});
self[field] = scrubbed;
@@ -300,7 +305,8 @@ _.extend(Slice.prototype, {
var fileOptions = _.clone(source.fileOptions) || {};
var absPath = path.resolve(self.pkg.sourceRoot, relPath);
var filename = path.basename(relPath);
var handler = !fileOptions.isAsset && self._getSourceHandler(filename);
var handler = !fileOptions.isAsset &&
self._getSourceHandler(filename, packageLoader);
var file = watch.readAndWatchFileWithHash(self.watchSet, absPath);
var contents = file.contents;
@@ -551,7 +557,7 @@ _.extend(Slice.prototype, {
// plugin program itself uses), as well as the package.js file from every
// package we directly use (since changing the package.js may add or remove
// a plugin).
_.each(self._activePluginPackages(), function (otherPkg) {
_.each(self._activePluginPackages(packageLoader), function (otherPkg) {
self.watchSet.merge(otherPkg.pluginWatchSet);
// XXX this assumes this is not overwriting something different
self.pkg.pluginProviderPackageDirs[otherPkg.name] =
@@ -600,9 +606,11 @@ _.extend(Slice.prototype, {
// is resolved at bundle time. (On the other hand, when it comes to
// the extension handlers we'll use, we previously commited to those
// versions at package build ('compile') time.)
getResources: function (bundleArch) {
//
// packageLoader is the PackageLoader that should be used to resolve
// the package's bundle-time dependencies.
getResources: function (bundleArch, packageLoader) {
var self = this;
var library = self.pkg.library;
if (! self.isBuilt)
throw new Error("getting resources of unbuilt slice?" + self.pkg.name + " " + self.sliceName + " " + self.arch);
@@ -621,7 +629,8 @@ _.extend(Slice.prototype, {
// unrelated package in the target depends on something).
var imports = {}; // map from symbol to supplying package name
self.eachUsedSlice(
bundleArch, {skipWeak: true, skipUnordered: true}, function (otherSlice) {
bundleArch, packageLoader,
{skipWeak: true, skipUnordered: true}, function (otherSlice) {
if (! otherSlice.isBuilt)
throw new Error("dependency wasn't built?");
_.each(otherSlice.packageVariables, function (symbol) {
@@ -664,7 +673,10 @@ _.extend(Slice.prototype, {
// are transitively "implied" by used slices. (But not slices that are used by
// slices that we use!) Options are skipWeak and skipUnordered, meaning to
// ignore direct "uses" that are weak or unordered.
eachUsedSlice: function (arch, options, callback) {
//
// packageLoader is the PackageLoader that should be used to resolve
// the package's bundle-time dependencies.
eachUsedSlice: function (arch, packageLoader, options, callback) {
var self = this;
if (typeof options === "function") {
callback = options;
@@ -685,7 +697,8 @@ _.extend(Slice.prototype, {
var use = usesToProcess.shift();
var slices =
self.pkg.library.getSlices(_.pick(use, 'package', 'spec'), arch);
packageLoader.getSlices(_.pick(use, 'package', 'spec'),
arch);
_.each(slices, function (slice) {
if (_.has(processedSliceId, slice.id))
return;
@@ -704,7 +717,7 @@ _.extend(Slice.prototype, {
// Return an array of all plugins that are active in this slice, as
// a list of Packages.
_activePluginPackages: function () {
_activePluginPackages: function (packageLoader) {
var self = this;
// XXX we used to include our own extensions only if we were the
@@ -725,9 +738,12 @@ _.extend(Slice.prototype, {
// We pass archinfo.host here, not self.arch, because it may be more
// specific, and because plugins always have to run on the host
// architecture.
self.eachUsedSlice(archinfo.host(), {skipWeak: true}, function (usedSlice) {
ret.push(usedSlice.pkg);
});
self.eachUsedSlice(
archinfo.host(), packageLoader, {skipWeak: true},
function (usedSlice) {
ret.push(usedSlice.pkg);
}
);
// Only need one copy of each package.
ret = _.uniq(ret);
@@ -742,7 +758,7 @@ _.extend(Slice.prototype, {
// Get all extensions handlers registered in this slice, as a map
// from extension (no leading dot) to handler function. Throws an
// exception if two packages are registered for the same extension.
_allHandlers: function () {
_allHandlers: function (packageLoader) {
var self = this;
var ret = {};
@@ -761,7 +777,7 @@ _.extend(Slice.prototype, {
}
});
_.each(self._activePluginPackages(), function (otherPkg) {
_.each(self._activePluginPackages(packageLoader), function (otherPkg) {
_.each(otherPkg.sourceHandlers, function (handler, ext) {
if (ext in ret && ret[ext] !== handler) {
buildmessage.error(
@@ -783,17 +799,17 @@ _.extend(Slice.prototype, {
// Return a list of all of the extension that indicate source files
// for this slice, not including leading dots. Computed based on
// this.uses, so should only be called once that has been set.
registeredExtensions: function () {
_registeredExtensions: function (packageLoader) {
var self = this;
return _.keys(self._allHandlers());
return _.keys(self._allHandlers(packageLoader));
},
// Find the function that should be used to handle a source file for
// this slice, or return null if there isn't one. We'll use handlers
// that are defined in this package and in its immediate dependencies.
_getSourceHandler: function (filename) {
_getSourceHandler: function (filename, packageLoader) {
var self = this;
var handlers = self._allHandlers();
var handlers = self._allHandlers(packageLoader);
var parts = filename.split('.');
for (var i = 0; i < parts.length; i++) {
var extension = parts.slice(i).join('.');
@@ -820,7 +836,7 @@ _.extend(Slice.prototype, {
// (find better names, though).
var nextPackageId = 1;
var Package = function (library, packageDirectoryForBuildInfo) {
var Package = function (packageDirectoryForBuildInfo) {
var self = this;
// A unique ID (guaranteed to not be reused in this process -- if
@@ -850,16 +866,12 @@ var Package = function (library, packageDirectoryForBuildInfo) {
// The package's directory. This is used only by other packages that use this
// package in their buildinfo.json (to detect that they need to be rebuilt if
// the library's resolution of the package name changes); it is not used to
// the PackageLoader resolves it to a different package); it is not used to
// read files or anything else. Notably, it should be the same if a package is
// read from a source tree or read from the .build unipackage inside that
// source tree.
self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo;
// Package library that should be used to resolve this package's
// dependencies
self.library = library;
// Package metadata. Keys are 'summary' and 'internal'. Currently
// both of these are optional.
self.metadata = {};
@@ -1165,10 +1177,14 @@ _.extend(Package.prototype, {
// completed, any errors detected in the package will have been
// emitted to buildmessage.
//
// build() may retrieve the package's dependencies from the library,
// so it is illegal to call build() from library.get() (until the
// package has actually been put in the loaded package list).
build: function () {
// packageLoader is the PackageLoader to use for determining the
// package's build-time dependencies.
//
// Since build() retrieves the package's dependencies from the
// PackageLoader, it is illegal to call build() from
// packageLoader.getPackage() (until the package has actually been
// put in the PackageCache's cached package list.
build: function (packageLoader) {
var self = this;
if (self.pluginsBuilt || self.slicesBuilt)
@@ -1185,7 +1201,7 @@ _.extend(Package.prototype, {
}, function () {
var buildResult = bundler.buildJsImage({
name: info.name,
library: self.library,
packageLoader: packageLoader,
use: info.use,
sourceRoot: self.sourceRoot,
sources: info.sources,
@@ -1205,8 +1221,8 @@ _.extend(Package.prototype, {
// Add this plugin's dependencies to our "plugin dependency" WatchSet.
self.pluginWatchSet.merge(buildResult.watchSet);
// Remember the library resolution of all packages used by the plugin.
// XXX assumes that this merges cleanly
// Remember the versions of all of the build-time dependencies
// that were used.
_.extend(self.pluginProviderPackageDirs,
buildResult.pluginProviderPackageDirs);
@@ -1221,7 +1237,7 @@ _.extend(Package.prototype, {
// Build slices. Might use our plugins, so needs to happen
// second.
_.each(self.slices, function (slice) {
slice.build();
slice.build(packageLoader);
});
self.slicesBuilt = true;
@@ -1230,8 +1246,8 @@ _.extend(Package.prototype, {
// Programmatically initialized a package from scratch. For now, cannot create
// browser packages or cross-targeted packages (eg os.linux when host is
// os.osx). This function does not retrieve the package's dependencies from
// the library, and on return, the package will be in an unbuilt state.
// os.osx). This function does not load the package's dependencies, and on
// return, the package will be in an unbuilt state.
//
// Unlike user-facing methods of creating a package
// (initFromPackageDir, initFromAppDir) this does not implicitly add
@@ -1301,9 +1317,9 @@ _.extend(Package.prototype, {
},
// Initialize a package from a legacy-style (package.js) package
// directory. This function does not retrieve the package's
// dependencies from the library, and on return, the package will be
// in an unbuilt state.
// directory. This function does not load the package's
// dependencies, and on return, the package will be in an unbuilt
// state.
initFromPackageDir: function (name, dir, options) {
var self = this;
var isPortable = true;
@@ -1867,10 +1883,11 @@ _.extend(Package.prototype, {
},
// Initialize a package from a legacy-style application directory
// (has .meteor/packages). This function does not retrieve the
// package's dependencies from the library, and on return, the
// package will be in an unbuilt state.
initFromAppDir: function (appDir, ignoreFiles) {
// (has .meteor/packages). This function does not load the
// package's dependencies, and on return, the package will be in an
// unbuilt state.
XXX XXX make dependencies provide packageLoader
initFromAppDir: function (appDir, packageLoader, ignoreFiles) {
var self = this;
appDir = path.resolve(appDir);
self.name = null;
@@ -1880,24 +1897,8 @@ _.extend(Package.prototype, {
_.each(["client", "server"], function (sliceName) {
// Determine used packages
var names = project.getPackages(appDir);
var vers = project.getAllDependencies(appDir);
var arch = sliceName === "server" ? "os" : "browser";
// XXXX: We actually want to run the constraint solver and also edit the library to use trops
// instead of an override.
_.each(names, function(name) {
var narr = name.split("@=");
return narr[0];
});
vers = _.map(vers, function(name) {
var newPath = tropohouse.packagePath(name.packageName, name.versionConstraint);
self.library.override(name.packageName, newPath);
return name.packageName;
});
names = _.union(names, vers);
// Create slice
var slice = new Slice(self, {
name: sliceName,
@@ -1915,9 +1916,12 @@ _.extend(Package.prototype, {
// Determine source files
slice.getSourcesFunc = function () {
var sourceInclude = _.map(slice.registeredExtensions(), function (ext) {
return new RegExp('\\.' + quotemeta(ext) + '$');
});
var sourceInclude = _.map(
slice._registeredExtensions(packageLoader),
function (ext) {
return new RegExp('\\.' + quotemeta(ext) + '$');
}
);
var sourceExclude = [/^\./].concat(ignoreFiles);
// Wrapper around watch.readAndWatchDirectory which takes in and returns
@@ -2075,8 +2079,7 @@ _.extend(Package.prototype, {
// Initialize a package from a prebuilt Unipackage on disk. On
// return, the package will be a built state. This function does not
// retrieve the package's dependencies from the library (it is not
// necessary).
// load the package's dependencies (it is not necessary).
//
// options:
// - onlyIfUpToDate: if true, then first check the unipackage's

3
tools/release.js
View File

@@ -38,7 +38,8 @@ var Release = function (options) {
releaseManifest: self._manifest
});
self.catalog = new catalog.Catalog(self.library, {
// XXX XXX make Catalog a global singleton
self.catalog = new catalog.Catalog({
localPackageDirs: packageDirs
});
};

4
tools/run-app.js
View File

@@ -423,6 +423,10 @@ _.extend(AppRunner.prototype, {
bundleResult.errors.merge(settingsMessages);
}
// HACK: Also make sure we notice when somebody adds a package to
// the app packages dir that may override a catalog package.
release.current.catalog.watchLocalPackageDirs(watchSet);
// Were there errors?
if (bundleResult.errors) {
return {

23
tools/tropohouse.js
View File

@@ -54,9 +54,25 @@ tropohouse.downloadedBuilds = function (packageName, version) {
tropohouse.downloadedBuildsDirectory(packageName, version));
};
// Returns null if the package isn't in the tropohouse.
tropohouse.packagePath = function (packageName, version) {
return path.join(tropohouse.getWarehouseDir(), "packages", packageName,
version);
// Check for invalid package names. Currently package names can only
// contain ASCII alphanumerics, dash, and dot, and must contain at
// least one letter.
//
// XXX we should factor this out somewhere else, but it's nice to
// make sure that package names that we get here are sanitized to
// make sure that we don't try to read random locations on disk
//
// XXX revisit this later. What about unicode package names?
if (/[^A-Za-z0-9.\-]/.test(packageName) || !/[A-Za-z]/.test(packageName) )
return null;
var loadPath = path.join(tropohouse.getWarehouseDir(), "packages",
packageName, version);
if (! fs.existsSync(loadPath))
return null;
return loadPath;
};
tropohouse.downloadSpecifiedBuild = function (buildRecord) {
@@ -114,8 +130,7 @@ tropohouse.maybeDownloadPackageForArchitectures = function (versionInfo,
// will work once implemented?
} else {
// We need to turn our builds into a unipackage.
// XXX should this go through the library?
var pkg = new packages.Package(null /* no library?? */);
var pkg = new packages.Package;
var builds = tropohouse.downloadedBuilds(packageName, version);
_.each(builds, function (build, i) {
pkg._loadSlicesFromUnipackage(