diff --git a/tools/bundler.js b/tools/bundler.js index 05eacc1b7c..a7ada07f68 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -394,15 +394,15 @@ _.extend(File.prototype, { /////////////////////////////////////////////////////////////////////////////// // options: -// - library: package library to use for resolving package dependenices +// - packageLoader: PackageLoader to use for resolving package dependenices // - arch: the architecture to build // // see subclasses for additional options var Target = function (options) { var self = this; - // Package library to use for resolving package dependenices. - self.library = options.library; + // PackageLoader to use for resolving package dependenices. + self.packageLoader = options.packageLoader; // Something like "browser.w3c" or "os" or "os.osx.x86_64" self.arch = options.arch; @@ -498,19 +498,19 @@ _.extend(Target.prototype, { // strings) whose test slices should be included _determineLoadOrder: function (options) { var self = this; - var library = self.library; + var packageLoader = self.packageLoader; // Find the roots var rootSlices = _.flatten([ _.map(options.packages || [], function (p) { if (typeof p === "string") - return library.getSlices(p, self.arch); + return packageLoader.getSlices(p, self.arch); else return p.getDefaultSlices(self.arch); }), _.map(options.test || [], function (p) { - var pkg = (typeof p === "string" ? library.get(p) : p); + var pkg = (typeof p === "string" ? packageLoader.getPackage(p) : p); return pkg.getTestSlices(self.arch); }) ]); @@ -528,7 +528,8 @@ _.extend(Target.prototype, { if (_.has(getsUsed, slice.id)) return; getsUsed[slice.id] = slice; - slice.eachUsedSlice(self.arch, {skipWeak: true}, addToGetsUsed); + slice.eachUsedSlice(self.arch, packageLoader, + {skipWeak: true}, addToGetsUsed); }; _.each(rootSlices, addToGetsUsed); @@ -559,7 +560,8 @@ _.extend(Target.prototype, { // those edge. Because we did follow those edges in Phase 1, any unordered // slices were at some point in `needed` and will not be left out). slice.eachUsedSlice( - self.arch, {skipUnordered: true}, function (usedSlice, useOptions) { + self.arch, packageLoader, {skipUnordered: true}, + function (usedSlice, useOptions) { // If this is a weak dependency, and nothing else in the target had a // strong dependency on it, then ignore this edge. if (useOptions.weak && ! _.has(getsUsed, usedSlice.id)) @@ -602,7 +604,7 @@ _.extend(Target.prototype, { var isApp = ! slice.pkg.name; // Emit the resources - var resources = slice.getResources(self.arch); + var resources = slice.getResources(self.arch, self.packageLoader); // First, find all the assets, so that we can associate them with each js // resource (for os slices). @@ -699,8 +701,8 @@ _.extend(Target.prototype, { // Depend on the source files that produced these resources. self.watchSet.merge(slice.watchSet); - // Remember the library resolution of all packages used in these - // resources. + // Remember the versions of all of the build-time dependencies + // that were used in these resources. // XXX assumes that this merges cleanly _.extend(self.pluginProviderPackageDirs, slice.pkg.pluginProviderPackageDirs) @@ -1363,7 +1365,7 @@ var ServerTarget = function (options) { self.clientTarget = options.clientTarget; self.releaseName = options.releaseName; - self.library = options.library; + self.packageLoader = options.packageLoader; if (! archinfo.matches(self.arch, "os")) throw new Error("ServerTarget targeting something that isn't a server?"); @@ -1636,6 +1638,10 @@ var writeSiteArchive = function (targets, outputPath, options) { * untarred bundle) should go. This directory will be created if it * doesn't exist, and removed first if it does exist. * + * - packageLoader: Required. The PackageLoader used to retrieve any + * packages needed by the app or its dependencies at the appropriate + * versions. + * * - nodeModulesMode: what to do about the core npm modules needed by * the server bootstrap. one of: * - 'copy': copy from a prebuilt local installation. used by @@ -1682,12 +1688,12 @@ exports.bundle = function (options) { var appDir = options.appDir; var outputPath = options.outputPath; var nodeModulesMode = options.nodeModulesMode || 'copy'; + var packageLoader = options.packageLoader; var buildOptions = options.buildOptions || {}; if (! release.usingRightReleaseForApp(appDir)) throw new Error("running wrong release for app?"); - var library = release.current.library; var releaseName = release.current.isCheckout() ? "none" : release.current.name; var builtBy = "Meteor" + (release.current.name ? @@ -1704,7 +1710,7 @@ exports.bundle = function (options) { var makeClientTarget = function (app) { var client = new ClientTarget({ - library: library, + packageLoader: packageLoader, arch: "browser" }); @@ -1720,7 +1726,7 @@ exports.bundle = function (options) { var makeBlankClientTarget = function () { var client = new ClientTarget({ - library: library, + packageLoader: packageLoader, arch: "browser" }); client.make({ @@ -1733,7 +1739,7 @@ exports.bundle = function (options) { var makeServerTarget = function (app, clientTarget) { var targetOptions = { - library: library, + packageLoader: packageLoader, arch: buildOptions.arch || archinfo.host(), releaseName: releaseName }; @@ -1761,7 +1767,8 @@ exports.bundle = function (options) { if (includeDefaultTargets) { // Create a Package object that represents the app - var app = library.getForApp(appDir, ignoreFiles); + var app = packageLoader.getPackageCache().loadAppAtPath(appDir, + ignoreFiles); // Client var client = makeClientTarget(app); @@ -1871,11 +1878,13 @@ exports.bundle = function (options) { _.each(programs, function (p) { // Read this directory as a package and create a target from // it - library.override(p.name, p.path); + + var pkg = packageLoader.getPackageCache(). + loadPackageAtPath(p.name, p.loadPath); var target; switch (p.type) { case "server": - target = makeServerTarget(p.name); + target = makeServerTarget(pkg); break; case "traditional": var clientTarget; @@ -1900,10 +1909,10 @@ exports.bundle = function (options) { // We don't check whether targets[p.client] is actually a // ClientTarget. If you want to be clever, go ahead. - target = makeServerTarget(p.name, clientTarget); + target = makeServerTarget(pkg, clientTarget); break; case "client": - target = makeClientTarget(p.name); + target = makeClientTarget(pkg); break; default: buildmessage.error( @@ -1912,7 +1921,6 @@ exports.bundle = function (options) { // recover by ignoring target return; }; - library.removeOverride(p.name); targets[p.name] = target; }); @@ -1921,10 +1929,6 @@ exports.bundle = function (options) { if (! (controlProgram in targets)) controlProgram = undefined; - // Make sure notice when somebody adds a package to the app packages dir - // that may override a warehouse package. - library.watchLocalPackageDirs(watchSet); - // Write to disk starResult = writeSiteArchive(targets, outputPath, { nodeModulesMode: nodeModulesMode, @@ -1960,7 +1964,7 @@ exports.bundle = function (options) { // letting exceptions escape? // // options: -// - library: required. the Library for resolving package dependencies +// - packageLoader: required. the PackageLoader for resolving dependencies // - name: required. a name for this image (cosmetic, but will appear // in, eg, error messages) -- technically speaking, this is the name // of the package created to contain the sources and package @@ -1987,7 +1991,7 @@ exports.buildJsImage = function (options) { if (! options.name) throw new Error("Must provide a name"); - var pkg = new packages.Package(options.library); + var pkg = new packages.Package; pkg.initFromOptions(options.name, { sliceName: "plugin", @@ -1998,10 +2002,10 @@ exports.buildJsImage = function (options) { npmDependencies: options.npmDependencies, npmDir: options.npmDir }); - pkg.build(); + pkg.build(options.packageLoader); var target = new JsImageTarget({ - library: options.library, + packageLoader: options.packageLoader, // This function does not yet support cross-compilation (neither does // initFromOptions). That's OK for now since we're only trying to support // cross-bundling, not cross-package-building, and this function is only diff --git a/tools/catalog.js b/tools/catalog.js index 651919c378..4a284a3747 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -11,8 +11,6 @@ var catalog = exports; // we know about (including packages on the package server that we // haven't actually download yet). // -// 'library' is a library to use for loading packages. -// // Options: // - localPackageDirs: paths on local disk, that contain // subdirectories, that each contain a package that should override @@ -22,11 +20,10 @@ var catalog = exports; // server. Directories that don't exist (or paths that aren't // directories) will be silently ignored. -catalog.Catalog = function (library, options) { +catalog.Catalog = function (options) { var self = this; options = options || {}; - self.library = library; self.loaded = false; // #CatalogLazyLoading // Package server data @@ -98,8 +95,7 @@ _.extend(catalog.Catalog.prototype, { // XXX XXX for now, get the package name from the // directory. in a future refactor, should instead build the // package right here and get the name from the (not yet - // added) 'name' attribute in package.js. in this future, - // Library caches packages by path rather than by name. + // added) 'name' attribute in package.js. if (! _.has(self.effectiveLocalPackages, item)) self.effectiveLocalPackages[item] = packageDir; } @@ -112,10 +108,10 @@ _.extend(catalog.Catalog.prototype, { // the collections, shadowing any versions of those packages from // the package server. _.each(self.effectiveLocalPackages, function (packageDir, name) { + var cache = new packageCache.PackageCache; // XXX make singleton + // Load the package - var pkg = self.library.get(name, { - packageDir: packageDir - }); + var pkg = cache.loadPackageAtPath(name, packageDir); // Hide any versions from the package server self.versions.find({ packageName: name }).forEach(function (versionInfo) { @@ -192,6 +188,81 @@ _.extend(catalog.Catalog.prototype, { return _.has(self.effectiveLocalPackages, name); }, + // Register local package directories with a watchSet. We want to know if a + // package is created or deleted, which includes both its top-level source + // directory and its main package metadata file. + watchLocalPackageDirs: function (watchSet) { + var self = this; + _.each(self.localPackageDirs, function (packageDir) { + var packages = watch.readAndWatchDirectory(watchSet, { + absPath: packageDir, + include: [/\/$/] + }); + _.each(packages, function (p) { + watch.readAndWatchFile(watchSet, + path.join(packageDir, p, 'package.js')); + watch.readAndWatchFile(watchSet, + path.join(packageDir, p, 'unipackage.json')); + }); + }); + }, + + // Rebuild all source packages in our search paths. If two packages + // have the same name only the one that we would load will get + // rebuilt. + // + // Returns a count of packages rebuilt. + rebuildLocalPackages: function () { + var self = this; + + // We're going to need a PackageCache -- just for the purpose of + // forcing builds of the packages. We'll just create a new one and + // throw it away when we're done. That will mean that the builds + // we do won't be cached in memory, but since this is only ever + // called to implement a command-line command, that shouldn't be a + // problem. + // + // Note that if we were reusing an existing PackageCache, we'd + // want to call refresh() on it first. + var packageCache = new packageCache.PackageCache; + + // Delete any that are source packages with builds. + var count = 0; + _.each(self.effectiveLocalPackages, function (loadPath, name) { + var buildDir = path.join(loadPath, '.build'); + files.rm_recursive(loadPath); + }); + + // Now reload them, forcing a rebuild. We have to do this in two + // passes because otherwise we might end up rebuilding a package + // and then immediately deleting it. + _.each(self.effectiveLocalPackages, function (loadPath, name) { + packageCache.loadPackageAtPath(name, loadPath, { throwOnError: false }); + count ++; + }); + + return count; + }, + + // Given a name and a version of a package, return a path on disk + // from which we can load it. If we don't have it on disk (we + // haven't downloaded it, or it just plain doesn't exist in the + // catalog) return null. + // + // Doesn't download packages. Downloading should be done at the time + // that .meteor/versions is updated. + getLoadPathForPackage: function (name, version) { + var self = this; + + if (_.has(self.effectiveLocalPackages, name)) { + // XXX should confirm that the version on disk actually matches + // the requested version + return self.effectiveLocalPackages[name]; + } + + return tropohouse.packagePath(name, version); + }, + // Return an array with the names of all of the packages that we // know about, in no particular order. getAllPackageNames: function () { diff --git a/tools/commands.js b/tools/commands.js index 47aa64bd9c..c0485d25da 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -20,6 +20,7 @@ var httpHelpers = require('./http-helpers.js'); var archinfo = require('./archinfo.js'); var tropohouse = require('./tropohouse.js'); var packages = require('./packages.js'); +var packageLoader = require('./package-loader.js'); // Given a site name passed on the command line (eg, 'mysite'), return // a fully-qualified hostname ('mysite.meteor.com'). @@ -48,18 +49,28 @@ var hostedWithGalaxy = function (site) { }; // Get all packages available. Returns a map from the package name to -// a Package object. +// a Package object -- for the latest version of the package. // // If problems happen while generating the list, print appropriate // messages to stderr and return null. var getPackages = function () { - var result = release.current.library.list(); - if (result.packages) - return result.packages; + var ret = {}; - process.stderr.write("=> Errors while scanning packages:\n\n"); - process.stderr.write(result.messages.formatMessages()); - return null; + var catalog = release.current.catalog; + var messages = buildmessage.capture(function () { + var names = catalog.getAllPackageNames(); + _.each(names, function (name) { + ret[name] = catalog.getLatestVersion(name); + }); + }); + + if (message.hasMessages()) { + process.stderr.write("=> Errors while scanning packages:\n\n"); + process.stderr.write(result.messages.formatMessages()); + return null; + } else { + return ret; + } }; var XXX_DEPLOY_ARCH = 'os.linux.x86_64'; @@ -148,8 +159,9 @@ main.registerCommand({ if (! packages) return 1; // build failed - // XXX we rely on the fact that library.list() forces all of the - // packages to be built. #ListingPackagesImpliesBuildingThem + // XXX we rely on the fact that loading a package, even to get its + // metadata, forces it to be built if it's a source + // package. #ListingPackagesImpliesBuildingThem } }); @@ -527,26 +539,7 @@ main.registerCommand({ maxArgs: Infinity, requiresApp: true }, function (options) { - var all = getPackages(); - if (! all) - return 1; - - var using = {}; - _.each(project.getPackages(options.appDir), function (name) { - using[name] = true; - }); - - _.each(options.args, function (name) { - if (! _.has(all, name)) { - process.stderr.write(name + ": no such package\n"); - } else if (_.has(using, name)) { - process.stderr.write(name + ": already using\n"); - } else { - project.addPackage(options.appDir, name); - var note = all[name].metadata.summary || ''; - process.stderr.write(name + ": " + note + "\n"); - } - }); + throw new Error("XXX replace with add-package"); }); @@ -733,16 +726,7 @@ main.registerCommand({ return; } - var list = getPackages(); - if (! list) - return 1; - var names = _.keys(list); - names.sort(); - var pkgs = []; - _.each(names, function (name) { - pkgs.push(list[name]); - }); - process.stdout.write("\n" + library.formatList(pkgs) + "\n"); + throw new Error("XXX replace with list-all or remove completely"); }); @@ -1184,7 +1168,7 @@ main.registerCommand({ // on each other. // // Note: testRunnerAppDir deliberately DOES NOT MATCH the app - // package search path baked into release.current.library: we are + // package search path baked into release.current.catalog: we are // bundling the test runner app, but finding app packages from the // current app (if any). var testRunnerAppDir = files.mkdtemp('meteor-test-run'); @@ -1235,7 +1219,7 @@ main.registerCommand({ hidden: true }, function (options) { if (options.appDir) { - // The library doesn't know about other programs in your app. Let's blow + // The catalog doesn't know about other programs in your app. Let's blow // away their .build directories if they have them, and not rebuild // them. Sort of hacky, but eh. var programsDir = path.join(options.appDir, 'programs'); @@ -1254,7 +1238,7 @@ main.registerCommand({ var count = null; var messages = buildmessage.capture(function () { - count = release.current.library.rebuildAll(); + count = release.current.catalog.rebuildLocalPackages(); }); if (count) console.log("Built " + count + " packages."); @@ -1585,9 +1569,17 @@ main.registerCommand({ return 1; } - var pkg = new packages.Package(release.current.library, packageDir); + // #RunningTheConstraintSolverToBuildAPackage + var versions = { }; // XXX XXX actually run the constraint solver! + var loader = new packageLoader.PackageLoader({ + catalog: release.current.catalog, + versions: versions, + packageCache: new PackageCache + }); + + var pkg = new packages.Package(packageDir); pkg.initFromPackageDir(options.name, packageDir); - pkg.build(); + pkg.build(loader); pkg.saveAsUnipackage(path.join(packageDir, '.build')); var conn = packageClient.loggedInPackagesConnection(); diff --git a/tools/library.js b/tools/library.js index a1f6db431e..eb73165e76 100644 --- a/tools/library.js +++ b/tools/library.js @@ -211,7 +211,7 @@ _.extend(Library.prototype, { if (! packageDir) { if (options.throwOnError === false) return null; - //XXX buildmessage.error("package not available: " + name); + buildmessage.error("package not available: " + name); // recover by returning a dummy (empty) package var pkg = new packages.Package(self); pkg.initEmpty(name); diff --git a/tools/package-cache.js b/tools/package-cache.js new file mode 100644 index 0000000000..b12a6353f4 --- /dev/null +++ b/tools/package-cache.js @@ -0,0 +1,161 @@ +// XXX XXX make this a global singleton and eliminate the calls to +// 'new PackageCache' and 'getPackageCache()' + +var packageCache = exports; + +packageCache.PackageCache = function () { + var self = this; + + // both map from package load path to: + // - pkg: cached Package object + // - packageDir: directory from which it was loaded + self.softReloadCache = {}; + self.loadedPackages = {}; +}; + +_.extend(packageCache.PackageCache, { + // Force reload of changed packages. See description at loadPackageAtPath(). + // + // If soft is false, the default, the cache is totally flushed and + // all packages are reloaded unconditionally. + // + // If soft is true, then built packages without dependency info (such as those + // from the warehouse) aren't reloaded (there's no way to rebuild them, after + // all), and if we loaded a built package with dependency info, we won't + // reload it if the dependency info says that its source files are still up to + // date. The ideas is that assuming the user is "following the rules", this + // will correctly reload any changed packages while in most cases avoiding + // nearly all reloading. + refresh: function (soft) { + var self = this; + soft = soft || false; + + self.softReloadCache = soft ? self.loadedPackages : {}; + self.loadedPackages = {}; + }, + + // Given a path to a package on disk, retrieve a Package + // object. Options are: + // - forceRebuild: see documentation in PackageLoader.getPackage + // + // loadPackageAtPath() caches the packages it returns, meaning if + // you call loadPackageAtPath('/foo/bar') and later /foo/bar changes + // on disk, you won't see the changes. To flush the package cache + // and force all of the packages to be reloaded the next time + // loadPackageAtPath() is called for them, see refresh(). + loadPackageAtPath: function (name, loadPath, options) { + var self = this; + + options = options || {}; + + // Packages cached from previous calls + if (! options.forceRebuild && _.has(self.loadedPackages, loadPath)) { + return self.loadedPackages[loadPath].pkg; + } + + // See if we can reuse a package that we have cached from before + // the last soft refresh. + if (! options.forceRebuild && _.has(self.softReloadCache, loadPath)) { + var entry = self.softReloadCache[loadPath]; + + // Either we will decide that the cache is invalid, or we will "upgrade" + // this entry into loadedPackages. Either way, it's not needed in + // softReloadCache any more. + delete self.softReloadCache[loadPath]; + + if (entry.pkg.checkUpToDate()) { + // Cache hit + self.loadedPackages[loadedPackages] = entry; + return entry.pkg; + } + } + + // Load package from disk + var pkg = new packages.Package(loadPath); + if (fs.existsSync(path.join(loadPath, 'unipackage.json'))) { + // It's an already-built package + if (options.forceRebuild) { + throw new Error('Cannot rebuild from a unipackage directory.'); + } + pkg.initFromUnipackage(name, loadPath); + self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath}; + } else { + // It's a source tree. Does it have a built unipackage inside it? + var buildDir = path.join(loadPath, '.build'); + if (! options.forceRebuild && + fs.existsSync(buildDir) && + pkg.initFromUnipackage(name, buildDir, + { onlyIfUpToDate: true, + buildOfPath: loadPath })) { + // We already had a build and it was up to date. + self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath}; + } else { + // Either we didn't have a build, or it was out of date, or the + // caller wanted us to rebuild no matter what. Build the + // package. + buildmessage.enterJob({ + title: "building package `" + name + "`", + rootPath: loadPath + }, function () { + // This has to be done in the right sequence: initialize + // (which loads the dependency list but does not get() those + // packages), then put the package into the package list, + // then call build() to get() the dependencies and finish + // the build. If you called build() before putting the + // package in the package list then you'd recurse + // forever. (build() needs the dependencies because it needs + // to look at the handlers registered by any plugins in the + // packages that we use.) + pkg.initFromPackageDir(name, loadPath); + self.loadedPackages[loadPath] = {pkg: pkg, packageDir: loadPath}; + + // #RunningTheConstraintSolverToBuildAPackage + var versions = { }; // XXX XXX actually run the constraint solver! + var loader = new packageLoader.PackageLoader({ + catalog: release.current.catalog, + versions: versions, + packageCache: self + }); + pkg.build(loader); + + if (! buildmessage.jobHasMessages()) { + // Save it, for a fast load next time + try { + files.addToGitignore(loadPath, '.build*'); + pkg.saveAsUnipackage(buildDir, { buildOfPath: loadPath }); + } catch (e) { + // If we can't write to this directory, we don't get to cache our + // output, but otherwise life is good. + if (!(e && (e.code === 'EACCES' || e.code === 'EPERM'))) + throw e; + } + } + }); + } + } + + return pkg; + }, + + // Get a package that represents an app. (ignoreFiles is optional + // and if given, it should be an array of regexps for filenames to + // ignore when scanning for source files.) + // XXX formerly called getForApp + loadAppAtPath: function (appDir, ignoreFiles) { + var self = this; + + // #RunningTheConstraintSolverToBuildAPackage + var versions = { }; // XXX XXX actually run the constraint solver! + var loader = new packageLoader.PackageLoader({ + catalog: release.current.catalog, + versions: versions, + packageCache: self + }); + pkg.build(loader); + + var pkg = new packages.Package; + pkg.initFromAppDir(appDir, ignoreFiles || []); + pkg.build(loader); + return pkg; + } +}); diff --git a/tools/package-loader.js b/tools/package-loader.js new file mode 100644 index 0000000000..2da87f2f2d --- /dev/null +++ b/tools/package-loader.js @@ -0,0 +1,84 @@ +var packageLoader = exports; + +// options: +// catalog: the Catalog used to locate the packages on disk +// versions: a map from package name to the version to use +// packageCache: the PackageCache used to load packages and cache them +// in memory +packageLoader.PackageLoader = function (options) { + var self = this; + self.catalog = options.catalog; + self.versions = options.versions; + self.packageCache = options.packageCache; +}; + +_.extend(packageLoader.PackageLoader, { + // Given the name of a package, return a Package object, or throw an + // error if the package wasn't included in the 'versions' passed on + // initalization or isn't available (for example, hasn't been + // downloaded yet). + // + // Options are: + // - throwOnError: if true (the default), throw an error if the + // package can't be found. (If false is passed for throwOnError, + // then return null if the package can't be found.) When called + // inside buildmessage.enterJob, however, instead of throwing an + // error it will record a build error and return a dummy (empty) + // package. + // - forceRebuild: defaults to false. If true, we will initialize the + // package from the source and ignore a built unipackage if it + // exists. This option is ignored if you pass `name` as a Package. + // + // XXX rename to throwOnNotFound + getPackage: function (name, options) { + var self = this; + + options = options || {}; + if (options.throwOnError === undefined) { + options.throwOnError = true; + } + + if (! _.has(self.versions, name)) + throw new Error("no version chosen for package?"); + + var loadPath = self.catalog.getLoadPathForPackage(name, + self.versions[name]); + if (! loadPath) { + if (options.throwOnError === false) + return null; + buildmessage.error("package not available: " + name); + // recover by returning a dummy (empty) package + var pkg = new packages.Package; + pkg.initEmpty(name); + return pkg; + } + + return self.packageCache.loadPackageAtPath(name, loadPath, { + forceRebuild: options.forceRebuild + }); + }, + + // Given a slice set spec -- either a package name like "ddp", or a particular + // slice within the package like "ddp/client", or a parsed object like + // {package: "ddp", slice: "client"} -- return the list of matching slices (as + // an array of Slice objects) for a given architecture. + getSlices: function (spec, arch) { + var self = this; + + if (typeof spec === "string") + spec = packages.parseSpec(spec); + + var pkg = self.getPackage(spec.package, { throwOnError: true }); + if (spec.slice) + return [pkg.getSingleSlice(spec.slice, arch)]; + else + return pkg.getDefaultSlices(arch); + }, + + // Return the PackageCache that backs this loader. + getPackageCache: function () { + var self = this; + return self.packageCache; + } + +}); \ No newline at end of file diff --git a/tools/packages.js b/tools/packages.js index 3e17837ed1..1edbfe3e47 100644 --- a/tools/packages.js +++ b/tools/packages.js @@ -250,7 +250,11 @@ _.extend(Slice.prototype, { // resulting JavaScript. Also add all provided source files to the // package dependencies. Sets fields such as dependencies, exports, // prelinkFiles, packageVariables, and resources. - build: function () { + // + // packageLoader is the PackageLoader to use to validate that the + // slice's dependencies actually exist (for cleaner error + // messages). + build: function (packageLoader) { var self = this; var isApp = ! self.pkg.name; @@ -269,11 +273,12 @@ _.extend(Slice.prototype, { _.each(['uses', 'implies'], function (field) { var scrubbed = []; _.each(self[field], function (u) { - var pkg = self.pkg.library.get(u.package, { throwOnError: false }); -/* if (! pkg) { + var pkg = packageLoader.getPackage(u.package, + { throwOnError: false }); + if (! pkg) { buildmessage.error("no such package: '" + u.package + "'"); // recover by omitting this package from the field - } else */ + } else scrubbed.push(u); }); self[field] = scrubbed; @@ -300,7 +305,8 @@ _.extend(Slice.prototype, { var fileOptions = _.clone(source.fileOptions) || {}; var absPath = path.resolve(self.pkg.sourceRoot, relPath); var filename = path.basename(relPath); - var handler = !fileOptions.isAsset && self._getSourceHandler(filename); + var handler = !fileOptions.isAsset && + self._getSourceHandler(filename, packageLoader); var file = watch.readAndWatchFileWithHash(self.watchSet, absPath); var contents = file.contents; @@ -551,7 +557,7 @@ _.extend(Slice.prototype, { // plugin program itself uses), as well as the package.js file from every // package we directly use (since changing the package.js may add or remove // a plugin). - _.each(self._activePluginPackages(), function (otherPkg) { + _.each(self._activePluginPackages(packageLoader), function (otherPkg) { self.watchSet.merge(otherPkg.pluginWatchSet); // XXX this assumes this is not overwriting something different self.pkg.pluginProviderPackageDirs[otherPkg.name] = @@ -600,9 +606,11 @@ _.extend(Slice.prototype, { // is resolved at bundle time. (On the other hand, when it comes to // the extension handlers we'll use, we previously commited to those // versions at package build ('compile') time.) - getResources: function (bundleArch) { + // + // packageLoader is the PackageLoader that should be used to resolve + // the package's bundle-time dependencies. + getResources: function (bundleArch, packageLoader) { var self = this; - var library = self.pkg.library; if (! self.isBuilt) throw new Error("getting resources of unbuilt slice?" + self.pkg.name + " " + self.sliceName + " " + self.arch); @@ -621,7 +629,8 @@ _.extend(Slice.prototype, { // unrelated package in the target depends on something). var imports = {}; // map from symbol to supplying package name self.eachUsedSlice( - bundleArch, {skipWeak: true, skipUnordered: true}, function (otherSlice) { + bundleArch, packageLoader, + {skipWeak: true, skipUnordered: true}, function (otherSlice) { if (! otherSlice.isBuilt) throw new Error("dependency wasn't built?"); _.each(otherSlice.packageVariables, function (symbol) { @@ -664,7 +673,10 @@ _.extend(Slice.prototype, { // are transitively "implied" by used slices. (But not slices that are used by // slices that we use!) Options are skipWeak and skipUnordered, meaning to // ignore direct "uses" that are weak or unordered. - eachUsedSlice: function (arch, options, callback) { + // + // packageLoader is the PackageLoader that should be used to resolve + // the package's bundle-time dependencies. + eachUsedSlice: function (arch, packageLoader, options, callback) { var self = this; if (typeof options === "function") { callback = options; @@ -685,7 +697,8 @@ _.extend(Slice.prototype, { var use = usesToProcess.shift(); var slices = - self.pkg.library.getSlices(_.pick(use, 'package', 'spec'), arch); + packageLoader.getSlices(_.pick(use, 'package', 'spec'), + arch); _.each(slices, function (slice) { if (_.has(processedSliceId, slice.id)) return; @@ -704,7 +717,7 @@ _.extend(Slice.prototype, { // Return an array of all plugins that are active in this slice, as // a list of Packages. - _activePluginPackages: function () { + _activePluginPackages: function (packageLoader) { var self = this; // XXX we used to include our own extensions only if we were the @@ -725,9 +738,12 @@ _.extend(Slice.prototype, { // We pass archinfo.host here, not self.arch, because it may be more // specific, and because plugins always have to run on the host // architecture. - self.eachUsedSlice(archinfo.host(), {skipWeak: true}, function (usedSlice) { - ret.push(usedSlice.pkg); - }); + self.eachUsedSlice( + archinfo.host(), packageLoader, {skipWeak: true}, + function (usedSlice) { + ret.push(usedSlice.pkg); + } + ); // Only need one copy of each package. ret = _.uniq(ret); @@ -742,7 +758,7 @@ _.extend(Slice.prototype, { // Get all extensions handlers registered in this slice, as a map // from extension (no leading dot) to handler function. Throws an // exception if two packages are registered for the same extension. - _allHandlers: function () { + _allHandlers: function (packageLoader) { var self = this; var ret = {}; @@ -761,7 +777,7 @@ _.extend(Slice.prototype, { } }); - _.each(self._activePluginPackages(), function (otherPkg) { + _.each(self._activePluginPackages(packageLoader), function (otherPkg) { _.each(otherPkg.sourceHandlers, function (handler, ext) { if (ext in ret && ret[ext] !== handler) { buildmessage.error( @@ -783,17 +799,17 @@ _.extend(Slice.prototype, { // Return a list of all of the extension that indicate source files // for this slice, not including leading dots. Computed based on // this.uses, so should only be called once that has been set. - registeredExtensions: function () { + _registeredExtensions: function (packageLoader) { var self = this; - return _.keys(self._allHandlers()); + return _.keys(self._allHandlers(packageLoader)); }, // Find the function that should be used to handle a source file for // this slice, or return null if there isn't one. We'll use handlers // that are defined in this package and in its immediate dependencies. - _getSourceHandler: function (filename) { + _getSourceHandler: function (filename, packageLoader) { var self = this; - var handlers = self._allHandlers(); + var handlers = self._allHandlers(packageLoader); var parts = filename.split('.'); for (var i = 0; i < parts.length; i++) { var extension = parts.slice(i).join('.'); @@ -820,7 +836,7 @@ _.extend(Slice.prototype, { // (find better names, though). var nextPackageId = 1; -var Package = function (library, packageDirectoryForBuildInfo) { +var Package = function (packageDirectoryForBuildInfo) { var self = this; // A unique ID (guaranteed to not be reused in this process -- if @@ -850,16 +866,12 @@ var Package = function (library, packageDirectoryForBuildInfo) { // The package's directory. This is used only by other packages that use this // package in their buildinfo.json (to detect that they need to be rebuilt if - // the library's resolution of the package name changes); it is not used to + // the PackageLoader resolves it to a different package); it is not used to // read files or anything else. Notably, it should be the same if a package is // read from a source tree or read from the .build unipackage inside that // source tree. self.packageDirectoryForBuildInfo = packageDirectoryForBuildInfo; - // Package library that should be used to resolve this package's - // dependencies - self.library = library; - // Package metadata. Keys are 'summary' and 'internal'. Currently // both of these are optional. self.metadata = {}; @@ -1165,10 +1177,14 @@ _.extend(Package.prototype, { // completed, any errors detected in the package will have been // emitted to buildmessage. // - // build() may retrieve the package's dependencies from the library, - // so it is illegal to call build() from library.get() (until the - // package has actually been put in the loaded package list). - build: function () { + // packageLoader is the PackageLoader to use for determining the + // package's build-time dependencies. + // + // Since build() retrieves the package's dependencies from the + // PackageLoader, it is illegal to call build() from + // packageLoader.getPackage() (until the package has actually been + // put in the PackageCache's cached package list. + build: function (packageLoader) { var self = this; if (self.pluginsBuilt || self.slicesBuilt) @@ -1185,7 +1201,7 @@ _.extend(Package.prototype, { }, function () { var buildResult = bundler.buildJsImage({ name: info.name, - library: self.library, + packageLoader: packageLoader, use: info.use, sourceRoot: self.sourceRoot, sources: info.sources, @@ -1205,8 +1221,8 @@ _.extend(Package.prototype, { // Add this plugin's dependencies to our "plugin dependency" WatchSet. self.pluginWatchSet.merge(buildResult.watchSet); - // Remember the library resolution of all packages used by the plugin. - // XXX assumes that this merges cleanly + // Remember the versions of all of the build-time dependencies + // that were used. _.extend(self.pluginProviderPackageDirs, buildResult.pluginProviderPackageDirs); @@ -1221,7 +1237,7 @@ _.extend(Package.prototype, { // Build slices. Might use our plugins, so needs to happen // second. _.each(self.slices, function (slice) { - slice.build(); + slice.build(packageLoader); }); self.slicesBuilt = true; @@ -1230,8 +1246,8 @@ _.extend(Package.prototype, { // Programmatically initialized a package from scratch. For now, cannot create // browser packages or cross-targeted packages (eg os.linux when host is - // os.osx). This function does not retrieve the package's dependencies from - // the library, and on return, the package will be in an unbuilt state. + // os.osx). This function does not load the package's dependencies, and on + // return, the package will be in an unbuilt state. // // Unlike user-facing methods of creating a package // (initFromPackageDir, initFromAppDir) this does not implicitly add @@ -1301,9 +1317,9 @@ _.extend(Package.prototype, { }, // Initialize a package from a legacy-style (package.js) package - // directory. This function does not retrieve the package's - // dependencies from the library, and on return, the package will be - // in an unbuilt state. + // directory. This function does not load the package's + // dependencies, and on return, the package will be in an unbuilt + // state. initFromPackageDir: function (name, dir, options) { var self = this; var isPortable = true; @@ -1867,10 +1883,11 @@ _.extend(Package.prototype, { }, // Initialize a package from a legacy-style application directory - // (has .meteor/packages). This function does not retrieve the - // package's dependencies from the library, and on return, the - // package will be in an unbuilt state. - initFromAppDir: function (appDir, ignoreFiles) { + // (has .meteor/packages). This function does not load the + // package's dependencies, and on return, the package will be in an + // unbuilt state. + XXX XXX make dependencies provide packageLoader + initFromAppDir: function (appDir, packageLoader, ignoreFiles) { var self = this; appDir = path.resolve(appDir); self.name = null; @@ -1880,24 +1897,8 @@ _.extend(Package.prototype, { _.each(["client", "server"], function (sliceName) { // Determine used packages var names = project.getPackages(appDir); - var vers = project.getAllDependencies(appDir); var arch = sliceName === "server" ? "os" : "browser"; - - // XXXX: We actually want to run the constraint solver and also edit the library to use trops - // instead of an override. - _.each(names, function(name) { - var narr = name.split("@="); - return narr[0]; - }); - - vers = _.map(vers, function(name) { - var newPath = tropohouse.packagePath(name.packageName, name.versionConstraint); - self.library.override(name.packageName, newPath); - return name.packageName; - }); - - names = _.union(names, vers); // Create slice var slice = new Slice(self, { name: sliceName, @@ -1915,9 +1916,12 @@ _.extend(Package.prototype, { // Determine source files slice.getSourcesFunc = function () { - var sourceInclude = _.map(slice.registeredExtensions(), function (ext) { - return new RegExp('\\.' + quotemeta(ext) + '$'); - }); + var sourceInclude = _.map( + slice._registeredExtensions(packageLoader), + function (ext) { + return new RegExp('\\.' + quotemeta(ext) + '$'); + } + ); var sourceExclude = [/^\./].concat(ignoreFiles); // Wrapper around watch.readAndWatchDirectory which takes in and returns @@ -2075,8 +2079,7 @@ _.extend(Package.prototype, { // Initialize a package from a prebuilt Unipackage on disk. On // return, the package will be a built state. This function does not - // retrieve the package's dependencies from the library (it is not - // necessary). + // load the package's dependencies (it is not necessary). // // options: // - onlyIfUpToDate: if true, then first check the unipackage's diff --git a/tools/release.js b/tools/release.js index ba02be61b9..bd5396d7e5 100644 --- a/tools/release.js +++ b/tools/release.js @@ -38,7 +38,8 @@ var Release = function (options) { releaseManifest: self._manifest }); - self.catalog = new catalog.Catalog(self.library, { + // XXX XXX make Catalog a global singleton + self.catalog = new catalog.Catalog({ localPackageDirs: packageDirs }); }; diff --git a/tools/run-app.js b/tools/run-app.js index 390636783c..c22d2f4d92 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -423,6 +423,10 @@ _.extend(AppRunner.prototype, { bundleResult.errors.merge(settingsMessages); } + // HACK: Also make sure we notice when somebody adds a package to + // the app packages dir that may override a catalog package. + release.current.catalog.watchLocalPackageDirs(watchSet); + // Were there errors? if (bundleResult.errors) { return { diff --git a/tools/tropohouse.js b/tools/tropohouse.js index 246fcff7f0..585a2f97ec 100644 --- a/tools/tropohouse.js +++ b/tools/tropohouse.js @@ -54,9 +54,25 @@ tropohouse.downloadedBuilds = function (packageName, version) { tropohouse.downloadedBuildsDirectory(packageName, version)); }; +// Returns null if the package isn't in the tropohouse. tropohouse.packagePath = function (packageName, version) { - return path.join(tropohouse.getWarehouseDir(), "packages", packageName, - version); + // Check for invalid package names. Currently package names can only + // contain ASCII alphanumerics, dash, and dot, and must contain at + // least one letter. + // + // XXX we should factor this out somewhere else, but it's nice to + // make sure that package names that we get here are sanitized to + // make sure that we don't try to read random locations on disk + // + // XXX revisit this later. What about unicode package names? + if (/[^A-Za-z0-9.\-]/.test(packageName) || !/[A-Za-z]/.test(packageName) ) + return null; + + var loadPath = path.join(tropohouse.getWarehouseDir(), "packages", + packageName, version); + if (! fs.existsSync(loadPath)) + return null; + return loadPath; }; tropohouse.downloadSpecifiedBuild = function (buildRecord) { @@ -114,8 +130,7 @@ tropohouse.maybeDownloadPackageForArchitectures = function (versionInfo, // will work once implemented? } else { // We need to turn our builds into a unipackage. - // XXX should this go through the library? - var pkg = new packages.Package(null /* no library?? */); + var pkg = new packages.Package; var builds = tropohouse.downloadedBuilds(packageName, version); _.each(builds, function (build, i) { pkg._loadSlicesFromUnipackage(