diff --git a/tools/catalog.js b/tools/catalog.js index e0e3ca8d78..afc7e32b70 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -51,6 +51,12 @@ var Catalog = function () { // All packages found either by localPackageDirs or localPackages self.effectiveLocalPackages = {}; // package name to source directory + + // Set this to true if we are not going to connect to the remote package + // server, and will only use the cached data.json file for our package + // information. This means that the catalog might be out of date on the latest + // developments. + self.offline = null; }; _.extend(Catalog.prototype, { @@ -93,15 +99,28 @@ _.extend(Catalog.prototype, { // OK, now initialize the catalog for real, with both local and // package server packages. console.log("XXX Loading catalog for real"); - self._refresh(true); + + // We should to figure out if we are intending to connect to the package + // server. + self.offline = options.offline ? options.offline : false; + self._refresh(true /* load server packages */); }, - // Set sync to true to try to synchronize from the package server. + // If sync is false, this will not synchronize with the remote server, even if + // the catalog is not in offline mode. This is an optimization for loading + // local packages. (An offline catalog will not sync with the server even if + // sync is true.) _refresh: function (sync) { var self = this; self._requireInitialized(); - var serverPackageData = packageClient.loadPackageData(sync); + var localData = packageClient.loadCachedServerData(); + var serverPackageData; + if (! self.offline && sync) { + serverPackageData = packageClient.updateServerPackageData(localData); + } else { + serverPackageData = localData.collections; + } self.initialized = false; self.packages = []; diff --git a/tools/commands.js b/tools/commands.js index 43e83aa228..08c66233d1 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -579,129 +579,160 @@ main.registerCommand({ var failed = false; // Read in existing package dependencies. - var usingDirectly = project.getDepsAsObj(project.getDirectDependencies(options.appDir)); + var packages = project.getDepsAsObj(project.getDirectDependencies(options.appDir)); - // For every package name specified, run it through the constraint - // solver and add the right stuff to .meteor/package and - // .meteor/versions files. + // For every package name specified, add it to our list of package + // constraints. Don't run the constraint solver until you have added all of + // them -- add should be an atomic operation regardless of the package + // order. Even though the package file should specify versions of its inputs, + // we don't specify these constraints until we get them back from the + // constraint solver. _.each(options.args, function (packageReq) { - if (failed) - return; - + // XXX: Use a util function. var constraint = project.processPackageConstraint(packageReq); + // Check that the package exists. if (! catalog.getPackage(constraint.packageName)) { process.stderr.write(constraint.packageName + ": no such package\n"); failed = true; return; } + // Check that the version exists. var versionInfo = catalog.getVersion( constraint.packageName, + // XXX: Use a util function. getVersionFromVersionConstraint(constraint.versionConstraint)); - if (! versionInfo) { process.stderr.write( -constraint.packageName + "@" + constraint.versionConstraint + ": no such version\n"); + constraint.packageName + "@" + constraint.versionConstraint + ": no such version\n"); failed = true; return; } - if (_.has(usingDirectly, constraint.packageName)) { - if (usingDirectly[constraint.packageName] === constraint.versionConstraint) { - process.stderr.write(constraint.packageName + "@" + constraint.versionConstraint + ": already added\n"); - failed = true; + // Check that the constraint is new. If we are already using the package at + // the same constraint, return from this function. + if (_.has(packages, constraint.packageName)) { + if (packages[constraint.packageName] === constraint.versionConstraint) { + process.stderr.write( + constraint.packageName + "@" + constraint.versionConstraint + ": already added\n"); return; - } else if (!constraint.versionConstraint && (usingDirectly[constraint.packageName] === "none")) { + } else if (!constraint.versionConstraint && (packages[constraint.packageName] === "none")) { + // XXX: In the brand new world where we have versioning in .meteor/packages, this will not happen. process.stderr.write(constraint.packageName + ": already added\n"); - failed = true; return; } + } + + // Add the package to our direct dependency constraints that we get from .meteor/packages. + packages[constraint.packageName] = constraint.versionConstraint; + }); + + // If the user asked for invalid packages, then the user probably expects a + // different result than what they are going to get. We have already logged an + // error, so we should exit. + if ( failed ) { + return 1; + } + + // Get the contents of our versions file. We need to pass them to the + // constraint solver, because our contract with the user says that we will + // never downgrade a dependency. + var versions = project.getDepsAsObj(project.getIndirectDependencies(options.appDir)); + + // Call the constraint solver. + var constraintSolver = require('./constraint-solver.js'); + var resolver = new constraintSolver.Resolver; + // XXX: constraint solver currently ignores versions, but it should not. + // XXX: this would also be the place to add no-update options. + var newVersions = resolver.resolve(packages); + if ( ! newVersions) { + // XXX: Better error handling. + process.stderr.write("Cannot resolve package dependencies."); + } + + // Don't tell the user what all the operations were until we finish -- we + // don't want to give a false sense of completeness until everything is + // written to disk. + var messageLog = []; + + // Remove the versions that don't exist + var removed = _.difference(_.keys(versions), _.keys(newVersions)); + _.each(removed, function(packageName) { + messageLog.push("removed dependency on " + packageName); + }); + + // Install the new versions. + _.each(newVersions, function(version, packageName) { + if ( failed ) + return; + if (_.has(versions, packageName) && + versions[packageName] == version ) { + // Nothing changed. Skip this. + return; + } + + // Make sure we have enough builds of the package downloaded such that + // we can load a browser slice and a slice that will run on this + // system. (Later we may also need to download more builds to be able to + // deploy to another architecture.) + var available = tropohouse.maybeDownloadPackageForArchitectures( + catalog.getVersion(packageName, version), + // XXX we also download the deploy arch now, because we don't run the + // constraint solver / downloader anywhere other than add-package yet. + ['browser', archinfo.host(), XXX_DEPLOY_ARCH]); + if (! available) { + // XXX maybe we shouldn't be letting the constraint solver choose + // things that don't have the right arches? + process.stderr.write("Package " + packageName + + " has no compatible build for version " + + version); + failed = true; + return; + } + + // Add a message to the update logs to show the user what we have done. + if ( _.contains(options.args, packageName)) { + // If we asked for this, we will log it later in more detail. + return; + } + + // If the previous versions file had this, then we are upgrading, if it did + // not, then we must be adding this package anew. + if ( _.has(versions, packageName )) { + messageLog.push("upgraded " + packageName + " from version " + + versions[packageName] + + " to version " + newVersions[packageName]); } else { - // Add the package to the list of packages that we use directly. - usingDirectly[constraint.packageName] = constraint.versionConstraint; - var usingIndirectly = project.getDepsAsObj(project.getIndirectDependencies(options.appDir)); + messageLog.push("added " + packageName + " from " + + " at version " + newVersions[packageName]); + }; + }); - // Call the constraint solver. - var ConstraintSolver = uniload.load({ - packages: ['constraint-solver'], - release: release.current.name - })['constraint-solver'].ConstraintSolver; + if (failed) + return 1; - var resolver = new ConstraintSolver.Resolver(catalog); - var newVersions = resolver.resolve(usingDirectly, - usingIndirectly, - { optionsGoHere : false }); + // Write the .meteor/packages file with the right versions + var oldPackages = project.getDepsAsObj(project.getDirectDependencies(options.appDir)); + project.rewriteDirectDependencies(options.appDir, packages); - var logMessage = ""; - _.forEach(newVersions, function(version, packageName) { - if (failed) - return; + // Write the .meteor/versions file with the new dependencies. + project.rewriteIndirectDependencies(options.appDir, newVersions); - // Check if it exists. - if (usingIndirectly[packageName] === version) { - // We are using this at this version, so do nothing. - } else { + // Show the user the messageLog of packages we added. + _.each(messageLog, function (msg) { + process.stdout.write(msg + "\n"); + }); - // Find the build. - // XXX: Find the one with the right architecture. - var versionInfo = catalog.getVersion(packageName, version); - - // Safety check, but this should not happen unless the - // constraint solver is doing something it shouldn't. - if (! versionInfo) { - process.stderr.write("This package has no version at this version"); - failed = true; - return; - } - - // Make sure we have enough builds of the package downloaded such that - // we can load a browser slice and a slice that will run on this - // system. (Later we may also need to download more builds to be able to - // deploy to another architecture.) - var available = tropohouse.maybeDownloadPackageForArchitectures( - // XXX we also download the deploy arch now, because we don't run the - // constraint solver / downloader anywhere other than add-package yet. - versionInfo, ['browser', archinfo.host(), XXX_DEPLOY_ARCH]); - if (! available) { - // XXX maybe we shouldn't be letting the constraint solver choose - // things that don't have the right arches? - process.stderr.write("Package " + packageName + - " has no compatible build for version " + - version); - failed = true; - return; - } - - if (_.has(usingIndirectly[packageName])) { - logMessage = logMessage + "Upgraded " + packageName + " from version " + - usingIndirectly + " to version " + version + "\n"; - } else { - logMessage = logMessage + "Added " + packageName + " at version " + - version + "\n"; - logMessage = logMessage + "(" + packageName + " : " + versionInfo.decription + ") \n"; - } - } - }); - - if (failed) - return; - - // Add to the new direct dependencies file. - // XXX: Write the current version into packages file, rather than the requested version - project.addDirectDependency(options.appDir, packageReq); - - // Write the new indirect dependencies file. - project.rewriteIndirectDependencies(options.appDir, newVersions); - - // Log that this happened! Yay! - process.stdout.write(logMessage); - process.stdout.write("Finished adding: \n"); - var note = versionInfo.description; - process.stdout.write(constraint.packageName + ": " + note + "\n"); + // Show the user the messageLog of the packages that they installed. + process.stdout.write("Successfully added the following packages. \n"); + _.each(packages, function (version, name) { + if ( ! _.has(oldPackages, name) ) { + var versionRecord = catalog.getVersion(name, version); + process.stdout.write(name + " : " + versionRecord.description + "\n"); } }); - return failed ? 1 : 0; + return 0; }); @@ -709,25 +740,73 @@ constraint.packageName + "@" + constraint.versionConstraint + ": no such versio // remove /////////////////////////////////////////////////////////////////////////////// + main.registerCommand({ name: 'remove', minArgs: 1, maxArgs: Infinity, requiresApp: true }, function (options) { - var using = {}; - _.each(project.getPackages(options.appDir), function (name) { - using[name] = true; + + // Read in existing package dependencies. + var packages = project.getDepsAsObj(project.getDirectDependencies(options.appDir)); + + // For every package name specified, add it to our list of package + // constraints. Don't run the constraint solver until you have added all of + // them -- add should be an atomic operation regardless of the package + // order. Even though the package file should specify versions of its inputs, + // we don't specify these constraints until we get them back from the + // constraint solver. + _.each(options.args, function (packageName) { + // Check that we are using the package. We don't check if the package + // exists. You should be able to remove non-existent packages. + if (! _.has(packages, packageName)) { + process.stderr.write( packageName + " is not in this project \n"); + } + + // Remove the package from our dependency list. + delete packages[packageName]; }); - _.each(options.args, function (name) { - if (! _.has(using, name)) { - process.stderr.write(name + ": not in project\n"); - } else { - project.removePackage(options.appDir, name); - process.stderr.write(name + ": removed\n"); - } + // Get the contents of our versions file. We need to pass them to the + // constraint solver, because our contract with the user says that we will + // never downgrade a dependency. + var versions = project.getDepsAsObj(project.getIndirectDependencies(options.appDir)); + + // Call the constraint solver. + var constraintSolver = require('./constraint-solver.js'); + var resolver = new constraintSolver.Resolver; + // XXX: constraint solver currently ignores versions, but it should not. + // XXX: this would also be the place to add no-update options. + var newVersions = resolver.resolve(packages); + if ( ! newVersions) { + // This should never really happen. + process.stderr.write("Cannot resolve package dependencies."); + } + + // Don't tell the user what all the operations were until we finish -- we + // don't want to give a false sense of completeness until everything is + // written to disk. + var messageLog = []; + + // Remove the versions that don't exist + var removed = _.difference(_.keys(versions), _.keys(newVersions)); + _.each(removed, function(packageName) { + messageLog.push("removed dependency on " + packageName); }); + + // Write the .meteor/packages file with the right versions + project.rewriteDirectDependencies(options.appDir, packages); + + // Write the .meteor/versions file with the new dependencies. + project.rewriteIndirectDependencies(options.appDir, newVersions); + + // Show the user the messageLog of everything we removed. + _.each(messageLog, function (msg) { + process.stdout.write(msg + "\n"); + }); + + return 0; }); /////////////////////////////////////////////////////////////////////////////// @@ -743,25 +822,15 @@ main.registerCommand({ using: { type: Boolean } } }, function (options) { - if (options.using) { - var using = project.getPackages(options.appDir); - - if (using.length) { - _.each(using, function (name) { - process.stdout.write(name + "\n"); - }); - } else { - process.stderr.write( -"This project doesn't use any packages yet. To add some packages:\n" + -" meteor add ...\n" + -"\n" + -"To see available packages:\n" + -" meteor list\n"); + var items = []; + _.each(catalog.getAllPackageNames(), function (name) { + var versionInfo = catalog.getLatestVersion(name); + if (versionInfo) { + items.push({ name: name, description: versionInfo.description }); } - return; - } + }); - throw new Error("XXX replace with list-all or remove completely"); + process.stdout.write(formatList(items)); }); @@ -1561,25 +1630,6 @@ main.registerCommand({ return 0; }); -// This command will list all packages in existence. -// This command may go away after testing is done. -main.registerCommand({ - name: 'list-all', - options: {}, - maxArgs: 0, - hidden: true -}, function (options) { - var items = []; - _.each(catalog.getAllPackageNames(), function (name) { - var versionInfo = catalog.getLatestVersion(name); - if (versionInfo) { - items.push({ name: name, description: versionInfo.description }); - } - }); - - process.stdout.write(formatList(items)); -}); - main.registerCommand({ name: 'publish-for-arch', minArgs: 0, diff --git a/tools/main.js b/tools/main.js index 36fac04406..797465aaa8 100644 --- a/tools/main.js +++ b/tools/main.js @@ -687,9 +687,16 @@ Fiber(function () { // Initialize the singleton Catalog. Only after this point is the // Catalog (and therefore unipackage.load) usable. // - // This will try to talk to the network to synchronize our package - // list with the package server. - catalog.initialize({ localPackageDirs: localPackageDirs }); + // If the --no-net option is set, the catalog will be offline and will never + // attempt to contact the server for more recent data. Otherwise, the catalog + // will attempt to synchronize with the remote package server. + catalog.initialize({ + localPackageDirs: localPackageDirs, + offline: _.has(rawOptions, '--no-net') + }); + // We need to delete the option or we will throw an error. + // XXX: This seems like a hack? + delete rawOptions['--no-net']; // Check for the '--help' option. var showHelp = false; diff --git a/tools/package-client.js b/tools/package-client.js index 4286abe71c..9f1446b5bd 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -49,7 +49,7 @@ var openPackageServerConnection = function () { // If there is no data.json file, or the file cannot be parsed, return null for // the collections and a default syncToken to ask the server for all the data // from the beginning of time. -var loadCachedServerData = function () { +exports.loadCachedServerData = function () { var noDataToken = { // XXX have a better sync token for "all" syncToken: {time: 'Sun, 01 Jan 2012 00:00:00 GMT'}, @@ -74,7 +74,6 @@ var loadCachedServerData = function () { return ret; }; - // Opens a connection to the server, requests and returns new package data that // we haven't cached on disk. We assume that data is cached chronologically, so // essentially, we are asking for a diff from the last time that we did this. @@ -96,7 +95,6 @@ var loadRemotePackageData = function (syncToken) { return collectionData; }; - // Take in an ordered list of javascript objects representing collections of // package data. In each object, the server-side names of collections are keys // and the values are the mongo records for that collection stored as an @@ -126,7 +124,6 @@ var mergeCollections = function (sources) { return ret; }; - // Writes the cached package data to the on-disk cache. Takes in the following // arguments: // - syncToken : the token representing our conversation with the server, that @@ -151,22 +148,24 @@ var writePackageDataToDisk = function (syncToken, collectionData) { files.writeFileAtomically(filename, JSON.stringify(finalWrite, null, 2)); }; -// Returns the package data. +// Contacts the package server to get the latest diff and writes changes to +// disk. // -exports.loadPackageData = function() { - //XXX: We can consider optimizing this with concurrency or something. +// Takes in cachedServerData, which is the processed contents of data.json. Uses +// those to talk to the server and get the latest updates. Applies the diff from +// the server to the in-memory version of the on-disk data, then writes the new +// file to disk as the new data.json. +exports.updateServerPackageData = function (cachedServerData) { var sources = []; - - var localData = loadCachedServerData(); - if (localData.collections) - sources.push(localData.collections); - var syncToken = localData.syncToken; - // XXX support offline use too -// var remoteData = loadRemotePackageData(syncToken); -// sources.push(remoteData.collections); + if (cachedServerData.collections) { + sources.push(cachedServerData.collections); + } + var syncToken = cachedServerData.syncToken; + var remoteData = loadRemotePackageData(syncToken); + sources.push(remoteData.collections); var allPackageData = mergeCollections(sources); -// writePackagesToDisk(remoteData.syncToken, allPackageData); + writePackageDataToDisk(remoteData.syncToken, allPackageData); return allPackageData; }; diff --git a/tools/project.js b/tools/project.js index f2d1277691..0e6e8332e6 100644 --- a/tools/project.js +++ b/tools/project.js @@ -131,21 +131,26 @@ project.rewriteIndirectDependencies = function (appDir, deps) { lines.join(''), 'utf8'); }; +project.rewriteDirectDependencies = function (appDir, deps) { + + var lines = []; + + //XXX: constraints, old stuff. + _.each(deps, function (version, name) { + lines.push(name + "@" + version + "\n"); + }); + lines.sort(); + + fs.writeFileSync(path.join(appDir, '.meteor', 'meteor'), + lines.join(''), 'utf8'); +}; + var meteorReleaseFilePath = function (appDir) { return path.join(appDir, '.meteor', 'release'); }; -// Add a direct dependency -project.addDirectDependency = function (appDir, constraintString) { - var lines = getPackagesLines(appDir); - - // XXX: Remove previous instance of constraint if one existed. - - lines.push(constraintString); - writePackages(appDir, lines); -};