add and remove work

tools/catalog.js

@@ -51,6 +51,12 @@ var Catalog = function () {
 
   // All packages found either by localPackageDirs or localPackages
   self.effectiveLocalPackages = {}; // package name to source directory
+
+  // Set this to true if we are not going to connect to the remote package
+  // server, and will only use the cached data.json file for our package
+  // information. This means that the catalog might be out of date on the latest
+  // developments.
+  self.offline = null;
 };
 
 _.extend(Catalog.prototype, {
@@ -93,15 +99,28 @@ _.extend(Catalog.prototype, {
     // OK, now initialize the catalog for real, with both local and
     // package server packages.
     console.log("XXX Loading catalog for real");
-    self._refresh(true);
+
+    // We should to figure out if we are intending to connect to the package
+    // server.
+    self.offline = options.offline ? options.offline : false;
+    self._refresh(true /* load server packages */);
   },
 
-  // Set sync to true to try to synchronize from the package server.
+  // If sync is false, this will not synchronize with the remote server, even if
+  // the catalog is not in offline mode. This is an optimization for loading
+  // local packages. (An offline catalog will not sync with the server even if
+  // sync is true.)
   _refresh: function (sync) {
     var self = this;
     self._requireInitialized();
 
-    var serverPackageData = packageClient.loadPackageData(sync);
+    var localData = packageClient.loadCachedServerData();
+    var serverPackageData;
+    if (! self.offline && sync) {
+      serverPackageData = packageClient.updateServerPackageData(localData);
+    } else {
+      serverPackageData = localData.collections;
+    }
 
     self.initialized = false;
     self.packages = [];

tools/commands.js

@@ -579,129 +579,160 @@ main.registerCommand({
   var failed = false;
 
   // Read in existing package dependencies.
-  var usingDirectly = project.getDepsAsObj(project.getDirectDependencies(options.appDir));
+  var packages = project.getDepsAsObj(project.getDirectDependencies(options.appDir));
 
-  // For every package name specified, run it through the constraint
-  // solver and add the right stuff to .meteor/package and
-  // .meteor/versions files.
+  // For every package name specified, add it to our list of package
+  // constraints. Don't run the constraint solver until you have added all of
+  // them -- add should be an atomic operation regardless of the package
+  // order. Even though the package file should specify versions of its inputs,
+  // we don't specify these constraints until we get them back from the
+  // constraint solver.
   _.each(options.args, function (packageReq) {
-    if (failed)
-      return;
-
+    // XXX: Use a util function.
     var constraint = project.processPackageConstraint(packageReq);
 
+    // Check that the package exists.
     if (! catalog.getPackage(constraint.packageName)) {
       process.stderr.write(constraint.packageName + ": no such package\n");
       failed = true;
       return;
     }
+    // Check that the version exists.
     var versionInfo = catalog.getVersion(
       constraint.packageName,
+      // XXX: Use a util function.
       getVersionFromVersionConstraint(constraint.versionConstraint));
-
     if (! versionInfo) {
       process.stderr.write(
-constraint.packageName + "@" + constraint.versionConstraint  + ": no such version\n");
+        constraint.packageName + "@" + constraint.versionConstraint  + ": no such version\n");
       failed = true;
       return;
     }
 
-    if (_.has(usingDirectly, constraint.packageName)) {
-      if (usingDirectly[constraint.packageName] === constraint.versionConstraint) {
-        process.stderr.write(constraint.packageName + "@" + constraint.versionConstraint   + ": already added\n");
-        failed = true;
+    // Check that the constraint is new. If we are already using the package at
+    // the same constraint, return from this function.
+    if (_.has(packages, constraint.packageName)) {
+      if (packages[constraint.packageName] === constraint.versionConstraint) {
+        process.stderr.write(
+          constraint.packageName + "@" + constraint.versionConstraint   + ": already added\n");
         return;
-      } else if (!constraint.versionConstraint && (usingDirectly[constraint.packageName] === "none")) {
+      } else if (!constraint.versionConstraint && (packages[constraint.packageName] === "none")) {
+        // XXX: In the brand new world where we have versioning in .meteor/packages, this will not happen.
         process.stderr.write(constraint.packageName + ": already added\n");
-        failed = true;
         return;
       }
+    }
+
+    // Add the package to our direct dependency constraints that we get from .meteor/packages.
+    packages[constraint.packageName] = constraint.versionConstraint;
+  });
+
+  // If the user asked for invalid packages, then the user probably expects a
+  // different result than what they are going to get. We have already logged an
+  // error, so we should exit.
+  if ( failed ) {
+    return 1;
+  }
+
+  // Get the contents of our versions file. We need to pass them to the
+  // constraint solver, because our contract with the user says that we will
+  // never downgrade a dependency.
+  var versions = project.getDepsAsObj(project.getIndirectDependencies(options.appDir));
+
+  // Call the constraint solver.
+  var constraintSolver = require('./constraint-solver.js');
+  var resolver = new constraintSolver.Resolver;
+  // XXX: constraint solver currently ignores versions, but it should not.
+  // XXX: this would also be the place to add no-update options.
+  var newVersions = resolver.resolve(packages);
+  if ( ! newVersions) {
+    // XXX: Better error handling.
+    process.stderr.write("Cannot resolve package dependencies.");
+  }
+
+  // Don't tell the user what all the operations were until we finish -- we
+  // don't want to give a false sense of completeness until everything is
+  // written to disk.
+  var messageLog = [];
+
+  // Remove the versions that don't exist
+  var removed = _.difference(_.keys(versions), _.keys(newVersions));
+  _.each(removed, function(packageName) {
+    messageLog.push("removed dependency on " + packageName);
+  });
+
+  // Install the new versions.
+  _.each(newVersions, function(version, packageName) {
+    if ( failed )
+      return;
+    if (_.has(versions, packageName) &&
+         versions[packageName] == version ) {
+      // Nothing changed. Skip this.
+      return;
+     }
+
+    // Make sure we have enough builds of the package downloaded such that
+    // we can load a browser slice and a slice that will run on this
+    // system. (Later we may also need to download more builds to be able to
+    // deploy to another architecture.)
+    var available = tropohouse.maybeDownloadPackageForArchitectures(
+      catalog.getVersion(packageName, version),
+      // XXX we also download the deploy arch now, because we don't run the
+      // constraint solver / downloader anywhere other than add-package yet.
+      ['browser', archinfo.host(), XXX_DEPLOY_ARCH]);
+    if (! available) {
+      // XXX maybe we shouldn't be letting the constraint solver choose
+      // things that don't have the right arches?
+      process.stderr.write("Package " + packageName +
+                           " has no compatible build for version " +
+                           version);
+      failed = true;
+      return;
+    }
+
+    // Add a message to the update logs to show the user what we have done.
+    if ( _.contains(options.args, packageName)) {
+      // If we asked for this, we will log it later in more detail.
+      return;
+    }
+
+    // If the previous versions file had this, then we are upgrading, if it did
+    // not, then we must be adding this package anew.
+    if ( _.has(versions, packageName )) {
+      messageLog.push("upgraded " + packageName + " from version " +
+                      versions[packageName] +
+                      " to version " + newVersions[packageName]);
     } else {
-      // Add the package to the list of packages that we use directly.
-      usingDirectly[constraint.packageName] = constraint.versionConstraint;
-      var usingIndirectly = project.getDepsAsObj(project.getIndirectDependencies(options.appDir));
+      messageLog.push("added " + packageName + " from " +
+                      " at version " + newVersions[packageName]);
+    };
+  });
 
-      // Call the constraint solver.
-      var ConstraintSolver = uniload.load({
-        packages: ['constraint-solver'],
-        release: release.current.name
-      })['constraint-solver'].ConstraintSolver;
+  if (failed)
+    return 1;
 
-      var resolver = new ConstraintSolver.Resolver(catalog);
-      var newVersions = resolver.resolve(usingDirectly,
-                                         usingIndirectly,
-                                         { optionsGoHere : false });
+  // Write the .meteor/packages file with the right versions
+  var oldPackages = project.getDepsAsObj(project.getDirectDependencies(options.appDir));
+  project.rewriteDirectDependencies(options.appDir, packages);
 
-      var logMessage = "";
-      _.forEach(newVersions, function(version, packageName) {
-        if (failed)
-          return;
+  // Write the .meteor/versions file with the new dependencies.
+  project.rewriteIndirectDependencies(options.appDir, newVersions);
 
-        // Check if it exists.
-        if (usingIndirectly[packageName] === version) {
-          // We are using this at this version, so do nothing.
-        } else {
+  // Show the user the messageLog of packages we added.
+  _.each(messageLog, function (msg) {
+    process.stdout.write(msg + "\n");
+  });
 
-            // Find the build.
-            // XXX: Find the one with the right architecture.
-            var versionInfo = catalog.getVersion(packageName, version);
-
-            // Safety check, but this should not happen unless the
-            // constraint solver is doing something it shouldn't.
-            if (! versionInfo) {
-                process.stderr.write("This package has no version at this version");
-                failed = true;
-                return;
-            }
-
-            // Make sure we have enough builds of the package downloaded such that
-            // we can load a browser slice and a slice that will run on this
-            // system. (Later we may also need to download more builds to be able to
-            // deploy to another architecture.)
-            var available = tropohouse.maybeDownloadPackageForArchitectures(
-                // XXX we also download the deploy arch now, because we don't run the
-                // constraint solver / downloader anywhere other than add-package yet.
-                versionInfo, ['browser', archinfo.host(), XXX_DEPLOY_ARCH]);
-            if (! available) {
-                // XXX maybe we shouldn't be letting the constraint solver choose
-                // things that don't have the right arches?
-                process.stderr.write("Package " + packageName +
-                                     " has no compatible build for version " +
-                                     version);
-                failed = true;
-                return;
-            }
-
-            if (_.has(usingIndirectly[packageName])) {
-                logMessage = logMessage + "Upgraded " + packageName + " from version " +
-                             usingIndirectly + " to version " + version + "\n";
-            } else {
-               logMessage = logMessage + "Added " + packageName + " at version " +
-                            version + "\n";
-               logMessage = logMessage + "(" + packageName + " : " + versionInfo.decription + ") \n";
-            }
-        }
-      });
-
-      if (failed)
-        return;
-
-      // Add to the new direct dependencies file.
-      // XXX: Write the current version into packages file, rather than the requested version
-      project.addDirectDependency(options.appDir, packageReq);
-
-      // Write the new indirect dependencies file.
-      project.rewriteIndirectDependencies(options.appDir, newVersions);
-
-      // Log that this happened! Yay!
-      process.stdout.write(logMessage);
-      process.stdout.write("Finished adding: \n");
-      var note = versionInfo.description;
-      process.stdout.write(constraint.packageName + ": " + note + "\n");
+  // Show the user the messageLog of the packages that they installed.
+  process.stdout.write("Successfully added the following packages. \n");
+  _.each(packages, function (version, name) {
+    if ( ! _.has(oldPackages, name) ) {
+      var versionRecord = catalog.getVersion(name, version);
+      process.stdout.write(name + " : " + versionRecord.description + "\n");
     }
   });
 
-  return failed ? 1 : 0;
+  return 0;
 });
 
 
@@ -709,25 +740,73 @@ constraint.packageName + "@" + constraint.versionConstraint  + ": no such versio
 // remove
 ///////////////////////////////////////////////////////////////////////////////
 
+
 main.registerCommand({
   name: 'remove',
   minArgs: 1,
   maxArgs: Infinity,
   requiresApp: true
 }, function (options) {
-  var using = {};
-  _.each(project.getPackages(options.appDir), function (name) {
-    using[name] = true;
+
+  // Read in existing package dependencies.
+  var packages = project.getDepsAsObj(project.getDirectDependencies(options.appDir));
+
+  // For every package name specified, add it to our list of package
+  // constraints. Don't run the constraint solver until you have added all of
+  // them -- add should be an atomic operation regardless of the package
+  // order. Even though the package file should specify versions of its inputs,
+  // we don't specify these constraints until we get them back from the
+  // constraint solver.
+  _.each(options.args, function (packageName) {
+    // Check that we are using the package. We don't check if the package
+    // exists. You should be able to remove non-existent packages.
+    if (! _.has(packages, packageName)) {
+      process.stderr.write( packageName  + " is not in this project \n");
+    }
+
+    // Remove the package from our dependency list.
+    delete packages[packageName];
   });
 
-  _.each(options.args, function (name) {
-    if (! _.has(using, name)) {
-      process.stderr.write(name + ": not in project\n");
-    } else {
-      project.removePackage(options.appDir, name);
-      process.stderr.write(name + ": removed\n");
-    }
+  // Get the contents of our versions file. We need to pass them to the
+  // constraint solver, because our contract with the user says that we will
+  // never downgrade a dependency.
+  var versions = project.getDepsAsObj(project.getIndirectDependencies(options.appDir));
+
+  // Call the constraint solver.
+  var constraintSolver = require('./constraint-solver.js');
+  var resolver = new constraintSolver.Resolver;
+  // XXX: constraint solver currently ignores versions, but it should not.
+  // XXX: this would also be the place to add no-update options.
+  var newVersions = resolver.resolve(packages);
+  if ( ! newVersions) {
+    // This should never really happen.
+    process.stderr.write("Cannot resolve package dependencies.");
+  }
+
+  // Don't tell the user what all the operations were until we finish -- we
+  // don't want to give a false sense of completeness until everything is
+  // written to disk.
+  var messageLog = [];
+
+  // Remove the versions that don't exist
+  var removed = _.difference(_.keys(versions), _.keys(newVersions));
+  _.each(removed, function(packageName) {
+    messageLog.push("removed dependency on " + packageName);
   });
+
+  // Write the .meteor/packages file with the right versions
+  project.rewriteDirectDependencies(options.appDir, packages);
+
+  // Write the .meteor/versions file with the new dependencies.
+  project.rewriteIndirectDependencies(options.appDir, newVersions);
+
+  // Show the user the messageLog of everything we removed.
+  _.each(messageLog, function (msg) {
+    process.stdout.write(msg + "\n");
+  });
+
+  return 0;
 });
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -743,25 +822,15 @@ main.registerCommand({
     using: { type: Boolean }
   }
 }, function (options) {
-  if (options.using) {
-    var using = project.getPackages(options.appDir);
-
-    if (using.length) {
-      _.each(using, function (name) {
-        process.stdout.write(name + "\n");
-      });
-    } else {
-      process.stderr.write(
-"This project doesn't use any packages yet. To add some packages:\n" +
-"  meteor add <package> <package> ...\n" +
-"\n" +
-"To see available packages:\n" +
-"  meteor list\n");
+  var items = [];
+  _.each(catalog.getAllPackageNames(), function (name) {
+    var versionInfo = catalog.getLatestVersion(name);
+    if (versionInfo) {
+      items.push({ name: name, description: versionInfo.description });
     }
-    return;
-  }
+  });
 
-  throw new Error("XXX replace with list-all or remove completely");
+  process.stdout.write(formatList(items));
 });
 
 
@@ -1561,25 +1630,6 @@ main.registerCommand({
   return 0;
 });
 
-// This command will list all packages in existence.
-// This command may go away after testing is done.
-main.registerCommand({
-  name: 'list-all',
-  options: {},
-  maxArgs: 0,
-  hidden: true
-}, function (options) {
-  var items = [];
-  _.each(catalog.getAllPackageNames(), function (name) {
-    var versionInfo = catalog.getLatestVersion(name);
-    if (versionInfo) {
-      items.push({ name: name, description: versionInfo.description });
-    }
-  });
-
-  process.stdout.write(formatList(items));
-});
-
 main.registerCommand({
   name: 'publish-for-arch',
   minArgs: 0,

tools/main.js

@@ -687,9 +687,16 @@ Fiber(function () {
   // Initialize the singleton Catalog. Only after this point is the
   // Catalog (and therefore unipackage.load) usable.
   //
-  // This will try to talk to the network to synchronize our package
-  // list with the package server.
-  catalog.initialize({ localPackageDirs: localPackageDirs });
+  // If the --no-net option is set, the catalog will be offline and will never
+  // attempt to contact the server for more recent data. Otherwise, the catalog
+  // will attempt to synchronize with the remote package server.
+  catalog.initialize({
+      localPackageDirs: localPackageDirs,
+      offline: _.has(rawOptions, '--no-net')
+  });
+  // We need to delete the option or we will throw an error.
+  // XXX: This seems like a hack?
+  delete rawOptions['--no-net'];
 
   // Check for the '--help' option.
   var showHelp = false;

tools/package-client.js

@@ -49,7 +49,7 @@ var openPackageServerConnection = function () {
 // If there is no data.json file, or the file cannot be parsed, return null for
 // the collections and a default syncToken to ask the server for all the data
 // from the beginning of time.
-var loadCachedServerData = function () {
+exports.loadCachedServerData = function () {
   var noDataToken =  {
     // XXX have a better sync token for "all"
     syncToken: {time: 'Sun, 01 Jan 2012 00:00:00 GMT'},
@@ -74,7 +74,6 @@ var loadCachedServerData = function () {
   return ret;
 };
 
-
 // Opens a connection to the server, requests and returns new package data that
 // we haven't cached on disk. We assume that data is cached chronologically, so
 // essentially, we are asking for a diff from the last time that we did this.
@@ -96,7 +95,6 @@ var loadRemotePackageData = function (syncToken) {
   return collectionData;
 };
 
-
 // Take in an ordered list of javascript objects representing collections of
 // package data. In each object, the server-side names of collections are keys
 // and the values are the mongo records for that collection stored as an
@@ -126,7 +124,6 @@ var mergeCollections = function (sources) {
   return ret;
 };
 
-
 // Writes the cached package data to the on-disk cache. Takes in the following
 // arguments:
 // - syncToken : the token representing our conversation with the server, that
@@ -151,22 +148,24 @@ var writePackageDataToDisk = function (syncToken, collectionData) {
   files.writeFileAtomically(filename, JSON.stringify(finalWrite, null, 2));
 };
 
-// Returns the package data.
+// Contacts the package server to get the latest diff and writes changes to
+// disk.
 //
-exports.loadPackageData = function() {
-  //XXX: We can consider optimizing this with concurrency or something.
+// Takes in cachedServerData, which is the processed contents of data.json. Uses
+// those to talk to the server and get the latest updates. Applies the diff from
+// the server to the in-memory version of the on-disk data, then writes the new
+// file to disk as the new data.json.
+exports.updateServerPackageData = function (cachedServerData) {
   var sources = [];
-
-  var localData = loadCachedServerData();
-  if (localData.collections)
-    sources.push(localData.collections);
-  var syncToken = localData.syncToken;
-  // XXX support offline use too
-//  var remoteData = loadRemotePackageData(syncToken);
-//  sources.push(remoteData.collections);
+  if (cachedServerData.collections) {
+    sources.push(cachedServerData.collections);
+  }
+  var syncToken = cachedServerData.syncToken;
+  var remoteData = loadRemotePackageData(syncToken);
+  sources.push(remoteData.collections);
 
   var allPackageData = mergeCollections(sources);
-//  writePackagesToDisk(remoteData.syncToken, allPackageData);
+  writePackageDataToDisk(remoteData.syncToken, allPackageData);
   return allPackageData;
 };
 

tools/project.js

@@ -131,21 +131,26 @@ project.rewriteIndirectDependencies = function (appDir, deps) {
                    lines.join(''), 'utf8');
 };
 
+project.rewriteDirectDependencies = function (appDir, deps) {
+
+  var lines = [];
+
+  //XXX: constraints, old stuff.
+  _.each(deps, function (version, name) {
+    lines.push(name + "@" + version + "\n");
+  });
+  lines.sort();
+
+  fs.writeFileSync(path.join(appDir, '.meteor', 'meteor'),
+                   lines.join(''), 'utf8');
+};
+
 
 var meteorReleaseFilePath = function (appDir) {
   return path.join(appDir, '.meteor', 'release');
 };
 
 
-// Add a direct dependency
-project.addDirectDependency = function (appDir, constraintString) {
-  var lines = getPackagesLines(appDir);
-
-  // XXX: Remove previous instance of constraint if one existed.
-
-  lines.push(constraintString);
-  writePackages(appDir, lines);
-};