use tree hash instead of tarball hash

less sensitive to mtime, uid, etc

tools/commands.js

@@ -1926,13 +1926,13 @@ main.registerCommand({
                   var somethingChanged = !existingBuild;
 
                   if (!somethingChanged) {
-                    // Bundle the build, just to get its hash.
+                    // Save the unipackage, just to get its hash.
                     // XXX this is redundant with the bundle build step that
                     // publishPackage will do later
                     var bundleBuildResult = packageClient.bundleBuild(
                       compileResult.unipackage);
-                    if (bundleBuildResult.tarballHash !==
-                        existingBuild.build.hash) {
+                    if (bundleBuildResult.treeHash !==
+                        existingBuild.build.treeHash) {
                       somethingChanged = true;
                     }
                   }
@@ -2020,6 +2020,9 @@ main.registerCommand({
     packages: relConf.packages
   });
 
+  // Get it back.
+  catalog.refresh(true);
+
   process.stdout.write("Done! \n");
   return 0;
 });

tools/files.js

@@ -264,6 +264,63 @@ var makeTreeReadOnly = function (p) {
   }
 };
 
+// Returns the base64 SHA256 of the given file.
+files.fileHash = function (filename) {
+  var crypto = require('crypto');
+  var hash = crypto.createHash('sha256');
+  hash.setEncoding('base64');
+  var rs = fs.createReadStream(filename);
+  var fut = new Future();
+  rs.on('end', function () {
+    rs.close();
+    fut.return(hash.digest('base64'));
+  });
+  rs.pipe(hash, { end: false });
+  return fut.wait();
+};
+
+
+// Returns a base64 SHA256 hash representing a tree on disk. It is not sensitive
+// to modtime, uid/gid, or any permissions bits other than the current-user-exec
+// bit on normal files.
+files.treeHash = function (root) {
+  var crypto = require('crypto');
+  var hash = crypto.createHash('sha256');
+
+  var traverse = function (relativePath) {
+    var absPath = path.join(root, relativePath);
+    var stat = fs.lstatSync(absPath);
+
+    if (stat.isDirectory()) {
+      if (relativePath) {
+        hash.update('dir ' + JSON.stringify(relativePath) + '\n');
+      }
+      _.each(fs.readdirSync(absPath), function (entry) {
+        traverse(path.join(relativePath, entry));
+      });
+    } else if (stat.isFile()) {
+      if (!relativePath) {
+        throw Error("must call files.treeHash on a directory");
+      }
+      hash.update('file ' + JSON.stringify(relativePath) + ' ' +
+                  stat.size + ' ' + files.fileHash(absPath) + '\n');
+      if (stat.mode & 0100) {
+        hash.update('exec\n');
+      }
+    } else if (stat.isSymbolicLink()) {
+      if (!relativePath) {
+        throw Error("must call files.treeHash on a directory");
+      }
+      hash.update('symlink ' + JSON.stringify(relativePath) + ' ' +
+                  JSON.stringify(fs.readlinkSync(absPath)) + '\n');
+    }
+    // ignore anything weirder
+  };
+
+  traverse('');
+  return hash.digest('base64');
+};
+
 // like mkdir -p. if it returns true, the item is a directory (even if
 // it was already created). if it returns false, the item is not a
 // directory and we couldn't make it one.

tools/package-client.js

@@ -289,21 +289,6 @@ exports.loggedInPackagesConnection = function () {
   }
 };
 
-var hashTarball = function (tarball) {
-  var crypto = require('crypto');
-  var hash = crypto.createHash('sha256');
-  hash.setEncoding('base64');
-  var rs = fs.createReadStream(tarball);
-  var fut = new Future();
-  rs.on('end', function () {
-    fut.return(hash.digest('base64'));
-  });
-  rs.pipe(hash, { end: false });
-  var tarballHash = fut.wait();
-  rs.close();
-  return tarballHash;
-};
-
 // XXX this is missing a few things:
 //    - locking down build-time dependencies: tools version, versions
 //      of all (not-built-from-source) plugins used
@@ -358,7 +343,7 @@ var bundleSource = function (unipackage, includeSources, packageDir) {
   var sourceTarball = path.join(tempDir, packageTarName + '.tgz');
   files.createTarball(dirToTar, sourceTarball);
 
-  var tarballHash = hashTarball(sourceTarball);
+  var tarballHash = files.fileHash(sourceTarball);
 
   return {
     sourceTarball: sourceTarball,
@@ -388,8 +373,7 @@ exports.uploadTarball = uploadTarball;
 
 var bundleBuild = function (unipackage) {
   var tempDir = files.mkdtemp('build-package-');
-  var packageTarName = unipackage.name + '-' + unipackage.version + '-' +
-        unipackage.architecturesString();
+  var packageTarName = unipackage.tarballName();
   var tarInputDir = path.join(tempDir, packageTarName);
 
   unipackage.saveToPath(tarInputDir);
@@ -403,11 +387,13 @@ var bundleBuild = function (unipackage) {
   var buildTarball = path.join(tempDir, packageTarName + '.tgz');
   files.createTarball(tarInputDir, buildTarball);
 
-  var tarballHash = hashTarball(buildTarball);
+  var tarballHash = files.fileHash(buildTarball);
+  var treeHash = files.treeHash(tarInputDir);
 
   return {
     buildTarball: buildTarball,
-    tarballHash: tarballHash
+    tarballHash: tarballHash,
+    treeHash: treeHash
   };
 };
 
@@ -429,7 +415,9 @@ var createAndPublishBuiltPackage = function (conn, unipackage) {
 
   process.stdout.write('Publishing package build...\n');
   conn.call('publishPackageBuild',
-            uploadInfo.uploadToken, bundleResult.tarballHash);
+            uploadInfo.uploadToken,
+            bundleResult.tarballHash,
+            bundleResult.treeHash);
 
   process.stdout.write('Published ' + unipackage.name +
                        ', version ' + unipackage.version);

tools/unipackage.js

@@ -307,6 +307,11 @@ _.extend(Unipackage.prototype, {
     return self.architectures().join('+');
   },
 
+  tarballName: function () {
+    var self = this;
+    return self.name + '-' + self.version + '-' + self.architecturesString();
+  },
+
   _toolArchitectures: function () {
     var self = this;
     var toolArches = _.pluck(self.toolsOnDisk, 'arch');