From 55f996fc6cdab971bf2a4332a355a55df95f0916 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 7 May 2014 15:44:11 -0700 Subject: [PATCH] use tree hash instead of tarball hash less sensitive to mtime, uid, etc --- tools/commands.js | 9 ++++--- tools/files.js | 57 +++++++++++++++++++++++++++++++++++++++++ tools/package-client.js | 30 +++++++--------------- tools/unipackage.js | 5 ++++ 4 files changed, 77 insertions(+), 24 deletions(-) diff --git a/tools/commands.js b/tools/commands.js index adc6d6e8fc..c31108597f 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1926,13 +1926,13 @@ main.registerCommand({ var somethingChanged = !existingBuild; if (!somethingChanged) { - // Bundle the build, just to get its hash. + // Save the unipackage, just to get its hash. // XXX this is redundant with the bundle build step that // publishPackage will do later var bundleBuildResult = packageClient.bundleBuild( compileResult.unipackage); - if (bundleBuildResult.tarballHash !== - existingBuild.build.hash) { + if (bundleBuildResult.treeHash !== + existingBuild.build.treeHash) { somethingChanged = true; } } @@ -2020,6 +2020,9 @@ main.registerCommand({ packages: relConf.packages }); + // Get it back. + catalog.refresh(true); + process.stdout.write("Done! \n"); return 0; }); diff --git a/tools/files.js b/tools/files.js index dd80d42715..e9b777e6a5 100644 --- a/tools/files.js +++ b/tools/files.js @@ -264,6 +264,63 @@ var makeTreeReadOnly = function (p) { } }; +// Returns the base64 SHA256 of the given file. +files.fileHash = function (filename) { + var crypto = require('crypto'); + var hash = crypto.createHash('sha256'); + hash.setEncoding('base64'); + var rs = fs.createReadStream(filename); + var fut = new Future(); + rs.on('end', function () { + rs.close(); + fut.return(hash.digest('base64')); + }); + rs.pipe(hash, { end: false }); + return fut.wait(); +}; + + +// Returns a base64 SHA256 hash representing a tree on disk. It is not sensitive +// to modtime, uid/gid, or any permissions bits other than the current-user-exec +// bit on normal files. +files.treeHash = function (root) { + var crypto = require('crypto'); + var hash = crypto.createHash('sha256'); + + var traverse = function (relativePath) { + var absPath = path.join(root, relativePath); + var stat = fs.lstatSync(absPath); + + if (stat.isDirectory()) { + if (relativePath) { + hash.update('dir ' + JSON.stringify(relativePath) + '\n'); + } + _.each(fs.readdirSync(absPath), function (entry) { + traverse(path.join(relativePath, entry)); + }); + } else if (stat.isFile()) { + if (!relativePath) { + throw Error("must call files.treeHash on a directory"); + } + hash.update('file ' + JSON.stringify(relativePath) + ' ' + + stat.size + ' ' + files.fileHash(absPath) + '\n'); + if (stat.mode & 0100) { + hash.update('exec\n'); + } + } else if (stat.isSymbolicLink()) { + if (!relativePath) { + throw Error("must call files.treeHash on a directory"); + } + hash.update('symlink ' + JSON.stringify(relativePath) + ' ' + + JSON.stringify(fs.readlinkSync(absPath)) + '\n'); + } + // ignore anything weirder + }; + + traverse(''); + return hash.digest('base64'); +}; + // like mkdir -p. if it returns true, the item is a directory (even if // it was already created). if it returns false, the item is not a // directory and we couldn't make it one. diff --git a/tools/package-client.js b/tools/package-client.js index d9e02cfddc..b77a5cb365 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -289,21 +289,6 @@ exports.loggedInPackagesConnection = function () { } }; -var hashTarball = function (tarball) { - var crypto = require('crypto'); - var hash = crypto.createHash('sha256'); - hash.setEncoding('base64'); - var rs = fs.createReadStream(tarball); - var fut = new Future(); - rs.on('end', function () { - fut.return(hash.digest('base64')); - }); - rs.pipe(hash, { end: false }); - var tarballHash = fut.wait(); - rs.close(); - return tarballHash; -}; - // XXX this is missing a few things: // - locking down build-time dependencies: tools version, versions // of all (not-built-from-source) plugins used @@ -358,7 +343,7 @@ var bundleSource = function (unipackage, includeSources, packageDir) { var sourceTarball = path.join(tempDir, packageTarName + '.tgz'); files.createTarball(dirToTar, sourceTarball); - var tarballHash = hashTarball(sourceTarball); + var tarballHash = files.fileHash(sourceTarball); return { sourceTarball: sourceTarball, @@ -388,8 +373,7 @@ exports.uploadTarball = uploadTarball; var bundleBuild = function (unipackage) { var tempDir = files.mkdtemp('build-package-'); - var packageTarName = unipackage.name + '-' + unipackage.version + '-' + - unipackage.architecturesString(); + var packageTarName = unipackage.tarballName(); var tarInputDir = path.join(tempDir, packageTarName); unipackage.saveToPath(tarInputDir); @@ -403,11 +387,13 @@ var bundleBuild = function (unipackage) { var buildTarball = path.join(tempDir, packageTarName + '.tgz'); files.createTarball(tarInputDir, buildTarball); - var tarballHash = hashTarball(buildTarball); + var tarballHash = files.fileHash(buildTarball); + var treeHash = files.treeHash(tarInputDir); return { buildTarball: buildTarball, - tarballHash: tarballHash + tarballHash: tarballHash, + treeHash: treeHash }; }; @@ -429,7 +415,9 @@ var createAndPublishBuiltPackage = function (conn, unipackage) { process.stdout.write('Publishing package build...\n'); conn.call('publishPackageBuild', - uploadInfo.uploadToken, bundleResult.tarballHash); + uploadInfo.uploadToken, + bundleResult.tarballHash, + bundleResult.treeHash); process.stdout.write('Published ' + unipackage.name + ', version ' + unipackage.version); diff --git a/tools/unipackage.js b/tools/unipackage.js index f55667ced9..f8bf80985f 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -307,6 +307,11 @@ _.extend(Unipackage.prototype, { return self.architectures().join('+'); }, + tarballName: function () { + var self = this; + return self.name + '-' + self.version + '-' + self.architecturesString(); + }, + _toolArchitectures: function () { var self = this; var toolArches = _.pluck(self.toolsOnDisk, 'arch');