diff --git a/.gitignore b/.gitignore index e2ce2903..c4eb24fe 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,6 @@ node_modules npm-debug.log # Project specific -test/assets/github-test-package \ No newline at end of file +test/assets/github-test-package +test/assets/github-test-package-copy +test/assets/temp \ No newline at end of file diff --git a/NOTES.md b/NOTES.md index 606dbaf6..9408d818 100644 --- a/NOTES.md +++ b/NOTES.md @@ -43,6 +43,6 @@ - Use yeomen insight!! - bower could setup a git hook on folders that are github repos to make validation of the json (if it conforms with the spec) - in prod dont forget to Q.longStackJumpLimit = 0; -- switch from events to promise progress - - wait for domenic response on twitter - - progress events: name_change, warn (deprecated json, mismatch version..), action +- add perf tests +- url resolver should work with fonts, e.g.: http://fonts.googleapis.com/css?family=Noto+Serif +- discuss ability to specify folders inside bower_components.. e.g. components/fonts/ diff --git a/README.md b/README.md index db2b099a..01dff5ea 100644 --- a/README.md +++ b/README.md @@ -225,7 +225,7 @@ Throws an error if the resolver is not yet resolved. ##### Public static functions -`Resolver#clearRuntimeCache` +`Resolver#clearRuntimeCache()` Clears the resolver runtime cache, that is, data stored statically. Resolvers may cache data based on the sources to speed up calls to `hasNew` and `resolve` for the diff --git a/lib/resolve/Resolver.js b/lib/resolve/Resolver.js index 9bccc833..1ca20e2a 100644 --- a/lib/resolve/Resolver.js +++ b/lib/resolve/Resolver.js @@ -152,18 +152,9 @@ Resolver.prototype._applyPkgMeta = function (meta) { if (meta.ignore && meta.ignore.length) { return Q.nfcall(glob, '**/*', { cwd: this._tempDir, dot: true, mark: true }) .then(function (files) { - var filter = this._createIgnoreFilter(meta.ignore), - promises = []; + var promises = []; - // For each file that passes the ignore filter, - // rimraf it - files.forEach(function (file) { - if (filter(file)) { - promises.push(Q.nfcall(rimraf, file)); - } - }); - - // Wait for all the rimraf's to finish + // TODO return Q.all(promises); }.bind(this)) .then(function () { @@ -183,12 +174,4 @@ Resolver.prototype._savePkgMeta = function (meta) { }.bind(this)); }; -Resolver.prototype._createIgnoreFilter = function (ignore) { - var list = pathspec.RelPathList.parse(ignore); - - return function (filename) { - return list.matches(filename); - }; -}; - module.exports = Resolver; \ No newline at end of file diff --git a/lib/resolve/resolvers/FsResolver.js b/lib/resolve/resolvers/FsResolver.js index 0ddfe990..40a14c75 100644 --- a/lib/resolve/resolvers/FsResolver.js +++ b/lib/resolve/resolvers/FsResolver.js @@ -1,33 +1,80 @@ var util = require('util'); +var fs = require('fs'); +var path = require('path'); +var mout = require('mout'); var Q = require('q'); var Resolver = require('../Resolver'); +var copy = require('../../util/copy'); +var extract = require('../../util/extract'); +var createError = require('../../util/createError'); var FsResolver = function (source, options) { + // Ensure absolute path + source = path.resolve(source); + Resolver.call(this, source, options); }; util.inherits(FsResolver, Resolver); +mout.object.mixIn(FsResolver, Resolver); // ----------------- FsResolver.prototype.hasNew = function (canonicalPkg) { + // If target was specified, simply reject the promise + if (this._target !== '*') { + return Q.reject(createError('File system sources can\'t resolve targets ("' + this._target + '")', 'ENORESTARGET')); + } + // TODO: should we store latest modified files in the resolution and compare? return Q.resolve(true); }; FsResolver.prototype._resolveSelf = function () { - return this._copy() + // If target was specified, simply reject the promise + if (this._target !== '*') { + return Q.reject(createError('File system sources can\'t resolve targets ("' + this._target + '")', 'ENORESTARGET')); + } + + return this._readJson(this._source) + .then(this._copy.bind(this)) .then(this._extract.bind(this)); }; // ----------------- -FsResolver.prototype._copy = function () { +FsResolver.prototype._copy = function (meta) { + return Q.nfcall(fs.stat, this._source) + .then(function (stat) { + var dstFile, + copyOpts; + // Pass in the ignore to the copy options to avoid copying ignore files + // Also, pass in the mode to avoid additional stat calls when copying + copyOpts = { + mode: stat.mode, + ignore: meta.ignore + }; + + // If it's a folder + if (stat.isDirectory()) { + return copy.copyDir(this._source, this._tempDir, copyOpts); + } + + // If it's a file + // We pass the mode to avoid additional stat calls + dstFile = path.join(this._tempDir, path.basename(this._source)); + return copy.copyFile(this._source, dstFile, copyOpts); + }.bind(this)); }; FsResolver.prototype._extract = function () { - + return extract.canExtract(this._source) + .then(function (canExtract) { + if (canExtract) { + return extract(this._tempDir); + } + }); }; module.exports = FsResolver; \ No newline at end of file diff --git a/lib/resolve/resolvers/GitFsResolver.js b/lib/resolve/resolvers/GitFsResolver.js index 1cd11c43..bbcdbd90 100644 --- a/lib/resolve/resolvers/GitFsResolver.js +++ b/lib/resolve/resolvers/GitFsResolver.js @@ -2,8 +2,8 @@ var util = require('util'); var fs = require('fs'); var Q = require('q'); var mout = require('mout'); -var ncp = require('ncp'); var GitResolver = require('./GitResolver'); +var copy = require('../../util/copy'); var cmd = require('../../util/cmd'); var path = require('path'); @@ -20,15 +20,7 @@ mout.object.mixIn(GitFsResolver, GitResolver); // ----------------- GitFsResolver.prototype._copy = function () { - // Copy folder permissions - return Q.nfcall(fs.stat, this._source) - .then(function (stat) { - return Q.nfcall(fs.chmod, this._tempDir, stat.mode); - }.bind(this)) - // Copy folder contents - .then(function () { - return Q.nfcall(ncp, this._source, this._tempDir); - }.bind(this)); + return copy.copyDir(this._source, this._tempDir); }; // Override the checkout function to work with the local copy diff --git a/lib/util/copy.js b/lib/util/copy.js new file mode 100644 index 00000000..c4f36c95 --- /dev/null +++ b/lib/util/copy.js @@ -0,0 +1,117 @@ +var fstream = require('fstream'); +var fstreamIgnore = require('fstream-ignore'); +var fs = require('fs'); +var Q = require('q'); + +function copy(reader, writer) { + var deferred = Q.defer(), + ignore, + finish; + + finish = function (err) { + writer.removeAllListeners(); + reader.removeAllListeners(); + + // If we got an error, simply reject the deferred + if (err) { + return deferred.reject(err); + } + + return deferred.resolve(); + }; + + // Reader + if (reader.type === 'Directory' && reader.ignore) { + ignore = reader.ignore; + reader = fstreamIgnore(reader); + reader.addIgnoreRules(ignore); + } else { + reader = fstream.Reader(reader); + } + + reader.on('error', finish); + + // Writer + writer = fstream.Writer(writer) + .on('error', finish) + .on('close', finish); + + // Finally pipe reader to writer + reader.pipe(writer); + + return deferred.promise; +} + +function copyMode(src, dst) { + return Q.nfcall(fs.stat, src) + .then(function (stat) { + return Q.nfcall(fs.chmod, dst, stat.mode); + }); +} + +function parseOptions(opts) { + opts = opts || {}; + + if (opts.mode != null) { + opts.copyMode = false; + } else if (opts.copyMode == null) { + opts.copyMode = true; + } + + return opts; +} + +// --------------------- + +// Available options: +// - mode: force final mode of dest +// - copyMode: copy mode of src to dest (only if mode is not specified) +function copyFile(src, dst, opts) { + var promise; + + opts = parseOptions(opts); + + promise = copy({ + path: src, + type: 'File' + }, { + path: dst, + mode: opts.mode, + type: 'File' + }); + + if (opts.copyMode) { + promise = promise.then(copyMode.bind(copyMode, src, dst)); + } + + return promise; +} + +// Available options: +// - ignore: array of patterns to be ignored +// - mode: force final mode of dest +// - copyMode: copy mode of src to dest (only if mode is not specified) +function copyDir(src, dst, opts) { + var promise; + + opts = parseOptions(opts); + + promise = copy({ + path: src, + type: 'Directory', + ignore: opts.ignore + }, { + path: dst, + mode: opts.mode, + type: 'Directory' + }); + + if (opts.copyMode) { + promise = promise.then(copyMode.bind(copyMode, src, dst)); + } + + return promise; +} + +module.exports.copyDir = copyDir; +module.exports.copyFile = copyFile; \ No newline at end of file diff --git a/lib/util/extract.js b/lib/util/extract.js new file mode 100644 index 00000000..9d13eded --- /dev/null +++ b/lib/util/extract.js @@ -0,0 +1,151 @@ +var path = require('path'); +var fs = require('fs'); +var zlib = require('zlib'); +var unzip = require('unzip'); +var tar = require('tar'); +var Q = require('Q'); +var mout = require('mout'); + +var extractors = { + '.zip': extractZip, + '.tar': extractTar, + '.tar.gz': extractTarGz +}; +var extractorTypes = Object.keys(extractors); + +function extractZip(archive, dest) { + var deferred = Q.defer(); + + fs.createReadStream(archive) + .pipe(unzip.Extract({ path: this.path })) + .on('error', deferred.reject) + .on('close', deferred.resolve.bind(deferred, dest)); + + return deferred.promise; +} + +function extractTar(archive, dest) { + var deferred = Q.defer(); + + fs.createReadStream(archive) + .pipe(tar.Extract({ path: this.path })) + .on('error', deferred.reject) + .on('close', deferred.resolve.bind(deferred, dest)); + + return deferred.promise; +} + +function extractTarGz(archive, dest) { + var deferred = Q.defer(); + + fs.createReadStream(archive) + .pipe(zlib.createGunzip()) + .pipe(tar.Extract({ path: this.path })) + .on('error', deferred.reject) + .on('close', deferred.resolve.bind(deferred, dest)); + + return deferred.promise; +} + +function getExtractor(archive) { + var type = mout.array.find(extractorTypes, function (type) { + return mout.string.endsWith(archive, type); + }); + + return type ? extractors[type] : null; +} + +function isSingleDir(dir) { + return Q.nfcall(fs.readdir, dir) + .then(function (files) { + var dir; + + if (files.length !== 1) { + return false; + } + + dir = files[0]; + + return Q.nfcall(fs.stat, dir) + .then(function (stat) { + return !stat.isDirectory() ? files[0] : false; + }); + }); +} + +function moveSingleDirContents(dir) { + var destDir = path.dirname(dir); + + return Q.nfcall(fs.readdir, dir) + .then(function (files) { + var promises; + + promises = files.map(function (file) { + var src = path.join(dir, file), + dest = path.join(destDir, file); + + return Q.nfcall(fs.rename, src, dest); + }); + + return Q.all(promises); + }) + .then(function () { + return Q.rmdir(dir); + }); +} + +// ----------------------------- + +function extract(archive, dest, options) { + var extractor, + promise; + + options = options || {}; + extractor = getExtractor(options.extension || archive); + + // If extractor is null, then the archive type is unknown + if (!extractor) { + return Q.reject(new Error('File "' + archive + '" is not a known archive')); + } + + // Extract archive + promise = extractor(archive, dest); + + // Remove archive + if (!options.keepArchive) { + promise = promise + .then(function () { + return Q.nfcall(fs.unlink, archive); + }); + } + + // Move contents if a single directory was extracted + if (!options.keepStructure) { + promise = promise + .then(function () { + return isSingleDir(dest); + }) + .then(function (singleDir) { + return singleDir ? moveSingleDirContents(singleDir) : null; + }); + } + + // Resolve promise to the dest dir + return promise.then(function () { + return dest; + }); +} + +function canExtract(archive) { + if (!getExtractor(archive)) { + return Q.resolve(false); + } + + return Q.nfcall(fs.stat, archive) + .then(function (stat) { + return stat.isFile(); + }); +} + +module.exports = extract; +module.exports.canExtract = canExtract; \ No newline at end of file diff --git a/package.json b/package.json index 371f38dc..86b80c15 100644 --- a/package.json +++ b/package.json @@ -23,12 +23,16 @@ "mkdirp": "~0.3.5", "nopt": "~2.1.1", "semver": "~1.1.4", - "ncp": "~0.4.2", "pathspec": "~0.9.2", "glob": "~3.1.21", "rimraf": "~2.1.4", "chmodr": "~0.1.0", - "request": "~2.20.0" + "request": "~2.20.0", + "unzip": "~0.1.7", + "tar": "~0.1.17", + "walkdir": "0.0.7", + "fstream": "~0.1.22", + "fstream-ignore": "0.0.6" }, "devDependencies": { "mocha": "~1.8.2", diff --git a/test/resolve/resolvers/fsResolver.js b/test/resolve/resolvers/fsResolver.js new file mode 100644 index 00000000..6b287c03 --- /dev/null +++ b/test/resolve/resolvers/fsResolver.js @@ -0,0 +1,148 @@ +var expect = require('expect.js'); +var path = require('path'); +var fs = require('fs'); +var path = require('path'); +var rimraf = require('rimraf'); +var cmd = require('../../../lib/util/cmd'); +var copy = require('../../../lib/util/copy'); +var FsResolver = require('../../../lib/resolve/resolvers/FsResolver'); + +describe('FsResolver', function () { + var testPackage = path.resolve(__dirname, '../../assets/github-test-package'), + tempSource; + + before(function (next) { + // Checkout test package to version 0.2.1 which has a bower.json + // with ignores + cmd('git', ['checkout', '0.2.1'], { cwd: testPackage }) + .then(next.bind(next, null), next); + }); + + afterEach(function (next) { + if (tempSource) { + rimraf(tempSource, next); + tempSource = null; + } else { + next(); + } + }); + + describe('.constructor', function () { + it('should guess the name from the path', function () { + var resolver = new FsResolver(testPackage); + + expect(resolver.getName()).to.equal('github-test-package'); + }); + + it('should make paths absolute and normalized', function () { + var resolver; + + resolver = new FsResolver(path.relative(process.cwd(), testPackage)); + expect(resolver.getSource()).to.equal(testPackage); + + resolver = new FsResolver(testPackage + '/something/..'); + expect(resolver.getSource()).to.equal(testPackage); + }); + }); + + describe('.hasNew', function () { + it.skip('should be false if the file modified date hasn\'t changed'); + it.skip('should be false if the directory modified date hasn\'t changed'); + it.skip('should be true if the file modified date has changed'); + it.skip('should be true if the directory modified date has changed'); + it.skip('should ignore files specified to be ignored'); + }); + + describe('.resolve', function () { + it('should copy the source file', function (next) { + var resolver = new FsResolver(path.join(testPackage, 'foo')); + + resolver.resolve() + .then(function (dir) { + expect(fs.existsSync(path.join(dir, 'foo'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'bar'))).to.be(false); + next(); + }) + .done(); + }); + + it('should copy the source directory contents', function (next) { + var resolver = new FsResolver(testPackage); + + resolver.resolve() + .then(function (dir) { + expect(fs.existsSync(path.join(dir, 'foo'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'bar'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'baz'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'README.md'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'more'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'more', 'more-foo'))).to.be(true); + next(); + }) + .done(); + }); + + it('should copy the source file permissions', function (next) { + var mode0777, + resolver; + + tempSource = path.resolve(__dirname, '../../assets/temp'); + resolver = new FsResolver(tempSource); + + copy.copyFile(path.join(testPackage, 'foo'), tempSource) + .then(function () { + // Change tempSource dir to 0777 + fs.chmodSync(tempSource, 0777); + // Get the mode to a variable + mode0777 = fs.statSync(tempSource).mode; + }) + .then(resolver.resolve.bind(resolver)) + .then(function (dir) { + // Check if file is 0777 + var stat = fs.statSync(path.join(dir, 'temp')); + expect(stat.mode).to.equal(mode0777); + next(); + }) + .done(); + }); + + it('should copy the source directory permissions', function (next) { + var mode0777, + resolver; + + tempSource = path.resolve(__dirname, '../../assets/github-test-package-copy'); + resolver = new FsResolver(tempSource); + + copy.copyDir(testPackage, tempSource) + .then(function () { + // Change tempSource dir to 0777 + fs.chmodSync(tempSource, 0777); + // Get the mode to a variable + mode0777 = fs.statSync(tempSource).mode; + }) + .then(resolver.resolve.bind(resolver)) + .then(function (dir) { + // Check if temporary dir is 0777 instead of default 0777 & ~process.umask() + var stat = fs.statSync(dir); + expect(stat.mode).to.equal(mode0777); + next(); + }) + .done(); + }); + + it('should not copy ignored paths', function (next) { + var resolver = new FsResolver(testPackage); + + // Override the _applyPkgMeta function to prevent it from deleting ignored files + resolver._applyPkgMeta = function () {}; + + resolver.resolve() + .then(function (dir) { + expect(fs.existsSync(path.join(dir, 'foo'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'test'))).to.be(false); + next(); + }) + .done(); + }); + }); +}); \ No newline at end of file diff --git a/test/resolve/resolvers/gitFsResolver.js b/test/resolve/resolvers/gitFsResolver.js index 6eba4608..f1777749 100644 --- a/test/resolve/resolvers/gitFsResolver.js +++ b/test/resolve/resolvers/gitFsResolver.js @@ -2,16 +2,28 @@ var expect = require('expect.js'); var path = require('path'); var fs = require('fs'); var path = require('path'); +var rimraf = require('rimraf'); var cmd = require('../../../lib/util/cmd'); +var copy = require('../../../lib/util/copy'); var GitFsResolver = require('../../../lib/resolve/resolvers/GitFsResolver'); describe('GitFsResolver', function () { - var testPackage = path.resolve(__dirname, '../../assets/github-test-package'); + var testPackage = path.resolve(__dirname, '../../assets/github-test-package'), + tempSource; function clearResolverRuntimeCache() { GitFsResolver.clearRuntimeCache(); } + afterEach(function (next) { + if (tempSource) { + rimraf(tempSource, next); + tempSource = null; + } else { + next(); + } + }); + describe('.constructor', function () { it('should guess the name from the path', function () { var resolver = new GitFsResolver(testPackage); @@ -150,16 +162,20 @@ describe('GitFsResolver', function () { }); it('should copy source folder permissions', function (next) { - var mode0777; + var mode0777, + resolver; - // Change testPackage dir to 0777 - fs.chmodSync(testPackage, 0777); - // Get the mode to a variable - mode0777 = fs.statSync(testPackage).mode; + tempSource = path.resolve(__dirname, '../../assets/github-test-package-copy'); + resolver = new GitFsResolver(tempSource, { target: 'some-branch' }); - var resolver = new GitFsResolver(testPackage, { target: 'some-branch' }); - - resolver.resolve() + copy.copyDir(testPackage, tempSource) + .then(function () { + // Change tempSource dir to 0777 + fs.chmodSync(tempSource, 0777); + // Get the mode to a variable + mode0777 = fs.statSync(tempSource).mode; + }) + .then(resolver.resolve.bind(resolver)) .then(function (dir) { // Check if temporary dir is 0777 instead of default 0777 & ~process.umask() var stat = fs.statSync(dir); diff --git a/test/resolve/resolvers/gitResolver.js b/test/resolve/resolvers/gitResolver.js index 7ad298fa..04afb0f3 100644 --- a/test/resolve/resolvers/gitResolver.js +++ b/test/resolve/resolvers/gitResolver.js @@ -5,9 +5,9 @@ var fs = require('fs'); var mkdirp = require('mkdirp'); var chmodr = require('chmodr'); var rimraf = require('rimraf'); -var ncp = require('ncp'); var Q = require('q'); var mout = require('mout'); +var copy = require('../../../lib/util/copy'); var GitResolver = require('../../../lib/resolve/resolvers/GitResolver'); describe('GitResolver', function () { @@ -639,18 +639,19 @@ describe('GitResolver', function () { dest = path.join(tempDir, '.git'); // Copy .git folder to the tempDir - ncp(path.resolve(__dirname, '../../../.git'), dest, function (err) { - if (err) return next(err); - + copy.copyDir(path.resolve(__dirname, '../../../.git'), dest, { + mode: 0777 + }) + .then(function () { resolver._tempDir = tempDir; - resolver._cleanup() + return resolver._cleanup() .then(function () { expect(fs.existsSync(dest)).to.be(false); next(); - }) - .done(); - }); + }); + }) + .done(); }); it('should not fail if .git does not exist for some reason', function (next) { diff --git a/test/test.js b/test/test.js index 2ee58545..db73f0ff 100644 --- a/test/test.js +++ b/test/test.js @@ -1,62 +1,11 @@ -var GitRemoteResolver = require('../lib/resolve/resolvers/GitRemoteResolver'); -var GitFsResolver = require('../lib/resolve/resolvers/GitFsResolver'); +// Cleanup the uncaughtException added by the tmp module +// It messes with the mocha uncaughtException event to caught errors +process.removeAllListeners('uncaughtException'); -function testGitRemoteResolver() { - var dejavuResolver = new GitRemoteResolver('git://github.com/IndigoUnited/dejavu.git', { - name: 'dejavu', - //target: '7d07190ca6fb7ffa63642526537e0c314cbaab12' - //target: 'master' - target: '~0.4.1' - }); - - return dejavuResolver.resolve() - .then(function () { - console.log('ok!'); - }); -} - -function testGitFsResolver() { - var bowerResolver = new GitFsResolver(__dirname + '/..', { - name: 'bower', - target: 'rewrite' - }); - - return bowerResolver.resolve() - .then(function () { - console.log('ok!'); - }); -} - -function testGitRemoteResolverNoTags() { - var spoonResolver = new GitRemoteResolver('git://github.com/IndigoUnited/spoon.js.git', { - name: 'spoonjs', - //target: '7d07190ca6fb7ffa63642526537e0c314cbaab12' - //target: 'master' - target: '*' - }); - - return spoonResolver.resolve() - .then(function () { - console.log('ok!'); - }); -} - -if (process.argv[1] && !/mocha/.test(process.argv[1])) { - testGitRemoteResolver() - .then(testGitFsResolver) - .then(testGitRemoteResolverNoTags); - - //testGitFsResolver(); - //testGitRemoteResolverNoTags(); -} else { - // Cleanup the uncaughtException added by the tmp module - // It messes with the mocha uncaughtException event to caught errors - process.removeAllListeners('uncaughtException'); - - require('./resolve/resolver'); - require('./resolve/resolvers/gitResolver'); - require('./resolve/resolvers/gitFsResolver'); - require('./resolve/resolvers/gitRemoteResolver'); - require('./resolve/worker'); - require('./resolve/resolverFactory'); -} \ No newline at end of file +require('./resolve/resolver'); +require('./resolve/resolvers/gitResolver'); +require('./resolve/resolvers/gitFsResolver'); +require('./resolve/resolvers/gitRemoteResolver'); +require('./resolve/resolvers/fsResolver'); +require('./resolve/worker'); +//require('./resolve/resolverFactory');