var fs = require('../util/fs'); var path = require('path'); var mout = require('mout'); var Q = require('q'); var mkdirp = require('mkdirp'); var rimraf = require('../util/rimraf'); var LRU = require('lru-cache'); var lockFile = require('lockfile'); var md5 = require('md5-hex'); var semver = require('../util/semver'); var readJson = require('../util/readJson'); var copy = require('../util/copy'); function ResolveCache(config) { // TODO: Make some config entries, such as: // - Max MB // - Max versions per source // - Max MB per source // - etc.. this._config = config; this._dir = this._config.storage.packages; this._lockDir = this._config.storage.packages; mkdirp.sync(this._lockDir); // Cache is stored/retrieved statically to ensure singularity // among instances this._cache = this.constructor._cache.get(this._dir); if (!this._cache) { this._cache = new LRU({ max: 100, maxAge: 60 * 5 * 1000 // 5 minutes }); this.constructor._cache.set(this._dir, this._cache); } // Ensure dir is created mkdirp.sync(this._dir); } // ----------------- ResolveCache.prototype.retrieve = function(source, target) { var sourceId = md5(source); var dir = path.join(this._dir, sourceId); var that = this; target = target || '*'; return this._getVersions(sourceId) .spread(function(versions) { var suitable; // If target is a semver, find a suitable version if (semver.validRange(target)) { suitable = semver.maxSatisfying(versions, target, true); if (suitable) { return suitable; } } // If target is '*' check if there's a cached '_wildcard' if (target === '*') { return mout.array.find(versions, function(version) { return version === '_wildcard'; }); } // Otherwise check if there's an exact match return mout.array.find(versions, function(version) { return version === target; }); }) .then(function(version) { var canonicalDir; if (!version) { return []; } // Resolve with canonical dir and package meta canonicalDir = path.join(dir, encodeURIComponent(version)); return that._readPkgMeta(canonicalDir).then( function(pkgMeta) { return [canonicalDir, pkgMeta]; }, function() { // If there was an error, invalidate the in-memory cache, // delete the cached package and try again that._cache.del(sourceId); return Q.nfcall(rimraf, canonicalDir).then(function() { return that.retrieve(source, target); }); } ); }); }; ResolveCache.prototype.store = function(canonicalDir, pkgMeta) { var sourceId; var release; var dir; var pkgLock; var promise; var that = this; promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalDir); return promise .then(function(pkgMeta) { sourceId = md5(pkgMeta._source); release = that._getPkgRelease(pkgMeta); dir = path.join(that._dir, sourceId, release); pkgLock = path.join( that._lockDir, sourceId + '-' + release + '.lock' ); // Check if destination directory exists to prevent issuing lock at all times return Q.nfcall(fs.stat, dir) .fail(function(err) { var lockParams = { wait: 250, retries: 25, stale: 60000 }; return Q.nfcall(lockFile.lock, pkgLock, lockParams) .then(function() { // Ensure other process didn't start copying files before lock was created return Q.nfcall(fs.stat, dir).fail(function(err) { // If stat fails, it is expected to return ENOENT if (err.code !== 'ENOENT') { throw err; } // Create missing directory and copy files there return Q.nfcall(mkdirp, path.dirname(dir)).then( function() { return Q.nfcall( fs.rename, canonicalDir, dir ).fail(function(err) { // If error is EXDEV it means that we are trying to rename // across different drives, so we copy and remove it instead if (err.code !== 'EXDEV') { throw err; } return copy.copyDir( canonicalDir, dir ); }); } ); }); }) .finally(function() { lockFile.unlockSync(pkgLock); }); }) .finally(function() { // Ensure no tmp dir is left on disk. return Q.nfcall(rimraf, canonicalDir); }); }) .then(function() { var versions = that._cache.get(sourceId); // Add it to the in memory cache // and sort the versions afterwards if (versions && versions.indexOf(release) === -1) { versions.push(release); that._sortVersions(versions); } // Resolve with the final location return dir; }); }; ResolveCache.prototype.eliminate = function(pkgMeta) { var sourceId = md5(pkgMeta._source); var release = this._getPkgRelease(pkgMeta); var dir = path.join(this._dir, sourceId, release); var that = this; return Q.nfcall(rimraf, dir).then(function() { var versions = that._cache.get(sourceId) || []; mout.array.remove(versions, release); // If this was the last package in the cache, // delete the parent folder (source) // For extra security, check against the file system // if this was really the last package if (!versions.length) { that._cache.del(sourceId); return that._getVersions(sourceId).spread(function(versions) { if (!versions.length) { // Do not keep in-memory cache if it's completely // empty that._cache.del(sourceId); return Q.nfcall(rimraf, path.dirname(dir)); } }); } }); }; ResolveCache.prototype.clear = function() { return Q.nfcall(rimraf, this._dir) .then( function() { return Q.nfcall(fs.mkdir, this._dir); }.bind(this) ) .then( function() { this._cache.reset(); }.bind(this) ); }; ResolveCache.prototype.reset = function() { this._cache.reset(); return this; }; ResolveCache.prototype.versions = function(source) { var sourceId = md5(source); return this._getVersions(sourceId).spread(function(versions) { return versions.filter(function(version) { return semver.valid(version); }); }); }; ResolveCache.prototype.list = function() { var promises; var dirs = []; var that = this; // Get the list of directories return ( Q.nfcall(fs.readdir, this._dir) .then(function(sourceIds) { promises = sourceIds.map(function(sourceId) { return Q.nfcall( fs.readdir, path.join(that._dir, sourceId) ).then( function(versions) { versions.forEach(function(version) { var dir = path.join( that._dir, sourceId, version ); dirs.push(dir); }); }, function(err) { // Ignore lurking files, e.g.: .DS_Store if the user // has navigated throughout the cache if (err.code === 'ENOTDIR' && err.path) { return Q.nfcall(rimraf, err.path); } throw err; } ); }); return Q.all(promises); }) // Read every package meta .then(function() { promises = dirs.map(function(dir) { return that._readPkgMeta(dir).then( function(pkgMeta) { return { canonicalDir: dir, pkgMeta: pkgMeta }; }, function() { // If it fails to read, invalidate the in memory // cache for the source and delete the entry directory var sourceId = path.basename(path.dirname(dir)); that._cache.del(sourceId); return Q.nfcall(rimraf, dir); } ); }); return Q.all(promises); }) // Sort by name ASC & release ASC .then(function(entries) { // Ignore falsy entries due to errors reading // package metas entries = entries.filter(function(entry) { return !!entry; }); return entries.sort(function(entry1, entry2) { var pkgMeta1 = entry1.pkgMeta; var pkgMeta2 = entry2.pkgMeta; var comp = pkgMeta1.name.localeCompare(pkgMeta2.name); // Sort by name if (comp) { return comp; } // Sort by version if (pkgMeta1.version && pkgMeta2.version) { return semver.compare( pkgMeta1.version, pkgMeta2.version ); } if (pkgMeta1.version) { return -1; } if (pkgMeta2.version) { return 1; } // Sort by target return pkgMeta1._target.localeCompare(pkgMeta2._target); }); }) ); }; // ------------------------ ResolveCache.clearRuntimeCache = function() { // Note that _cache refers to the static _cache variable // that holds other caches per dir! // Do not confuse it with the instance cache // Clear cache of each directory this._cache.forEach(function(cache) { cache.reset(); }); // Clear root cache this._cache.reset(); }; // ------------------------ ResolveCache.prototype._getPkgRelease = function(pkgMeta) { var release = pkgMeta.version || (pkgMeta._target === '*' ? '_wildcard' : pkgMeta._target); // Encode some dangerous chars such as / and \ release = encodeURIComponent(release); return release; }; ResolveCache.prototype._readPkgMeta = function(dir) { var filename = path.join(dir, '.bower.json'); return readJson(filename).spread(function(json) { return json; }); }; ResolveCache.prototype._getVersions = function(sourceId) { var dir; var versions = this._cache.get(sourceId); var that = this; if (versions) { return Q.resolve([versions, true]); } dir = path.join(this._dir, sourceId); return Q.nfcall(fs.readdir, dir).then( function(versions) { // Sort and cache in memory that._sortVersions(versions); versions = versions.map(decodeURIComponent); that._cache.set(sourceId, versions); return [versions, false]; }, function(err) { // If the directory does not exists, resolve // as an empty array if (err.code === 'ENOENT') { versions = []; that._cache.set(sourceId, versions); return [versions, false]; } throw err; } ); }; ResolveCache.prototype._sortVersions = function(versions) { // Sort DESC versions.sort(function(version1, version2) { var validSemver1 = semver.valid(version1); var validSemver2 = semver.valid(version2); // If both are semvers, compare them if (validSemver1 && validSemver2) { return semver.rcompare(version1, version2); } // If one of them are semvers, give higher priority if (validSemver1) { return -1; } if (validSemver2) { return 1; } // Otherwise they are considered equal return 0; }); }; // ------------------------ ResolveCache._cache = new LRU({ max: 5, maxAge: 60 * 30 * 1000 // 30 minutes }); module.exports = ResolveCache;