github/bower
1
1
Fork 0
mirror of https://github.com/bower/bower.git synced 2026-04-24 03:00:19 -04:00
Code Issues Packages Projects Releases Wiki Activity

Add cache clean and list commands.

Browse Source 
This was an huge commit.
- Allow nested commands
- Switch resolve cache in memory object cache to LRU
- Store non-semver packages in the cache
- Tweak help messages
- Various fixes and tweaks
This commit is contained in:
André Cruz
2013-06-11 00:39:18 +01:00
parent 10fd2b5b5a
commit 6d3b3e6de2
29 changed files with 670 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
lib/core/Manager.js
View File

@@ -13,7 +13,7 @@ var endpointParser = require('../util/endpointParser');
function Manager(config, logger) {
this._config = config;
this._logger = logger;
this._repository = new PackageRepository(this._config);
this._repository = new PackageRepository(this._config, this._logger);
this.setResolutions();
this.configure();
@@ -202,7 +202,6 @@ Manager.prototype.areCompatible = function (first, second) {
Manager.prototype._fetch = function (decEndpoint) {
var name = decEndpoint.name;
var logger;
// Check if the whole process started to fail fast
if (this._hasFailed) {
@@ -214,18 +213,10 @@ Manager.prototype._fetch = function (decEndpoint) {
this._fetching[name].push(decEndpoint);
this._nrFetching++;
// Create a new logger that pipes everything to ours that will be
// used to fetch
// The endpoint is added for each log made
logger = this._logger.geminate().intercept(function (log) {
log.data = log.data || [];
log.data.endpoint = mout.object.pick(decEndpoint, ['name', 'source', 'target']);
});
// Fetch it from the repository
// Note that the promise is stored in the decomposed endpoint
// because it might be reused if a similar endpoint needs to be resolved
return decEndpoint.promise = this._repository.fetch(decEndpoint, logger)
return decEndpoint.promise = this._repository.fetch(decEndpoint)
// When done, call onFetchSuccess
.spread(this._onFetchSuccess.bind(this, decEndpoint))
// If it fails, call onFetchFailure
@@ -428,15 +419,9 @@ Manager.prototype._dissect = function () {
return !!decEndpoint.pkgMeta.version;
});
// Sort semver ones
// Sort semver ones DESC
semvers.sort(function (first, second) {
if (semver.gt(first, second)) {
return -1;
}
if (semver.lt(first, second)) {
return 1;
}
return 0;
return semver.rcompare(first, second);
});
// Filter non-semver ones
@@ -462,14 +447,14 @@ Manager.prototype._dissect = function () {
if (this._saveResolutions) {
this._logger.info('resolution', 'Removed unnecessary ' + name + '#' + resolution + ' resolution', {
package: name,
name: name,
resolution: resolution,
action: 'delete'
});
delete this._resolutions[name];
} else {
this._logger.warn('resolution', 'Unnecessary ' + name + '#' + resolution + ' resolution', {
package: name,
name: name,
resolution: resolution
});
}
@@ -534,14 +519,13 @@ Manager.prototype._electSuitable = function (name, semvers, nonSemvers) {
picks.sort(function (pick1, pick2) {
var version1 = pick1.pkgMeta.version;
var version2 = pick2.pkgMeta.version;
var comp;
// If both have versions, compare their versions using semver
if (version1 && version2) {
if (semver.gt(version1, version2)) {
return 1;
}
if (semver.lt(version1, version2)) {
return -1;
comp = semver.compare(version1, version2);
if (!comp) {
return comp;
}
} else {
// Give priority to the one that is a version
@@ -637,7 +621,7 @@ Manager.prototype._electSuitable = function (name, semvers, nonSemvers) {
if (this._saveResolutions) {
this._logger.info('resolution', 'Saved ' + name + '#' + resolution + ' as resolution', {
package: name,
name: name,
resolution: resolution,
action: this._resolutions[name] ? 'edit' : 'add'
});

51
lib/core/PackageRepository.js
View File

@@ -4,10 +4,11 @@ var ResolveCache = require('./ResolveCache');
var resolverFactory = require('./resolverFactory');
var createError = require('../util/createError');
function PackageRepository(config) {
function PackageRepository(config, logger) {
var registryOptions;
this._config = config;
this._logger = logger;
// Instantiate the registry
registryOptions = mout.object.deepMixIn({}, this._config);
@@ -20,12 +21,20 @@ function PackageRepository(config) {
// -----------------
PackageRepository.prototype.fetch = function (decEndpoint, logger) {
var info = {};
PackageRepository.prototype.fetch = function (decEndpoint) {
var logger;
var that = this;
var info = {
decEndpoint: decEndpoint
};
// Create a new logger that pipes everything to ours that will be
// used to fetch
logger = this._logger.geminate();
// Intercept all logs, adding additional information
logger.intercept(this._extendLog.bind(this, info));
logger.intercept(function (log) {
that._extendLog(log, info);
});
// Get the appropriate resolver
return resolverFactory(decEndpoint, this._config, logger, this._registryClient)
@@ -37,7 +46,7 @@ PackageRepository.prototype.fetch = function (decEndpoint, logger) {
// If force flag is used, bypass cache
if (that._config.force) {
logger.action('resolve', resolver.getSource() + '#' + resolver.getTarget());
return that._resolve(resolver);
return that._resolve(resolver, logger);
}
// Note that we use the resolver methods to query the
@@ -58,7 +67,7 @@ PackageRepository.prototype.fetch = function (decEndpoint, logger) {
logger.info('not-cached', resolver.getSource() + (resolver.getTarget() ? '#' + resolver.getTarget() : ''));
logger.action('resolve', resolver.getSource() + '#' + resolver.getTarget());
return that._resolve(resolver);
return that._resolve(resolver, logger);
}
info.canonicalPkg = canonicalPkg;
@@ -87,7 +96,7 @@ PackageRepository.prototype.fetch = function (decEndpoint, logger) {
logger.info('new', 'version for ' + resolver.getSource() + '#' + resolver.getTarget());
logger.action('resolve', resolver.getSource() + '#' + resolver.getTarget());
return that._resolve(resolver);
return that._resolve(resolver, logger);
});
});
})
@@ -98,16 +107,21 @@ PackageRepository.prototype.fetch = function (decEndpoint, logger) {
});
};
PackageRepository.prototype.empty = function (name, logger) {
// TODO Think of a way to remove specific packages of a given name from the cache
// Since the ResolveCache.empty only works with source, one possible solution is to implement
// a forEach method that calls a function with the canonicalPackage and the pkgMeta
// so that we can match against the pkgMeta.name and call ResolveCache.empty with it
PackageRepository.prototype.eliminate = function (pkgMeta) {
return this._resolveCache.eliminate(pkgMeta);
};
PackageRepository.prototype.clean = function () {
return this._resolveCache.clean();
};
PackageRepository.prototype.list = function () {
return this._resolveCache.list();
};
// ---------------------
PackageRepository.prototype._resolve = function (resolver) {
PackageRepository.prototype._resolve = function (resolver, logger) {
// Resolve the resolver
return resolver.resolve()
// Store in the cache
@@ -116,13 +130,20 @@ PackageRepository.prototype._resolve = function (resolver) {
}.bind(this))
// Resolve promise with canonical package and package meta
.then(function (dir) {
return [dir, resolver.getPkgMeta()];
var pkgMeta = resolver.getPkgMeta();
logger.info('resolved', resolver.getSource() + (pkgMeta._release ? '#' + pkgMeta._release : ''));
return [dir, pkgMeta];
}.bind(this));
};
PackageRepository.prototype._extendLog = function (info, log) {
PackageRepository.prototype._extendLog = function (log, info) {
log.data = log.data || {};
// Store endpoint info in each log
if (info.decEndpoint) {
log.data.endpoint = mout.object.pick(info.decEndpoint, ['name', 'source', 'target']);
}
// Store the resolver info in each log
if (info.resolver) {
log.data.resolver = {

36
lib/core/Project.js
View File

@@ -247,7 +247,7 @@ Project.prototype.uninstall = function (names, options) {
// even with dependants
message = dependants.map(function (dep) { return dep.name; }).join(', ') + ' depends on ' + decEndpoint.name;
data = {
package: decEndpoint.name,
name: decEndpoint.name,
dependants: dependants.map(function (decEndpoint) {
return decEndpoint.name;
})
@@ -325,7 +325,7 @@ Project.prototype.analyse = function () {
}
release = decEndpoint.pkgMeta._release;
this._logger.log('warn', 'extraneous', decEndpoint.name + (release ? '#' + release : release), {
this._logger.log('warn', 'extraneous', decEndpoint.name + (release ? '#' + release : ''), {
pkgMeta: decEndpoint.pkgMeta,
canonicalPkg: decEndpoint.canonicalPkg
});
@@ -392,7 +392,10 @@ Project.prototype._readJson = function () {
return Q.nfcall(bowerJson.read, filename)
.fail(function (err) {
throw createError('Something went wrong while reading ' + filename, err.code, {
details: err.message
details: err.message,
data: {
filename: filename
}
});
});
}, function () {
@@ -432,20 +435,29 @@ Project.prototype._readInstalled = function () {
filenames.forEach(function (filename) {
var promise;
var name = path.dirname(filename);
var jsonFile = path.join(componentsDir, filename);
filename = path.join(componentsDir, filename);
// Read package metadata
promise = Q.nfcall(fs.readFile, jsonFile)
promise = Q.nfcall(fs.readFile, filename)
.then(function (contents) {
var pkgMeta = JSON.parse(contents.toString());
return JSON.parse(contents.toString());
})
.then(function (pkgMeta) {
decEndpoints[name] = {
name: name,
source: pkgMeta._source,
target: pkgMeta.version || '*',
canonicalPkg: path.dirname(jsonFile),
canonicalPkg: path.dirname(filename),
pkgMeta: pkgMeta
};
}, function (err) {
throw createError('Something went wrong while reading "' + filename + '"', err.code, {
details: err.message,
data: {
json: filename
}
});
});
promises.push(promise);
@@ -469,8 +481,8 @@ Project.prototype._removePackages = function (packages, options) {
// Delete directory
if (!dir) {
promise = Q.resolve();
this._logger.info('absent', name, {
package: name
this._logger.warn('not-installed', name, {
name: name
});
} else {
promise = Q.nfcall(rimraf, dir);
@@ -503,7 +515,9 @@ Project.prototype._removePackages = function (packages, options) {
.then(this._saveJson.bind(this))
// Resolve with removed packages
.then(function () {
return packages;
return mout.object.filter(packages, function (dir) {
return !!dir;
});
});
};

234
lib/core/ResolveCache.js
View File

@@ -6,6 +6,8 @@ var mout = require('mout');
var Q = require('q');
var mkdirp = require('mkdirp');
var rimraf = require('rimraf');
var LRU = require('lru-cache');
var createError = require('../util/createError');
function ResolveCache(config) {
// TODO: Make some config entries, such as:
@@ -15,23 +17,36 @@ function ResolveCache(config) {
// - etc..
this._config = config;
this._dir = this._config.roaming.cache;
this._versions = {};
this._cache = this.constructor._cache.get(this._dir);
if (!this._cache) {
this._cache = new LRU({
max: 100,
maxAge: 60 * 30 * 1000 // 30 minutes
});
this.constructor._cache.set(this._dir, this._cache);
}
// Ensure dir is created
mkdirp.sync(this._dir);
}
// -----------------
ResolveCache.prototype.retrieve = function (source, target) {
var fromCache;
var sourceId = this._getSourceId(source);
var dir = path.join(this._dir, sourceId);
var that = this;
target = target || '*';
return this._getVersions(source)
.then(function (versions) {
return this._getVersions(sourceId)
.spread(function (versions, cached) {
var suitable;
fromCache = cached;
// If target is a semver, find a suitable version
if (semver.valid(target) != null || semver.validRange(target) != null) {
suitable = mout.array.find(versions, function (version) {
@@ -43,10 +58,10 @@ ResolveCache.prototype.retrieve = function (source, target) {
}
}
// If target is '*' check if there's a cached '_unversioned'
// If target is '*' check if there's a cached '_wildcard'
if (target === '*') {
return mout.array.find(versions, function (version) {
return version === '_unversioned';
return version === '_wildcard';
});
}
@@ -64,24 +79,35 @@ ResolveCache.prototype.retrieve = function (source, target) {
// Resolve with canonical package and package meta
canonicalPkg = path.join(dir, version);
return this._readPkgMeta(canonicalPkg)
return that._readPkgMeta(canonicalPkg)
.then(function (pkgMeta) {
return [canonicalPkg, pkgMeta];
}, function (err) {
// If the version was fetched from the cache and we got
// a ENOENT error, it means that the in memory cache is
// no longer valid.
// As such we eliminate it.
if (fromCache && err.code === 'ENOENT') {
that._cache.del(sourceId);
}
});
}.bind(this));
});
};
ResolveCache.prototype.store = function (canonicalPkg, pkgMeta) {
var promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalPkg);
var sourceId;
var pkgVersion;
var release;
var dir;
var promise;
var that = this;
promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalPkg);
return promise
.then(function (pkgMeta) {
sourceId = this._getSourceId(pkgMeta._source);
pkgVersion = pkgMeta.version || '_unversioned';
dir = path.join(this._dir, sourceId, pkgVersion);
release = pkgMeta.version || (pkgMeta._target === '*' ? '_wildcard' : pkgMeta._target);
sourceId = that._getSourceId(pkgMeta._source);
dir = path.join(that._dir, sourceId, release);
// Check if directory exists
return Q.nfcall(fs.stat, dir)
@@ -101,100 +127,185 @@ ResolveCache.prototype.store = function (canonicalPkg, pkgMeta) {
.then(function () {
return Q.nfcall(fs.rename, canonicalPkg, dir);
});
}.bind(this))
})
.then(function () {
var pkgVersion = pkgMeta.version || '_unversioned';
var versions = this._versions[sourceId];
var inCache;
var versions = that._cache.get(sourceId);
if (versions) {
// Check if this exact version already exists in the
// memory cache
inCache = versions.some(function (version) {
return pkgVersion === version;
});
// If it doesn't, add it to the in memory cache
// and sort the versions afterwards
if (!inCache) {
versions.push(pkgVersion);
this._sortVersions(versions);
}
// Add it to the in memory cache
// and sort the versions afterwards
if (versions && versions.indexOf(release) === -1) {
versions.push(release);
that._sortVersions(versions);
}
// Resolve with the final location
return dir;
});
};
ResolveCache.prototype.eliminate = function (pkgMeta) {
var sourceId = this._getSourceId(pkgMeta._source);
var version = pkgMeta.version || '_wildcard';
var dir = path.join(this._dir, sourceId, version);
var that = this;
return Q.nfcall(rimraf, dir)
.then(function () {
var versions = that._cache.get(sourceId) || [];
mout.array.remove(versions, version);
// If this was the last package in the cache,
// delete the parent folder (source)
// For extra security, we check against the file system
// this was the last package
if (!versions.length) {
that._cache.del(sourceId);
versions = that._getVersions(sourceId);
if (!versions.length) {
return Q.nfcall(rimraf, path.dirname(dir));
}
}
});
};
ResolveCache.prototype.clean = function () {
return Q.nfcall(rimraf, this._dir)
.then(function () {
this._cache.reset();
}.bind(this));
};
ResolveCache.prototype.eliminate = function (source, version) {
// TODO:
};
ResolveCache.prototype.list = function () {
var promises;
var dirs = [];
var that = this;
ResolveCache.prototype.empty = function (source) {
// TODO:
// Get the list of directories
return Q.nfcall(fs.readdir, this._dir)
.then(function (sourceIds) {
promises = sourceIds.map(function (sourceId) {
return Q.nfcall(fs.readdir, path.join(that._dir, sourceId))
.then(function (versions) {
versions.forEach(function (version) {
var dir = path.join(that._dir, sourceId, version);
dirs.push(dir);
});
}, function (err) {
// Ignore lurking files
if (err.code === 'ENOTDIR') {
return;
}
throw err;
});
});
return Q.all(promises);
})
// Read every package meta
.then(function () {
promises = dirs.map(function (dir) {
return that._readPkgMeta(dir)
.then(function (pkgMeta) {
return pkgMeta;
});
});
return Q.all(promises);
})
// Sort by name ASC & version ASC
.then(function (pkgMetas) {
return pkgMetas.sort(function (pkgMeta1, pkgMeta2) {
var comp = pkgMeta1.name.localeCompare(pkgMeta2.name);
if (comp) {
return comp;
}
if (pkgMeta1.version && pkgMeta2.version) {
return semver.compare(pkgMeta1.version, pkgMeta2.version);
}
if (pkgMeta1.version) {
return 1;
}
if (pkgMeta2.version) {
return -1;
}
return 0;
});
});
};
// ------------------------
ResolveCache.clearRuntimeCache = function () {
this._cache.reset();
};
// ------------------------
ResolveCache.prototype._getSourceId = function (source) {
return crypto.createHash('md5').update(source).digest('hex');
};
ResolveCache.prototype._readPkgMeta = function (dir) {
return Q.nfcall(fs.readFile, path.join(dir, '.bower.json'))
var filename = path.join(dir, '.bower.json');
return Q.nfcall(fs.readFile, filename)
.then(function (contents) {
return JSON.parse(contents.toString());
})
.fail(function (err) {
throw createError('Something went wrong while reading "' + filename + '"', err.code, {
details: err.message,
data: {
json: filename
}
});
});
};
ResolveCache.prototype._getVersions = function (source) {
ResolveCache.prototype._getVersions = function (sourceId) {
var dir;
var sourceId = this._getSourceId(source);
var cache = this._versions[sourceId];
var versions = this._cache.get(sourceId);
var that = this;
if (cache) {
return Q.resolve(cache);
if (versions) {
return Q.resolve([versions, true]);
}
dir = path.join(this._dir, sourceId);
return Q.nfcall(fs.readdir, dir)
.then(function (versions) {
// If there are no versions there, do not cache in memory
if (!versions.length) {
return versions;
}
// Sort and cache in memory
this._sortVersions(versions);
return this._versions[sourceId] = versions;
}.bind(this), function (err) {
that._sortVersions(versions);
that._cache.set(sourceId, versions);
return [versions, false];
}, function (err) {
// If the directory does not exists, resolve
// as an empty array
if (err.code === 'ENOENT') {
return this._versions[sourceId] = [];
versions = [];
that._cache.set(sourceId, versions);
return [versions, false];
}
throw err;
}.bind(this));
});
};
ResolveCache.prototype._sortVersions = function (versions) {
// Sort DESC
versions.sort(function (version1, version2) {
var validSemver1 = semver.valid(version1) != null;
var validSemver2 = semver.valid(version2) != null;
// If both are semvers, compare them
if (validSemver1 && validSemver2) {
if (semver.gt(version1, version2)) {
return -1;
}
if (semver.lt(version1, version2)) {
return 1;
}
return 0;
return semver.rcompare(version1, version2);
}
// If one of them are semvers, give higher priority
@@ -210,4 +321,11 @@ ResolveCache.prototype._sortVersions = function (versions) {
});
};
// ------------------------
ResolveCache._cache = new LRU({
max: 5,
maxAge: 60 * 30 * 1000 // 30 minutes
});
module.exports = ResolveCache;

7
lib/core/resolvers/GitResolver.js
View File

@@ -227,9 +227,9 @@ GitResolver.fetchVersions = function (source) {
}
}
// Sort them by desc order
versions = versions.sort(function (a, b) {
return semver.gt(a.version, b.version) ? -1 : 1;
// Sort them by DESC order
versions.sort(function (a, b) {
return semver.rcompare(a.version, b.version);
});
this._versions = this._versions || {};
@@ -287,6 +287,7 @@ GitResolver.fetchBranches = function (source) {
}.bind(this));
};
// TODO: switch to LRU cache
GitResolver.clearRuntimeCache = function () {
this._branches = null;
this._tags = null;

3
lib/core/resolvers/Resolver.js
View File

@@ -198,8 +198,9 @@ Resolver.prototype._applyPkgMeta = function (meta) {
Resolver.prototype._savePkgMeta = function (meta) {
var contents;
// Store original source
// Store original source & target
meta._source = this._source;
meta._target = this._target;
// Stringify contents
contents = JSON.stringify(meta, null, 2);