From e09a3b8cbf2ece7aa664c8d20651ac962ca88a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=CC=81=20Cruz?= Date: Mon, 13 May 2013 11:09:04 +0100 Subject: [PATCH] Huge commit, implement rough working version of the whole resolve process. --- .gitignore | 2 + NOTES.md | 8 + README.md | 106 +++++- lib/config.js | 76 ++-- lib/core/Manager.js | 337 ++++++++++++++++++ lib/core/PackageRepository.js | 91 +++++ lib/core/Project.js | 274 ++++++++++++++ lib/core/ResolveCache.js | 184 ++++++++++ lib/{resolve => core}/Worker.js | 0 lib/{resolve => core}/resolverFactory.js | 45 ++- lib/{resolve => core}/resolvers/FsResolver.js | 2 +- .../resolvers/GitFsResolver.js | 0 .../resolvers/GitRemoteResolver.js | 0 .../resolvers/GitResolver.js | 34 +- lib/{resolve => core/resolvers}/Resolver.js | 35 +- .../resolvers/UrlResolver.js | 25 +- lib/resolve/Manager.js | 25 -- lib/resolve/PackageRepository.js | 27 -- lib/util/decomposeEndpoint.js | 18 - lib/util/endpointParser.js | 35 ++ lib/util/extract.js | 5 + package.json | 6 +- test/assets/test-temp-dir/test-exception.js | 2 +- test/assets/test-temp-dir/test.js | 2 +- test/{resolve => core}/resolverFactory.js | 14 +- .../{resolve => core}/resolvers/fsResolver.js | 2 +- .../resolvers/gitFsResolver.js | 2 +- .../resolvers/gitRemoteResolver.js | 2 +- .../resolvers/gitResolver.js | 29 +- test/{resolve => core/resolvers}/resolver.js | 46 +-- .../resolvers/urlResolver.js | 17 +- test/{resolve => core}/worker.js | 2 +- test/test.js | 18 +- 33 files changed, 1247 insertions(+), 224 deletions(-) create mode 100644 lib/core/Manager.js create mode 100644 lib/core/PackageRepository.js create mode 100644 lib/core/Project.js create mode 100644 lib/core/ResolveCache.js rename lib/{resolve => core}/Worker.js (100%) rename lib/{resolve => core}/resolverFactory.js (60%) rename lib/{resolve => core}/resolvers/FsResolver.js (98%) rename lib/{resolve => core}/resolvers/GitFsResolver.js (100%) rename lib/{resolve => core}/resolvers/GitRemoteResolver.js (100%) rename lib/{resolve => core}/resolvers/GitResolver.js (91%) rename lib/{resolve => core/resolvers}/Resolver.js (87%) rename lib/{resolve => core}/resolvers/UrlResolver.js (89%) delete mode 100644 lib/resolve/Manager.js delete mode 100644 lib/resolve/PackageRepository.js delete mode 100644 lib/util/decomposeEndpoint.js create mode 100644 lib/util/endpointParser.js rename test/{resolve => core}/resolverFactory.js (96%) rename test/{resolve => core}/resolvers/fsResolver.js (99%) rename test/{resolve => core}/resolvers/gitFsResolver.js (99%) rename test/{resolve => core}/resolvers/gitRemoteResolver.js (98%) rename test/{resolve => core}/resolvers/gitResolver.js (97%) rename test/{resolve => core/resolvers}/resolver.js (94%) rename test/{resolve => core}/resolvers/urlResolver.js (96%) rename test/{resolve => core}/worker.js (99%) diff --git a/.gitignore b/.gitignore index 860be41b..1743854d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ node_modules +npm-debug.log + test/assets/github-test-package test/assets/github-test-package-copy test/assets/temp diff --git a/NOTES.md b/NOTES.md index 78ee4dd0..6395030c 100644 --- a/NOTES.md +++ b/NOTES.md @@ -44,11 +44,19 @@ - bower could setup a git hook on folders that are github repos to make validation of the json (if it conforms with the spec) - in prod dont forget to Q.longStackJumpLimit = 0; - add perf tests + - http://trace.gl - url resolver should work with fonts, e.g.: http://fonts.googleapis.com/css?family=Noto+Serif - discuss ability to specify folders inside bower_components.. e.g. components/fonts/ - discuss namespaces in the registry - cache dir location: https://github.com/bower/bower/issues/448 +- implement shrinkwrap? +- switch everything related with fs. to .graceful-fs + - don't forget to do the same on all bower org modules +- use dependency injection more? we are passing a lot of options around for deep arch components.. Not BC changes: - shorthand_resolver -> shorthandResolver - shorthand resolver syntax {{{}}} to just {{}} +- "latest" targets are no longer supported, they might conflict with branches or tags +- remove json property from the config +- remove read of local .bowerrc in favor of bower.json config key diff --git a/README.md b/README.md index 3a6a659c..a3d01b97 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Main issues are: - **Endpoint:** name|source#target - **Decomposed endpoint:** An object containing the `name`, `source` and `target` keys. - **Components folder:** The folder in which components are installed (`bower_components` by default). -- **Package meta:** A data structure similar to the one found in `bower.json`, which might also contain additional information. This is usually stored in a `.bower.json` file, inside a canonical package. +- **Package meta:** A data structure similar to the one found in `bower.json`, which might also contain additional information. This is stored in a `.bower.json` file, inside a canonical package. ### Overall strategy @@ -53,15 +53,15 @@ Bower is composed of the following components: - `.bowerrc`: Allows for customisations of Bower behaviour at the project/user level. - `bower.json`: Main purpose is to declare the component dependencies and other component related information. - `Manager`: Main coordinator, responsible for: - - Checking which packages are already installed in the current `bower folder`. - Deciding which version of the dependencies should be fetched from the `PackageRepository`, while keeping every dependant compatible (note that the `Manager` is `semver` aware). - Tracking which dependencies have been fetched, which ones failed to fetch, and which ones are being fetched. - - Requesting the `PackageRepository` to fail-fast, in case it realises there is no resolution for the current dependency tree. + - Expanding the dependency tree, analysing the dependencies of each fetched package. - `PackageRepository`: Abstraction to the underlying complexity of heterogeneous source types. Responsible for: + - Collecting concrete `Resolver`s for each endpoint + - Querying the `Resolve` cache for already resolved packages of the same target + - Decide if the cached package can be used. - Storing new entries in `ResolveCache`. - - Queueing resolvers into the `Worker`, if no suitable entry is found in the `ResolveCache`. - `ResolveCache`: Keeps a cache of previously resolved endpoints. Lookup can be done using an endpoint. -- `Worker`: A service responsible for limiting amount of parallel executions of tasks of the same type. - `ResolverFactory`: Parses an endpoint and returns a `Resolver` capable of resolving the source type. - `Resolver`: Base resolver, which can be extended by concrete resolvers, like `UrlResolver`, `GitRemoteResolver`, etc. @@ -113,28 +113,68 @@ Here's an overview of the dependency resolve process: #### Manager +Main resolve coordinator. + + +##### Constructor + +`Manager(options)` + +Available options: + +- `force` - true to force fetch remote sources (e.g.: bypass registry cache, defaults to false) +- `offline` - true to not fetch remote sources and use only the cache (defaults to false) +- `config` - the config to use (defaults to the global config) + +Note that `force` and `offline` are mutually exclusive. + +##### Public methods + +`Manager#configure(targets, resolved)`: Promise + +Configures the manager with an array of *decomposed endpoint*s (`targets`) and +an array of *decomposed endpoint*s that are considered `resolved` (optional). + +If the Manager is already resolving, the promise is immediately rejected. + +`Manager#resolve()`: Promise + +Starts the resolve promise, returning a promise of an object which keys are package names and +values the associated resolve info (decomposed endpoints plus package meta and other info). + +If the Manager is already resolving, the promise is immediately rejected. + +`Manager#areCompatible(source, subject)`: Boolean + TODO #### PackageRepository +Abstraction to the underlying complexity of heterogeneous source types + + ##### Constructor -`PackageRepository()` +`PackageRepository(options)` + +Available options: + +- `force` - true to force fetch remote sources (e.g.: bypass registry cache, resolve cache, defaults to false) +- `offline` - true to not fetch remote sources and use only the cache (defaults to false) +- `config` - the config to use (defaults to the global config) + +Note that `force` and `offline` are mutually exclusive. ##### Public methods -`PackageRepository#get(decEndpoint)`: Promise +`PackageRepository#fetch(decEndpoint)`: Promise Enqueues an decomposed endpoint to be fetched, and returns a promise of a *canonical package*. -`PackageRepository#abort()`: Promise +`PackageRepository#empty(name)`: Promise -Aborts any queued package lookup as soon as possible, and returns a promise that everything has been aborted. - -##### Protected methods - -*CONTINUE HERE* +Empties any resolved cache for package `name` or all the resolved cache if no `name` is passed. #### ResolverFactory @@ -148,12 +188,46 @@ function createResolver(decEndpoint, options) -> Promise The function is async to allow querying the Bower registry, etc. Options: -- `skipCache` - true to not use cache (e.g.: bypass registry cache, defaults to false) +- `force` - true to force fetch remote sources (e.g.: bypass registry cache, defaults to false) +- `offline` - true to not fetch remote sources and use only the cache (defaults to false) - `config` - the config to use (defaults to the global config) +Note that `force` and `offline` are mutually exclusive. + + #### ResolveCache -TODO +The cache, stored in disk, of resolved packages (canonical packages). + +##### Constructor + +`ResolveCache(cacheDir, options)` + +TODO: options, such as max size in MB, etc + +------------ + +##### Public functions + +`ResolveCache#retrieve(source, target)`: Promise + +Retrieves *canonical package* for a given `source` and `target` (optional, defaults to `*`). +The promise is resolved with both the *canonical package* and *package meta*. + +`ResolveCache#store(canonicalPackage, pkgMeta)`: Promise + +Stores `canonicalPackage` into the cache. +The `pkgMeta` is optional and will be read if not passed. + +`ResolveCache#eliminate(source, version)`: Promise + +Eliminates entry with given `source` and `version` from the cache. +Note that `version` can be empty because some *canonical package*s do not have a version associated. +In that case, only the unversioned entry will be removed. + +`ResolveCache#empty(source)`: Promise + +Eliminates *canonical package*s that match the `source` or everything if `source` is not passed. #### Resolver @@ -303,6 +377,8 @@ A worker responsible for limiting execution of parallel tasks. The number of parallel tasks may be limited and configured per type. This component will be a service that can be accessed to perform tasks. +*NOTE*: This component is not being used YET + ------------ #### Constructor diff --git a/lib/config.js b/lib/config.js index cc6796d3..e2781fe8 100644 --- a/lib/config.js +++ b/lib/config.js @@ -1,73 +1,69 @@ var path = require('path'); -var fs = require('fs'); +var os = require('os'); var mout = require('mout'); -var mkdirp = require('mkdirp'); var rc = require('rc'); // Guess some needed properties based on the user OS -var temp = process.env.TMPDIR - || process.env.TMP - || process.env.TEMP - || process.platform === 'win32' ? 'c:\\windows\\temp' : '/tmp'; +var temp = os.tmpdir ? os.tmpdir() : os.tmpDir(); var home = (process.platform === 'win32' ? process.env.USERPROFILE : process.env.HOME) || temp; var roaming = process.platform === 'win32' - ? path.join(path.resolve(process.env.APPDATA || home || temp), 'bower') - : path.join(path.resolve(home || temp), '.bower'); + ? path.join(path.resolve(process.env.APPDATA || home || temp), 'bower_new') // TODO: change this to bower before release + : path.join(path.resolve(home || temp), '.bower_new'); // TODO: change this to bower before release // Guess proxy defined in the env -var proxy = process.env.HTTPS_PROXY +var proxy = process.env.HTTP_PROXY + || process.env.http_proxy || null; + +var httpsProxy = process.env.HTTPS_PROXY || process.env.https_proxy || process.env.HTTP_PROXY - || process.env.http_proxy; + || process.env.http_proxy + || null; // ----------- -// Read global bower config -var config; +// TODO: there are some options that are not yet being applied in the codebase + +// Read rc +var rc; try { - config = rc('bower', { - directory: 'bower_components', - shorthandResolver: 'git://github.com/{{owner}}/{{package}}.git', - proxy: proxy, - roaming: roaming, - cwd: process.cwd() + rc = rc('bower', { + 'cwd': process.cwd(), + 'directory': 'bower_components', + 'registry': 'https://bower.herokuapp.com', + 'shorthand-resolver': 'git://github.com/{{owner}}/{{package}}.git', + 'roaming': roaming, + 'tmp': temp, + 'proxy': proxy, + 'https-proxy': httpsProxy, + 'ca': null, + 'strict-ssl': true, + 'user-agent': 'node/' + process.version + ' ' + process.platform + ' ' + process.arch, + 'color': true, + 'git': 'git' }); } catch (e) { - throw new Error('Unable to parse global .bowerrc file: ' + e.message); + throw new Error('Unable to parse runtime configuration: ' + e.message); } -// Merge global with local bower config -var localConfig = path.join(config.cwd, '.bowerrc'); -try { - localConfig = fs.readFileSync(localConfig); - try { - mout.object.mixIn(config, JSON.parse(localConfig)); - } catch (e) { - throw new Error('Unable to parse local .bowerrc file: ' + e.message); - } -} catch (e) {} +// Generate config based on the rc, making every key camelCase +var config = {}; +mout.object.forOwn(rc, function (value, key) { + key = key.replace(/_/g, '-'); // For backwards compatibility + config[mout.string.camelCase(key)] = value; +}); // Create some aliases to be used internally mout.object.mixIn(config, { _cache: path.join(config.roaming, 'cache'), _links: path.join(config.roaming, 'links'), _completion: path.join(config.roaming, 'completion'), + _registry: path.join(config.roaming, 'registry'), _gitTemplate: path.join(config.roaming, 'git_template') }); -// ----------- - -// Make sure that we have our git template directory -// The git template directory is an empty dir that will be set up for every git command -// So that the user git hooks won't be used -try { - mkdirp.sync(config._gitTemplate); -} catch (e) { - throw new Error('Unable to create git_template directory: ' + e.message); -} - module.exports = config; diff --git a/lib/core/Manager.js b/lib/core/Manager.js new file mode 100644 index 00000000..e7ef28f5 --- /dev/null +++ b/lib/core/Manager.js @@ -0,0 +1,337 @@ +var Q = require('q'); +var mout = require('mout'); +var semver = require('semver'); +var PackageRepository = require('./PackageRepository'); +var defaultConfig = require('../config'); +var createError = require('../util/createError'); +var endpointParser = require('../util/endpointParser'); + +var Manager = function (options) { + options = options || {}; + + this._config = options.config || defaultConfig; + this._repository = new PackageRepository(options); +}; + +Manager.prototype.configure = function (targets, resolved) { + // If working, error out + if (this._working) { + throw createError('Can\'t configure while resolving', 'EWORKING'); + } + + // Reset stuff + this._targets = {}; + this._resolved = {}; + + // Parse targets + targets.forEach(function (decEndpoint) { + this._targets[decEndpoint.name] = decEndpoint; + }.bind(this)); + + // Set resolved based on the passed endpoints + if (resolved) { + resolved.forEach(function (decEndpoint) { + // Only accept resolved endpoints with a name + if (!decEndpoint.name) { + throw createError('Name must be set when configuring resolved endpoints'); + } + this._resolved[decEndpoint.name] = [decEndpoint]; + decEndpoint.initial = true; + }, this); + } + + return this; +}; + +Manager.prototype.resolve = function () { + // If already resolving, error out + if (this._working) { + return Q.reject(createError('Already resolving', 'EWORKING')); + } + + // Reset stuff + this._fetching = {}; + this._nrFetching = 0; + this._failed = {}; + + this._deferred = Q.defer(); + + // Foreach endpoint, fetch it from the repository + mout.object.forOwn(this._targets, this._fetch.bind(this)); + + return this._deferred.promise + .fin(function () { + this._working = false; + }.bind(this)); +}; + +Manager.prototype.areCompatible = function (source, subject) { + var validSource = semver.valid(source.target) != null; + var validSubject = semver.valid(subject.target) != null; + var validRangeSource = semver.validRange(source.target) != null; + var validRangeSubject = semver.validRange(subject.target) != null; + + var highestSubject; + var highestSource; + + // Version -> version + if (validSource && validSubject) { + return semver.eq(source.target, subject.target); + } + + // Range -> version + if (validRangeSource && validSubject) { + return semver.satisfies(subject.target, source.target); + } + + // Version -> Range + if (validSource && validRangeSubject) { + return semver.satisfies(source.target, subject.target); + } + + // Range -> Range + if (validRangeSource && validRangeSubject) { + // Special case which both targets are * + if (source.target === '*' && subject.target === '*') { + return true; + } + + // Grab the highest version possible for both + highestSubject = this._findHighestVersion(semver.toComparators(subject.target)); + highestSource = this._findHighestVersion(semver.toComparators(source.target)); + + // Check if the highest resolvable version for the + // subject is the same as the source one + return semver.eq(highestSubject, highestSource); + } + + // Otherwise check if both targets are the same + return source.target === subject.target; +}; + +// ----------------- + +Manager.prototype._fetch = function (decEndpoint) { + var name = decEndpoint.name; + + // Mark as being fetched + this._fetching[name] = this._fetching[name] || []; + this._fetching[name].push(decEndpoint); + this._nrFetching++; + + // Fetch it from the repository + // Note that the promise is stored in the decomposed endpoint + // because it might be reused if a similar endpoint needs to be resolved + decEndpoint.promise = this._repository.fetch(decEndpoint) + // When done, call onFetch + .spread(this._onFetch.bind(this, decEndpoint)) + // Listen to progress to proxy them to the resolve deferred + // Note that we mark where the notification is coming from + .progress(function (notification) { + notification.endpoint = decEndpoint; + this._deferred.notify(notification); + }.bind(this)); + + return decEndpoint.promise; +}; + +Manager.prototype._onFetch = function (decEndpoint, canonicalPkg, pkgMeta) { + var json; + var name; + var resolved; + var index; + var initialName = decEndpoint.name; + + // Remove from being fetched list + mout.array.remove(this._fetching[initialName], decEndpoint); + this._nrFetching--; + + // Set the name, dir, json property in the decomposed endpoint + decEndpoint.dir = canonicalPkg; + decEndpoint.name = name = decEndpoint.name || pkgMeta.name; + decEndpoint.json = json = pkgMeta; + + // Add to the resolved list, marking it as resolved + resolved = this._resolved[name] = this._resolved[name] || []; + resolved.push(decEndpoint); + delete decEndpoint.promise; + + // If the fetched package was an initial target and had no name, + // we need to remove initially resolved ones that match the new name + if (!initialName) { + index = mout.array.findIndex(resolved, function (decEndpoint) { + return decEndpoint.initial; + }); + + if (index !== -1) { + resolved.splice(index, 1); + } + } + + // Parse dependencies + this._parseDependencies(decEndpoint, json); + + // If the resolve process ended, parse the resolved packages + // to find the most suitable version for each package + if (this._nrFetching <= 0) { + process.nextTick(this._finish.bind(this)); + } +}; + +Manager.prototype._parseDependencies = function (decEndpoint, json) { + console.log('fetched', decEndpoint.name); + + // Parse package dependencies + mout.object.forOwn(json.dependencies, function (endpoint, name) { + var decEndpoints; + var compatible; + var childDecEndpoint = endpointParser.decompose(endpoint); + + // Check if source is a semver version/range + // If so, the endpoint is probably a registry entry + if (semver.valid(childDecEndpoint.source) != null || semver.validRange(childDecEndpoint.source) != null) { + childDecEndpoint.target = childDecEndpoint.source; + childDecEndpoint.source = name; + } + + // Ensure name of the endpoint based on the key + childDecEndpoint.name = name; + + // Check if a compatible one is already resolved + // If there's one, we don't need to resolve it twice + decEndpoints = this._resolved[name]; + if (decEndpoints) { + compatible = mout.array.find(decEndpoints, function (resolved) { + return this.areCompatible(resolved, childDecEndpoint); + }, this); + + // Simply mark it as resolved + if (compatible) { + childDecEndpoint.dir = compatible.dir; + childDecEndpoint.json = compatible.json; + this._resolved[name].push(childDecEndpoint); + return; + } + } + + // Check if a compatible one is being fetched + // If there's one, we reuse it to avoid resolving it twice + decEndpoints = this._fetching[name]; + if (decEndpoints) { + compatible = mout.array.find(decEndpoints, function (beingFetched) { + return this.areCompatible(beingFetched, childDecEndpoint); + }, this); + + // Wait for it to resolve and then add it to the resolved packages + if (compatible) { + childDecEndpoint = compatible.promise.then(function () { + childDecEndpoint.dir = compatible.dir; + childDecEndpoint.json = compatible.json; + this._resolved[name].push(childDecEndpoint); + }.bind(this)); + + return; + } + } + + // Otherwise, just fetch it from the repository + console.log('will fetch', name); + this._fetch(childDecEndpoint); + }, this); +}; + +Manager.prototype._finish = function () { + var parsed = {}; + + mout.object.forOwn(this._resolved, function (decEndpoints, name) { + var configured = this._targets[name]; + var nonSemver; + var validSemver; + var suitable; + + // If this was initially configured without a valid semver target, + // the user wants it, regardless of other ones + if (configured && configured.target && !semver.valid(configured.target)) { + parsed[name] = this._targets[name]; + // TODO: issue warning + return; + } + + // Filter non-semver ones + nonSemver = decEndpoints.filter(function (decEndpoint) { + return !decEndpoint.json.version; + }); + + // Filter semver ones + validSemver = decEndpoints.filter(function (decEndpoint) { + return !!decEndpoint.json.version; + }); + + // Sort semver ones + validSemver.sort(function (first, second) { + if (semver.gt(first, second)) { + return -1; + } else if (semver.lt(first, second)) { + return 1; + } else { + return 0; + } + }); + + // If there are no semver targets + if (!validSemver.length) { + // TODO: if various non-semver were found, resolve conflicts + suitable = nonSemver[0]; + // Otherwise, find most suitable semver + } else { + // TODO: handle conflicts if there is no suitable version + suitable = mout.array.find(validSemver, function (subject) { + return validSemver.every(function (decEndpoint) { + return semver.satisfies(subject.json.version, decEndpoint.target); + }); + }); + } + + // TODO: handle case which there is a suitable version but there are no-semver ones too + + if (suitable) { + parsed[name] = suitable; + } else { + throw new Error('No suitable version for "' + name + '"'); + } + }, this); + + this._deferred.resolve(parsed); +}; + +Manager.prototype._findHighestVersion = function (comparators) { + var highest; + var matches; + var version; + + comparators.forEach(function (comparator) { + // Get version of this comparator + // If it's an array, call recursively + if (Array.isArray(comparator)) { + version = this._findHighestVersion(comparator); + // Otherwise extract the version from the comparator + // using a simple regexp + } else { + matches = comparator.match(/\d+\.\d+\.\d+.*$/); + if (!matches) { + return; + } + + version = matches[0]; + } + + // Compare with our know highest version + if (!highest || semver.gt(version, highest)) { + highest = version; + } + }, this); + + return highest; +}; + +module.exports = Manager; \ No newline at end of file diff --git a/lib/core/PackageRepository.js b/lib/core/PackageRepository.js new file mode 100644 index 00000000..bcb4206b --- /dev/null +++ b/lib/core/PackageRepository.js @@ -0,0 +1,91 @@ +var mout = require('mout'); +var RegistryClient = require('bower-registry-client'); +var ResolveCache = require('./ResolveCache'); +var resolverFactory = require('./resolverFactory'); +var defaultConfig = require('../config'); + +var PackageRepository = function (options) { + options = options || {}; + + this._options = options; + this._config = options.config || defaultConfig; + + // Instantiate the registry and store it in the options object + // because it will be passed to the resolver factory + this._options.registry = new RegistryClient(mout.object.fillIn({ + cache: this._config._registry + }, options.config)); + + this._cache = new ResolveCache(this._config._cache); +}; + +// ----------------- + +PackageRepository.prototype.fetch = function (decEndpoint) { + var resolver; + + // Get the appropriate resolver + return resolverFactory(decEndpoint, this._options) + // Retrieve from the resolve cache + .then(function (res) { + resolver = res; + + // If force flag is used, bypass cache + if (this._options.force) { + return []; + } + + // Note that we use the resolver methods to query the + // cache because transformations/normalisations can occur + return this._cache.retrieve(resolver.getSource(), resolver.getTarget()); + }.bind(this)) + // Decide if we can use the one from the resolve cache + .spread(function (canonicalPkg, pkgMeta) { + // If there's no package in the cache, resolve it + if (!canonicalPkg) { + return this._resolve(resolver); + } + + // If offline flag is used, use directly the cached one + if (this._options.offline) { + return [canonicalPkg, pkgMeta]; + } + + // Otherwise check for new contents + return resolver.hasNew(canonicalPkg, pkgMeta) + .then(function (hasNew) { + // If there are no new contents, resolve to + // the cached one + if (!hasNew) { + return [canonicalPkg, pkgMeta]; + } + + // Otherwise resolve to the newest one + return this._resolve(resolver); + }); + }.bind(this)); +}; + +PackageRepository.prototype.empty = function (name) { + // TODO Think of a way to remove specific packages of a given name from the cache + // Since the ResolveCache.empty only works with source, one possible solution is to implement + // a forEach method that calls a function with the canonicalPackage and the pkgMeta + // so that we can match against the pkgMeta.name and call ResolveCache.empty with it +}; + +// --------------------- + +PackageRepository.prototype._resolve = function (resolver) { + // Resolve the resolver + return resolver.resolve() + // Store in the cache + .then(function (canonicalPkg) { + return this._cache.store(canonicalPkg, resolver.getPkgMeta()); + }.bind(this)) + // Resolve promise with canonical package and package meta + .then(function () { + return [resolver.getTempDir(), resolver.getPkgMeta()]; + }.bind(this)); +}; + +module.exports = PackageRepository; \ No newline at end of file diff --git a/lib/core/Project.js b/lib/core/Project.js new file mode 100644 index 00000000..1200b02f --- /dev/null +++ b/lib/core/Project.js @@ -0,0 +1,274 @@ +var glob = require('glob'); +var path = require('path'); +var fs = require('fs'); +var Q = require('q'); +var mout = require('mout'); +var mkdirp = require('mkdirp'); +var rimraf = require('rimraf'); +var bowerJson = require('bower-json'); +var semver = require('semver'); +var Manager = require('./Manager'); +var defaultConfig = require('../config'); +var createError = require('../util/createError'); +var endpointParser = require('../util/endpointParser'); +var copy = require('../util/copy'); + +var Project = function (options) { + options = options || {}; + + this._config = options.config || defaultConfig; + this._manager = new Manager(options); +}; + +Project.prototype.install = function (endpoints) { + // If already working, error out + if (this._working) { + return Q.reject(createError('Already working', 'EWORKING')); + } + + // If an empty array was passed, null it out + if (endpoints && !endpoints.length) { + endpoints = null; + } + + // Collect local, json and specified endpoints + return Q.all([ + this._collectLocal(), + this._collectFromJson(), + this._collectFromEndpoints(endpoints) + ]) + .spread(function (locals, jsons, endpoints) { + var toBeResolved = []; + var resolved = []; + + // If endpoints were passed + if (endpoints) { + // Mark each of the endpoint to be resolved + mout.object.forOwn(endpoints, function (decEndpoint) { + toBeResolved.push(decEndpoint); + }, this); + + // Mark locals as resolved if they were not specified as endpoints + // and if they are specified in the jsons + mout.object.forOwn(locals, function (decEndpoint) { + if (jsons[decEndpoint.name] && !endpoints[decEndpoint.name]) { + resolved.push(decEndpoint); + } + }, this); + // Otherwise use jsons + } else { + // Mark jsons to be resolved if they are not installed + // Even if they are installed, its semver must match + // against the installed ones + mout.object.forOwn(jsons, function (decEndpoint) { + var local = locals[decEndpoint.name]; + + if (!local || !this._manager.areCompatible(local, decEndpoint)) { + toBeResolved.push(decEndpoint); + } else { + resolved.push(local); + } + }, this); + } + + // Configure the manager with the targets and resolved + // endpoints + this._manager.configure(toBeResolved, resolved); + + console.log('--------------------------'); + console.log('> resolved...'); + console.log('--------------------------'); + resolved.forEach(function (decEndpoint) { + console.log(decEndpoint.source, decEndpoint.target); + }); + console.log('--------------------------'); + console.log('> to be resolved...'); + console.log('--------------------------'); + toBeResolved.forEach(function (decEndpoint) { + console.log(decEndpoint.source, decEndpoint.target); + }); + console.log('--------------------------'); + console.log('> resolving...'); + console.log('--------------------------'); + + // Kick in the resolve process + return this._manager.resolve(); + }.bind(this)) + .then(this._copyResolved.bind(this)) + .fin(function () { + this._working = false; + }.bind(this)); +}; + +Project.prototype.update = function (names, options) { + +}; + +Project.prototype.uninstall = function (names, options) { + +}; + +Project.prototype.list = function (options) { + +}; + +// ----------------- + +Project.prototype._copyResolved = function (decEndpoints) { + var destDir = path.join(this._config.cwd, this._config.directory); + + return Q.nfcall(mkdirp, destDir) + .then(function () { + var promises = []; + + console.log('--------------------------------------'); + + mout.object.forOwn(decEndpoints, function (decEndpoint) { + var promise; + var dest; + + console.log('will install', decEndpoint.name, decEndpoint.json.version); + + // Do not copy if already installed (local) + if (decEndpoint.local) { + return; + } + + dest = path.join(destDir, decEndpoint.name); + + // Remove existent + promise = Q.nfcall(rimraf, dest) + // Copy dir + .then(copy.copyDir.bind(copy, decEndpoint.dir, dest)); + + promises.push(promise); + + }); + + return Q.all(promises); + }); +}; + +Project.prototype._collectFromJson = function () { + var deferred = Q.defer(); + + // Read local json + Q.nfcall(bowerJson.find, this._config.cwd) + .then(function (filename) { + // If it is a component.json, warn about the deprecation + if (path.basename(filename) === 'component.json') { + deferred.notify({ + type: 'warn', + data: 'You are using the deprecated component.json file' + }); + } + + // Read it + return Q.nfcall(bowerJson.read, filename) + .fail(function (err) { + throw createError('Something went wrong while reading "' + filename + '"', err.code, { + details: err.message + }); + }); + }.bind(this), function () { + // No json file was found, assume one + return Q.nfcall(bowerJson.parse, { name: this._name }); + }) + // For each dependency, decompose the endpoint, generating + // an object which keys are package names and values the decomposed + // endpoints + .then(function (json) { + var name; + var decEndpoint; + var decEndpoints = {}; + + for (name in json.dependencies) { + decEndpoint = endpointParser.decompose(json.dependencies[name]); + + // Check if source is a semver version/range + // If so, the endpoint is probably a registry entry + if (semver.valid(decEndpoint.source) != null || semver.validRange(decEndpoint.source) != null) { + decEndpoint.target = decEndpoint.source; + decEndpoint.source = name; + } + + // Ensure name of the endpoint based on the key + decEndpoint.name = name; + + decEndpoints[name] = decEndpoint; + } + + return decEndpoints; + }) + .then(deferred.resolve, deferred.reject, deferred.notify); + + return deferred.promise; +}; + +Project.prototype._collectFromEndpoints = function (endpoints) { + var decEndpoints; + + if (!endpoints) { + return Q.resolve(); + } + + decEndpoints = {}; + endpoints.forEach(function (endpoint) { + var decEndpoint = endpointParser.decompose(endpoint); + decEndpoints[decEndpoint.name] = decEndpoint; + }); + + return Q.resolve(decEndpoints); +}; + +Project.prototype._collectLocal = function () { + var componentsDir = path.join(this._config.cwd, this._config.directory); + + // Gather all folders that are actual packages by + // looking for the package metadata file + return Q.nfcall(glob, '*/.bower.json', { + cwd: componentsDir, + dot: true + }) + .then(function (filenames) { + var promises = []; + + // Foreach bower.json found + filenames.forEach(function (filename) { + var promise; + var name = path.dirname(filename); + + // Read package metadata + promise = Q.nfcall(fs.readFile, path.join(componentsDir, filename)) + .then(function (contents) { + var json = JSON.parse(contents.toString()); + + // Set decomposed endpoint manually + return { + name: name, + source: path.join(componentsDir, name), + target: json.version || '*', + json: json, + local: true + }; + }); + + promises.push(promise); + }); + + // Wait until all files have been read + // to form the final object of decomposed endpoints + return Q.all(promises) + .then(function (locals) { + var decEndpoints = {}; + + locals.forEach(function (decEndpoint) { + decEndpoints[decEndpoint.name] = decEndpoint; + }); + + return decEndpoints; + }); + }); +}; + +module.exports = Project; \ No newline at end of file diff --git a/lib/core/ResolveCache.js b/lib/core/ResolveCache.js new file mode 100644 index 00000000..e5457f8c --- /dev/null +++ b/lib/core/ResolveCache.js @@ -0,0 +1,184 @@ +var crypto = require('crypto'); +var fs = require('fs'); +var path = require('path'); +var semver = require('semver'); +var mout = require('mout'); +var Q = require('q'); +var mkdirp = require('mkdirp'); + +var ResolveCache = function (dir) { + // TODO: Make some options, such as: + // - Max MB + // - Max versions per source + // - Max MB per source + // - etc.. + this._dir = dir; + this._versions = {}; + + mkdirp.sync(dir); +}; + +ResolveCache.prototype.retrieve = function (source, target) { + var sourceId = this._getSourceId(source); + var dir = path.join(this._dir, sourceId); + + target = target || '*'; + + return this._getVersions(source) + .then(function (versions) { + var suitable; + + console.log('cached versions for', source, versions); + // If target is a semver, find a suitable version + if (semver.valid(target) != null || semver.validRange(target) != null) { + suitable = mout.array.find(versions, function (version) { + return semver.satisfies(version, target); + }); + + if (suitable) { + return suitable; + } + } + + // If target is '*' check if there's a cached '_unversioned' + if (target === '*') { + return mout.array.find(versions, function (version) { + return version === '_unversioned'; + }); + } + + // Otherwise check if there's an exact match + return mout.array.find(versions, function (version) { + return version === target; + }); + }) + .then(function (version) { + var canonicalPkg; + + if (!version) { + console.log('no cached package', source, target); + return []; + } + + // Resolve with canonical package and package meta + canonicalPkg = path.join(dir, version); + return this._readPkgMeta(canonicalPkg) + .then(function (pkgMeta) { + return [canonicalPkg, pkgMeta]; + }); + }.bind(this)); +}; + +ResolveCache.prototype.store = function (canonicalPkg, pkgMeta) { + var promise = pkgMeta ? Q.resolve(pkgMeta) : this._readPkgMeta(canonicalPkg); + var sourceId; + var pkgVersion; + + return promise + .then(function (pkgMeta) { + var dir; + + sourceId = this._getSourceId(pkgMeta._source); + pkgVersion = pkgMeta.version || '_unversioned'; + dir = path.join(this._dir, sourceId, pkgVersion); + + // Create sourceId directory + return Q.nfcall(mkdirp, path.dirname(dir)) + // Move the canonical to sourceId/target + .then(function () { + return Q.nfcall(fs.rename, canonicalPkg, dir); + }); + }.bind(this)) + .then(function () { + var pkgVersion = pkgMeta.version || '_unversioned'; + var versions = this._versions[sourceId]; + var inCache; + + // Check if this exact version already exists in cache + inCache = versions && versions.some(function (version) { + return pkgVersion === version; + }); + + // If it doesn't, add it to the in memory cache + // and sort the versions afterwards + if (!inCache) { + versions.push(pkgVersion); + this._sortVersions(versions); + } + }.bind(this)); +}; + +ResolveCache.prototype.eliminate = function (source, version) { + // TODO: +}; + +ResolveCache.prototype.empty = function (source) { + // TODO: +}; + +// ------------------------ + +ResolveCache.prototype._getSourceId = function (source) { + return crypto.createHash('md5').update(source).digest('hex'); +}; + + +ResolveCache.prototype._readPkgMeta = function (dir) { + return Q.nfcall(fs.readFile, path.join(dir, '.bower.json')) + .then(function (contents) { + return JSON.parse(contents.toString()); + }); +}; + +ResolveCache.prototype._getVersions = function (source) { + var dir; + var sourceId = this._getSourceId(source); + var cache = this._versions[sourceId]; + + if (cache) { + return Q.resolve(cache); + } + + dir = path.join(this._dir, sourceId); + return Q.nfcall(fs.readdir, dir) + .then(function (versions) { + this._sortVersions(versions); + return this._versions[sourceId] = versions; + }.bind(this), function (err) { + // If the directory does not exists, resolve + // as an empty array + if (err.code === 'ENOENT') { + return this._versions[sourceId] = []; + } + + throw err; + }.bind(this)); +}; + +ResolveCache.prototype._sortVersions = function (versions) { + versions.sort(function (version1, version2) { + var validSemver1 = semver.valid(version1) != null; + var validSemver2 = semver.valid(version2) != null; + + // If both are semvers, compare them + if (validSemver1 && validSemver2) { + if (semver.gt(validSemver1, validSemver2)) { + return -1; + } else if (semver.lt(validSemver1, validSemver2)) { + return 1; + } else { + return 0; + } + // If one of them are semvers, give higher priority + } else if (validSemver1) { + return -1; + } else if (validSemver2) { + return 1; + } + + // Otherwise they are considered equal + return 0; + }); +}; + +module.exports = ResolveCache; \ No newline at end of file diff --git a/lib/resolve/Worker.js b/lib/core/Worker.js similarity index 100% rename from lib/resolve/Worker.js rename to lib/core/Worker.js diff --git a/lib/resolve/resolverFactory.js b/lib/core/resolverFactory.js similarity index 60% rename from lib/resolve/resolverFactory.js rename to lib/core/resolverFactory.js index fec43b51..27b465d4 100644 --- a/lib/resolve/resolverFactory.js +++ b/lib/core/resolverFactory.js @@ -6,7 +6,7 @@ var GitFsResolver = require('./resolvers/GitFsResolver'); var GitRemoteResolver = require('./resolvers/GitRemoteResolver'); var FsResolver = require('./resolvers/FsResolver'); var UrlResolver = require('./resolvers/UrlResolver'); -var config = require('../config'); +var defaultConfig = require('../config'); var createError = require('../util/createError'); function createResolver(decEndpoint, options) { @@ -15,7 +15,7 @@ function createResolver(decEndpoint, options) { var resolvedPath; options = options || {}; - options.config = options.config || config; + options.config = options.config || defaultConfig; // Setup resolver options resOptions = { @@ -43,10 +43,17 @@ function createResolver(decEndpoint, options) { // Check if source is a git repository resolvedPath = path.resolve(options.config.cwd, source); + // Below we try a series of asyc tests to guess the type of resolver to use + // If a step was unable to guess the resolver, it throws an error + // If a step was able to guess the resolver, it resolves with a function + // That function returns a promise that will resolve with the concrete type, + // ready to be used return Q.nfcall(fs.stat, path.join(resolvedPath, '.git')) .then(function (stats) { if (stats.isDirectory()) { - return { resolver: GitFsResolver, source: resolvedPath }; + return function () { + return Q.resolve(new GitFsResolver(resolvedPath, resOptions)); + }; } throw new Error('Not a Git repository'); @@ -55,7 +62,9 @@ function createResolver(decEndpoint, options) { .fail(function () { return Q.nfcall(fs.stat, source) .then(function () { - return { resolver: FsResolver, source: resolvedPath }; + return function () { + return Q.resolve(new FsResolver(resolvedPath, resOptions)); + }; }); }) // If not, check if is a shorthand and expand it @@ -69,16 +78,34 @@ function createResolver(decEndpoint, options) { package: parts[1] }); - return { resolver: GitRemoteResolver, source: source }; + return function () { + return Q.resolve(new GitRemoteResolver(source, resOptions)); + }; } throw err; }) - // TODO: if not, check against the registry - // note that the registry should also have a persistent cache for offline usage + // As last resort, we try the registry + .fail(function (err) { + var registry = options.registry; + + if (!registry) { + throw err; + } + + return function () { + return Q.nfcall(registry.lookup.bind(registry), source, options) + .then(function (entry) { + // TODO: Handle entry.type.. for now it's only 'alias' + // When we got published packages, this needs to be adjusted + return new GitRemoteResolver(entry.url, resOptions); + }); + }; + }) + // If we got the func, simply call and return + .then(function (func) { + return func(); // Finally throw a meaningful error - .then(function (ConcreteResolver) { - return new ConcreteResolver.resolver(source, resOptions); }, function () { throw new createError('Could not find appropriate resolver for source "' + source + '"', 'ENORESOLVER'); }); diff --git a/lib/resolve/resolvers/FsResolver.js b/lib/core/resolvers/FsResolver.js similarity index 98% rename from lib/resolve/resolvers/FsResolver.js rename to lib/core/resolvers/FsResolver.js index a9ecc876..ad56277b 100644 --- a/lib/resolve/resolvers/FsResolver.js +++ b/lib/core/resolvers/FsResolver.js @@ -3,7 +3,7 @@ var fs = require('fs'); var path = require('path'); var mout = require('mout'); var Q = require('q'); -var Resolver = require('../Resolver'); +var Resolver = require('./Resolver'); var copy = require('../../util/copy'); var extract = require('../../util/extract'); var createError = require('../../util/createError'); diff --git a/lib/resolve/resolvers/GitFsResolver.js b/lib/core/resolvers/GitFsResolver.js similarity index 100% rename from lib/resolve/resolvers/GitFsResolver.js rename to lib/core/resolvers/GitFsResolver.js diff --git a/lib/resolve/resolvers/GitRemoteResolver.js b/lib/core/resolvers/GitRemoteResolver.js similarity index 100% rename from lib/resolve/resolvers/GitRemoteResolver.js rename to lib/core/resolvers/GitRemoteResolver.js diff --git a/lib/resolve/resolvers/GitResolver.js b/lib/core/resolvers/GitResolver.js similarity index 91% rename from lib/resolve/resolvers/GitResolver.js rename to lib/core/resolvers/GitResolver.js index d304b9ad..1c7c7f58 100644 --- a/lib/resolve/resolvers/GitResolver.js +++ b/lib/core/resolvers/GitResolver.js @@ -5,7 +5,7 @@ var semver = require('semver'); var chmodr = require('chmodr'); var rimraf = require('rimraf'); var mout = require('mout'); -var Resolver = require('../Resolver'); +var Resolver = require('./Resolver'); var createError = require('../../util/createError'); var GitResolver = function (source, options) { @@ -38,14 +38,27 @@ GitResolver.prototype._hasNew = function (pkgMeta) { }; GitResolver.prototype._resolve = function () { - return this._findResolution() + var deferred = Q.defer(); + + deferred.notify({ type: 'action', data: 'Finding resolution' }); + + this._findResolution() .then(function () { + deferred.notify({ type: 'action', data: 'Checking out' }); + return this._checkout() // Always run cleanup after checkout to ensure that .git is removed! // If it's not removed, problems might arrise when the "tmp" module attemps // to delete the temporary folder - .fin(this._cleanup.bind(this)); - }.bind(this)); + .fin(function () { + deferred.notify({ type: 'action', data: 'Cleaning up' }); + + this._cleanup(); + }.bind(this)); + }.bind(this)) + .then(deferred.resolve, deferred.reject, deferred.notify); + + return deferred.promise; }; // ----------------- @@ -62,8 +75,8 @@ GitResolver.fetchRefs = function (source) { // ----------------- GitResolver.prototype._findResolution = function (target) { - var self = this.constructor; var err; + var self = this.constructor; target = target || this._target; @@ -158,20 +171,23 @@ GitResolver.prototype._cleanup = function () { GitResolver.prototype._savePkgMeta = function (meta) { var deferred = Q.defer(); + var version; + + if (this._resolution.type === 'version') { + version = semver.clean(this._resolution.tag); - if (this._resolution.version) { // Warn if the package meta version is different than the resolved one - if (typeof meta.version === 'string' && meta.version !== this._resolution.version) { + if (typeof meta.version === 'string' && semver.neq(meta.version, version)) { process.nextTick(function (metaVersion) { deferred.notify({ type: 'warn', - data: 'Version declared in the json (' + metaVersion + ') is different than the resolved one (' + this._resolution.version + ')' + data: 'Version declared in the json (' + metaVersion + ') is different than the resolved one (' + version + ')' }); }.bind(this, meta.version)); } // Ensure package meta version is the same as the resolution - meta.version = this._resolution.version; + meta.version = version; } else { // If resolved to a target that is not a version, // remove the version from the meta diff --git a/lib/resolve/Resolver.js b/lib/core/resolvers/Resolver.js similarity index 87% rename from lib/resolve/Resolver.js rename to lib/core/resolvers/Resolver.js index 101ba052..25285e78 100644 --- a/lib/resolve/Resolver.js +++ b/lib/core/resolvers/Resolver.js @@ -4,9 +4,9 @@ var Q = require('q'); var tmp = require('tmp'); var mkdirp = require('mkdirp'); var bowerJson = require('bower-json'); -var config = require('../config'); -var createError = require('../util/createError'); -var removeIgnores = require('../util/removeIgnores'); +var defaultConfig = require('../../config'); +var createError = require('../../util/createError'); +var removeIgnores = require('../../util/removeIgnores'); tmp.setGracefulCleanup(); @@ -17,7 +17,7 @@ var Resolver = function (source, options) { this._target = options.target || '*'; this._name = options.name || path.basename(this._source); this._guessedName = !options.name; - this._config = options.config || config; + this._config = options.config || defaultConfig; }; // ----------------- @@ -43,6 +43,10 @@ Resolver.prototype.hasNew = function (canonicalPkg) { var promise; var metaFile; + // TODO: Change arguments to canonicalPkg, pkgMeta + // where pkgMeta is optional + // Change _hasNew to the same + // If already working, error out if (this._working) { return Q.reject(createError('Already working', 'EWORKING')); @@ -126,12 +130,12 @@ Resolver.prototype._resolve = function () { // ----------------- -Resolver.prototype._hasNew = function (pkgMeta, canonicalPkg) { +Resolver.prototype._hasNew = function (pkgMeta) { return Q.resolve(true); }; Resolver.prototype._createTempDir = function () { - var baseDir = path.join(tmp.tmpdir, 'bower'); + var baseDir = path.join(this._config.tmp, 'bower'); return Q.nfcall(mkdirp, baseDir) .then(function () { @@ -178,14 +182,9 @@ Resolver.prototype._readJson = function (dir) { Resolver.prototype._applyPkgMeta = function (meta) { // Check if name defined in the json is different - if (meta.name !== this._name) { - // If so and if the name was "guessed", assume the json name - if (this._guessedName) { - this._name = meta.name; - // Otherwise force the configured one - } else { - meta.name = this._name; - } + // If so and if the name was "guessed", assume the json name + if (meta.name !== this._name && this._guessedName) { + this._name = meta.name; } // Handle ignore property, deleting all files from the temporary directory @@ -202,7 +201,13 @@ Resolver.prototype._applyPkgMeta = function (meta) { }; Resolver.prototype._savePkgMeta = function (meta) { - var contents = JSON.stringify(meta, null, 2); + var contents; + + // Store original source + meta._source = this._source; + + // Stringify contents + contents = JSON.stringify(meta, null, 2); return Q.nfcall(fs.writeFile, path.join(this._tempDir, '.bower.json'), contents) .then(function () { diff --git a/lib/resolve/resolvers/UrlResolver.js b/lib/core/resolvers/UrlResolver.js similarity index 89% rename from lib/resolve/resolvers/UrlResolver.js rename to lib/core/resolvers/UrlResolver.js index 4714b3c0..36020e74 100644 --- a/lib/resolve/resolvers/UrlResolver.js +++ b/lib/core/resolvers/UrlResolver.js @@ -1,10 +1,11 @@ var util = require('util'); var path = require('path'); var fs = require('fs'); +var url = require('url'); var request = require('request'); var Q = require('q'); var mout = require('mout'); -var Resolver = require('../Resolver'); +var Resolver = require('./Resolver'); var extract = require('../../util/extract'); var createError = require('../../util/createError'); var junk = require('junk'); @@ -26,6 +27,8 @@ var UrlResolver = function (source, options) { this._name = path.basename(this._name.substr(0, pos)); } } + + this._remote = url.parse(source); }; util.inherits(UrlResolver, Resolver); @@ -42,9 +45,14 @@ UrlResolver.prototype._hasNew = function (pkgMeta) { reqHeaders['If-None-Match'] = oldCacheHeaders.ETag; } + if (this._config.userAgent) { + reqHeaders['User-Agent'] = this._config.userAgent; + } + // Make an HEAD request to the source return Q.nfcall(request.head, this._source, { - proxy: this._config.proxy, + proxy: this._remote.protocol === 'https:' ? this._config.httpsProxy : this._config.proxy, + strictSSL: this._config.strictSsl, timeout: 5000, headers: reqHeaders }) @@ -92,13 +100,20 @@ UrlResolver.prototype._resolve = function () { // ----------------- UrlResolver.prototype._download = function () { - var file = path.join(this._tempDir, this._name); + var file = path.join(this._tempDir, path.basename(this._source)); var deferred = Q.defer(); + var reqHeaders = {}; + + if (this._config.userAgent) { + reqHeaders['User-Agent'] = this._config.userAgent; + } // Download the file request(this._source, { - proxy: this._config.proxy, - timeout: 5000 + proxy: this._remote.protocol === 'https:' ? this._config.httpsProxy : this._config.proxy, + strictSSL: this._config.strictSsl, + timeout: 5000, + headers: reqHeaders }) .on('response', function (response) { this._response = response; diff --git a/lib/resolve/Manager.js b/lib/resolve/Manager.js deleted file mode 100644 index d1781fa3..00000000 --- a/lib/resolve/Manager.js +++ /dev/null @@ -1,25 +0,0 @@ -var config = require('../../config'); - -var Manager = function (options) { - options = options || {}; - - this._offline = !!options.offline; - this._config = options.config || config; -}; - -// ----------------- - -Manager.prototype.install = function (endpoints) { - this._packages = {}; - - // If some endpoints were passed, use those - // Otherwise grab the ones specified in the json - - // Check which packages are already installed - // and not install those if the target range is matched - - // Query the PackageRepository - // TODO -}; - -module.exports = Manager; \ No newline at end of file diff --git a/lib/resolve/PackageRepository.js b/lib/resolve/PackageRepository.js deleted file mode 100644 index bdb27803..00000000 --- a/lib/resolve/PackageRepository.js +++ /dev/null @@ -1,27 +0,0 @@ -var resolverFactory = require('./resolverFactory'); -var config = require('../../config'); - -var PackageRepository = function (options) { - options = options || {}; - - this._offline = !!options.offline; - this._force = !!options.force; - this._config = options.config || config; -}; - -// ----------------- - -PackageRepository.prototype.get = function (decEndpoint) { - return resolverFactory(decEndpoint, { - skipCache: this._force - }) - .then(function (resolver) { - return resolver.resolve(); - }); -}; - -PackageRepository.prototype.abort = function () { - // TODO -}; - -module.exports = PackageRepository; \ No newline at end of file diff --git a/lib/util/decomposeEndpoint.js b/lib/util/decomposeEndpoint.js deleted file mode 100644 index 5d7e132e..00000000 --- a/lib/util/decomposeEndpoint.js +++ /dev/null @@ -1,18 +0,0 @@ -var createError = require('./createError'); - -function decomposeEndpoint(endpoint) { - var regExp = /^(?:([\w\-]|(?:[\w\.\-]+[\w\-])?)\|)?([^\|#]+)(?:#(.*))?$/; - var matches = endpoint.match(regExp); - - if (!matches) { - throw createError('Invalid endpoint: "' + endpoint + '"', 'EINVEND'); - } - - return { - name: matches[1], - source: matches[2], - target: matches[3] - }; -} - -module.exports = decomposeEndpoint; \ No newline at end of file diff --git a/lib/util/endpointParser.js b/lib/util/endpointParser.js new file mode 100644 index 00000000..923fba1d --- /dev/null +++ b/lib/util/endpointParser.js @@ -0,0 +1,35 @@ +var createError = require('./createError'); + +function decompose(endpoint) { + var regExp = /^(?:([\w\-]|(?:[\w\.\-]+[\w\-])?)\|)?([^\|#]+)(?:#(.*))?$/; + var matches = endpoint.match(regExp); + + if (!matches) { + throw createError('Invalid endpoint: "' + endpoint + '"', 'EINVEND'); + } + + return { + name: matches[1] || '', + source: matches[2], + target: matches[3] || '*' + }; +} + +function compose(decEndpoint) { + var composed = ''; + + if (decEndpoint.name) { + composed += decEndpoint.name + '|'; + } + + composed += decEndpoint.source; + + if (decEndpoint.target) { + composed += '#' + decEndpoint.target; + } + + return composed; +} + +module.exports.decompose = decompose; +module.exports.compose = compose; \ No newline at end of file diff --git a/lib/util/extract.js b/lib/util/extract.js index 532ff1b4..26f44a2a 100644 --- a/lib/util/extract.js +++ b/lib/util/extract.js @@ -159,6 +159,11 @@ function extract(src, dest, opts) { // Extract archive promise = extractor(src, dest); + // TODO: There's an issue here if the src and dest are the same and + // The zip name is the same as some of the zip file contents + // Maybe create a temp directory inside dest, unzip it there, + // unlink zip and then move contents + // Remove archive if (!opts.keepArchive) { promise = promise diff --git a/package.json b/package.json index ae309f6a..c9ab1df6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "bower-json": "~0.0.0", + "bower-registry-client": "~0.0.0", "mout": "~0.4.0", "q": "~0.9.2", "tmp": "0.0.17", @@ -30,7 +31,8 @@ "tar": "~0.1.17", "fstream": "~0.1.22", "fstream-ignore": "0.0.6", - "junk": "~0.2.0" + "junk": "~0.2.0", + "glob": "~3.2.1" }, "devDependencies": { "mocha": "~1.8.2", @@ -41,7 +43,7 @@ "test": "node test/assets/downloader && mocha -R spec" }, "bin": { - "bower": "bin/bower" + "bower_new": "bin/bower" }, "preferGlobal": true, "private": true diff --git a/test/assets/test-temp-dir/test-exception.js b/test/assets/test-temp-dir/test-exception.js index 4cb53877..62c8b277 100644 --- a/test/assets/test-temp-dir/test-exception.js +++ b/test/assets/test-temp-dir/test-exception.js @@ -1,6 +1,6 @@ var fs = require('fs'); var path = require('path'); -var Resolver = require('../../../lib/resolve/Resolver'); +var Resolver = require('../../../lib/core/resolvers/Resolver'); var resolver = new Resolver('foo'); resolver._createTempDir() diff --git a/test/assets/test-temp-dir/test.js b/test/assets/test-temp-dir/test.js index 6f37fa38..0021c7ee 100644 --- a/test/assets/test-temp-dir/test.js +++ b/test/assets/test-temp-dir/test.js @@ -1,6 +1,6 @@ var fs = require('fs'); var path = require('path'); -var Resolver = require('../../../lib/resolve/Resolver'); +var Resolver = require('../../../lib/core/resolvers/Resolver'); var resolver = new Resolver('foo'); resolver._createTempDir() diff --git a/test/resolve/resolverFactory.js b/test/core/resolverFactory.js similarity index 96% rename from test/resolve/resolverFactory.js rename to test/core/resolverFactory.js index 978ecc76..d829104b 100644 --- a/test/resolve/resolverFactory.js +++ b/test/core/resolverFactory.js @@ -4,12 +4,12 @@ var path = require('path'); var mout = require('mout'); var Q = require('q'); var rimraf = require('rimraf'); -var config = require('../../lib/config'); -var resolverFactory = require('../../lib/resolve/resolverFactory'); -var FsResolver = require('../../lib/resolve/resolvers/FsResolver'); -var GitFsResolver = require('../../lib/resolve/resolvers/GitFsResolver'); -var GitRemoteResolver = require('../../lib/resolve/resolvers/GitRemoteResolver'); -var UrlResolver = require('../../lib/resolve/resolvers/UrlResolver'); +var defaultConfig = require('../../lib/config'); +var resolverFactory = require('../../lib/core/resolverFactory'); +var FsResolver = require('../../lib/core/resolvers/FsResolver'); +var GitFsResolver = require('../../lib/core/resolvers/GitFsResolver'); +var GitRemoteResolver = require('../../lib/core/resolvers/GitRemoteResolver'); +var UrlResolver = require('../../lib/core/resolvers/UrlResolver'); describe('resolverFactory', function () { var tempSource; @@ -278,7 +278,7 @@ describe('resolverFactory', function () { }, { config: mout.object.fillIn({ shorthandResolver: 'git://bower.io/{{owner}}/{{package}}/{{shorthand}}' - }, config) + }, defaultConfig) }); }) .then(function (resolver) { diff --git a/test/resolve/resolvers/fsResolver.js b/test/core/resolvers/fsResolver.js similarity index 99% rename from test/resolve/resolvers/fsResolver.js rename to test/core/resolvers/fsResolver.js index 84e92039..7bce1790 100644 --- a/test/resolve/resolvers/fsResolver.js +++ b/test/core/resolvers/fsResolver.js @@ -6,7 +6,7 @@ var rimraf = require('rimraf'); var Q = require('q'); var cmd = require('../../../lib/util/cmd'); var copy = require('../../../lib/util/copy'); -var FsResolver = require('../../../lib/resolve/resolvers/FsResolver'); +var FsResolver = require('../../../lib/core/resolvers/FsResolver'); describe('FsResolver', function () { var testPackage = path.resolve(__dirname, '../../assets/github-test-package'); diff --git a/test/resolve/resolvers/gitFsResolver.js b/test/core/resolvers/gitFsResolver.js similarity index 99% rename from test/resolve/resolvers/gitFsResolver.js rename to test/core/resolvers/gitFsResolver.js index 43b06974..eeefe53b 100644 --- a/test/resolve/resolvers/gitFsResolver.js +++ b/test/core/resolvers/gitFsResolver.js @@ -6,7 +6,7 @@ var rimraf = require('rimraf'); var Q = require('q'); var cmd = require('../../../lib/util/cmd'); var copy = require('../../../lib/util/copy'); -var GitFsResolver = require('../../../lib/resolve/resolvers/GitFsResolver'); +var GitFsResolver = require('../../../lib/core/resolvers/GitFsResolver'); describe('GitFsResolver', function () { var testPackage = path.resolve(__dirname, '../../assets/github-test-package'), diff --git a/test/resolve/resolvers/gitRemoteResolver.js b/test/core/resolvers/gitRemoteResolver.js similarity index 98% rename from test/resolve/resolvers/gitRemoteResolver.js rename to test/core/resolvers/gitRemoteResolver.js index abbcb3f0..f702895c 100644 --- a/test/resolve/resolvers/gitRemoteResolver.js +++ b/test/core/resolvers/gitRemoteResolver.js @@ -2,7 +2,7 @@ var expect = require('expect.js'); var path = require('path'); var fs = require('fs'); var Q = require('q'); -var GitRemoteResolver = require('../../../lib/resolve/resolvers/GitRemoteResolver'); +var GitRemoteResolver = require('../../../lib/core/resolvers/GitRemoteResolver'); describe('GitRemoteResolver', function () { var testPackage = path.resolve(__dirname, '../../assets/github-test-package'); diff --git a/test/resolve/resolvers/gitResolver.js b/test/core/resolvers/gitResolver.js similarity index 97% rename from test/resolve/resolvers/gitResolver.js rename to test/core/resolvers/gitResolver.js index 3a824f0f..075b766a 100644 --- a/test/resolve/resolvers/gitResolver.js +++ b/test/core/resolvers/gitResolver.js @@ -7,7 +7,7 @@ var rimraf = require('rimraf'); var Q = require('q'); var mout = require('mout'); var copy = require('../../../lib/util/copy'); -var GitResolver = require('../../../lib/resolve/resolvers/GitResolver'); +var GitResolver = require('../../../lib/core/resolvers/GitResolver'); describe('GitResolver', function () { var tempDir = path.resolve(__dirname, '../../assets/tmp'); @@ -718,7 +718,7 @@ describe('GitResolver', function () { it('should save the resolution to the .bower.json to be used later by .hasNew', function (next) { var resolver = new GitResolver('foo'); - resolver._resolution = { type: 'version', version: '0.0.1', tag: '0.0.1' }; + resolver._resolution = { type: 'version', tag: '0.0.1' }; resolver._tempDir = tempDir; resolver._savePkgMeta({ name: 'foo', version: '0.0.1' }) @@ -737,7 +737,7 @@ describe('GitResolver', function () { it('should add the version to the package meta if not present and resolution is a version', function (next) { var resolver = new GitResolver('foo'); - resolver._resolution = { type: 'version', version: '0.0.1', tag: '0.0.1' }; + resolver._resolution = { type: 'version', tag: 'v0.0.1' }; resolver._tempDir = tempDir; resolver._savePkgMeta({ name: 'foo' }) @@ -776,7 +776,7 @@ describe('GitResolver', function () { var resolver = new GitResolver('foo'); var notified = false; - resolver._resolution = { type: 'version', version: '0.0.1', tag: '0.0.1' }; + resolver._resolution = { type: 'version', tag: '0.0.1' }; resolver._tempDir = tempDir; resolver._savePkgMeta({ name: 'foo', version: '0.0.0' }) @@ -798,6 +798,27 @@ describe('GitResolver', function () { }) .done(); }); + + it('should not warn if the resolution version and the package meta version are the same', function (next) { + var resolver = new GitResolver('foo'); + var notified = false; + + resolver._resolution = { type: 'version', tag: 'v0.0.1' }; + resolver._tempDir = tempDir; + + resolver._savePkgMeta({ name: 'foo', version: '0.0.1' }) + .then(function () { + return Q.nfcall(fs.readFile, path.join(tempDir, '.bower.json')); + }, null) + .then(function (contents) { + var json = JSON.parse(contents.toString()); + expect(json.version).to.equal('0.0.1'); + expect(notified).to.be(false); + + next(); + }) + .done(); + }); }); describe('#fetchBranches', function () { diff --git a/test/resolve/resolver.js b/test/core/resolvers/resolver.js similarity index 94% rename from test/resolve/resolver.js rename to test/core/resolvers/resolver.js index e0dacfe8..4dde9d54 100644 --- a/test/resolve/resolver.js +++ b/test/core/resolvers/resolver.js @@ -4,13 +4,13 @@ var path = require('path'); var util = require('util'); var rimraf = require('rimraf'); var tmp = require('tmp'); -var cmd = require('../../lib/util/cmd'); -var copy = require('../../lib/util/copy'); -var Resolver = require('../../lib/resolve/Resolver'); +var cmd = require('../../../lib/util/cmd'); +var copy = require('../../../lib/util/copy'); +var Resolver = require('../../../lib/core/resolvers/Resolver'); describe('Resolver', function () { - var tempDir = path.resolve(__dirname, '../assets/tmp'); - var testPackage = path.resolve(__dirname, '../assets/github-test-package'); + var tempDir = path.resolve(__dirname, '../../assets/tmp'); + var testPackage = path.resolve(__dirname, '../../assets/github-test-package'); describe('.getSource', function () { it('should return the resolver source', function () { @@ -46,6 +46,12 @@ describe('Resolver', function () { expect(resolver.getTarget()).to.equal('*'); }); + + it('should return * if latest was configured (for backwards compatibility)', function () { + var resolver = new Resolver('foo'); + + expect(resolver.getTarget()).to.equal('*'); + }); }); describe('.hasNew', function () { @@ -399,7 +405,7 @@ describe('Resolver', function () { rimraf(bowerOsTempDir, function (err) { if (err) return next(err); - cmd('node', ['test/assets/test-temp-dir/test.js'], { cwd: path.resolve(__dirname, '../..') }) + cmd('node', ['test/assets/test-temp-dir/test.js'], { cwd: path.resolve(__dirname, '../../..') }) .then(function () { expect(fs.existsSync(bowerOsTempDir)).to.be(true); expect(fs.readdirSync(bowerOsTempDir)).to.eql([]); @@ -417,7 +423,7 @@ describe('Resolver', function () { rimraf(bowerOsTempDir, function (err) { if (err) return next(err); - cmd('node', ['test/assets/test-temp-dir/test-exception.js'], { cwd: path.resolve(__dirname, '../..') }) + cmd('node', ['test/assets/test-temp-dir/test-exception.js'], { cwd: path.resolve(__dirname, '../../..') }) .then(function () { next(new Error('The command should have failed')); }, function () { @@ -531,30 +537,6 @@ describe('Resolver', function () { .done(); }); - it('should use the json name if the name was guessed', function (next) { - var resolver = new Resolver('foo'); - - resolver._applyPkgMeta({ name: 'bar' }) - .then(function (retMeta) { - expect(retMeta.name).to.equal('bar'); - expect(resolver.getName()).to.equal('bar'); - next(); - }) - .done(); - }); - - it('should not use the json name if a name was passed in the constructor', function (next) { - var resolver = new Resolver('foo', { name: 'foo' }); - - resolver._applyPkgMeta({ name: 'bar' }) - .then(function (retMeta) { - expect(retMeta.name).to.equal('foo'); - expect(resolver.getName()).to.equal('foo'); - next(); - }) - .done(); - }); - it('should remove files that match the ignore patterns', function (next) { var resolver = new Resolver('foo', { name: 'foo' }); @@ -628,6 +610,8 @@ describe('Resolver', function () { .done(); }); + it.skip('should set the original source in package meta file'); + it('should save the package meta to the package meta file (.bower.json)', function (next) { var resolver = new Resolver('foo'); diff --git a/test/resolve/resolvers/urlResolver.js b/test/core/resolvers/urlResolver.js similarity index 96% rename from test/resolve/resolvers/urlResolver.js rename to test/core/resolvers/urlResolver.js index 55b6a535..dc815a21 100644 --- a/test/resolve/resolvers/urlResolver.js +++ b/test/core/resolvers/urlResolver.js @@ -6,7 +6,7 @@ var nock = require('nock'); var Q = require('q'); var rimraf = require('rimraf'); var cmd = require('../../../lib/util/cmd'); -var UrlResolver = require('../../../lib/resolve/resolvers/UrlResolver'); +var UrlResolver = require('../../../lib/core/resolvers/UrlResolver'); describe('UrlResolver', function () { var testPackage = path.resolve(__dirname, '../../assets/github-test-package'); @@ -299,6 +299,10 @@ describe('UrlResolver', function () { .get('/package-zip-folder.zip') .replyWithFile(200, path.resolve(__dirname, '../../assets/package-zip-folder.zip')); + nock('http://bower.io') + .get('/package-zip.zip') + .replyWithFile(200, path.resolve(__dirname, '../../assets/package-zip-folder.zip')); + resolver = new UrlResolver('http://bower.io/package-zip-folder.zip'); resolver.resolve() @@ -308,6 +312,17 @@ describe('UrlResolver', function () { expect(fs.existsSync(path.join(dir, 'package-zip'))).to.be(false); expect(fs.existsSync(path.join(dir, 'package-zip-folder'))).to.be(false); expect(fs.existsSync(path.join(dir, 'package-zip-folder.zip'))).to.be(false); + + resolver = new UrlResolver('http://bower.io/package-zip.zip', { name: 'package-zip' }); + + return resolver.resolve(); + }) + .then(function (dir) { + expect(fs.existsSync(path.join(dir, 'foo.js'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'bar.js'))).to.be(true); + expect(fs.existsSync(path.join(dir, 'package-zip'))).to.be(false); + expect(fs.existsSync(path.join(dir, 'package-zip.zip'))).to.be(false); + next(); }) .done(); diff --git a/test/resolve/worker.js b/test/core/worker.js similarity index 99% rename from test/resolve/worker.js rename to test/core/worker.js index d070dea0..8b0bb5be 100644 --- a/test/resolve/worker.js +++ b/test/core/worker.js @@ -1,6 +1,6 @@ var expect = require('expect.js'); var Q = require('q'); -var Worker = require('../../lib/resolve/Worker'); +var Worker = require('../../lib/core/Worker'); describe('Worker', function () { var timeout; diff --git a/test/test.js b/test/test.js index 00a36198..c7c41891 100644 --- a/test/test.js +++ b/test/test.js @@ -2,14 +2,14 @@ // It messes with the mocha uncaughtException event to caught errors // Please note that is the Resolver that calls tmp.setGracefulCleanup() // so we need to require that before -require('../lib/resolve/Resolver'); +require('../lib/core/resolvers/Resolver'); process.removeAllListeners('uncaughtException'); -require('./resolve/resolver'); -require('./resolve/resolvers/urlResolver'); -require('./resolve/resolvers/fsResolver'); -require('./resolve/resolvers/gitResolver'); -require('./resolve/resolvers/gitFsResolver'); -require('./resolve/resolvers/gitRemoteResolver'); -require('./resolve/resolverFactory'); -require('./resolve/worker'); \ No newline at end of file +require('./core/resolvers/resolver'); +require('./core/resolvers/urlResolver'); +require('./core/resolvers/fsResolver'); +require('./core/resolvers/gitResolver'); +require('./core/resolvers/gitFsResolver'); +require('./core/resolvers/gitRemoteResolver'); +require('./core/resolverFactory'); +require('./core/worker'); \ No newline at end of file