From b968b4a9a4f15315fea9eaac0c940cf4c83c8dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andre=CC=81=20Cruz?= Date: Fri, 19 Apr 2013 19:05:33 +0100 Subject: [PATCH] Change UnitOfWork to Worker. --- .gitignore | 3 +- README.md | 62 +++++++++------- lib/resolve/UnitOfWork.js | 144 ------------------------------------ lib/resolve/Worker.js | 150 ++++++++++++++++++++++++++++++++++++++ package.json | 5 +- resolve_diagram.graphml | 82 +++------------------ 6 files changed, 200 insertions(+), 246 deletions(-) delete mode 100644 lib/resolve/UnitOfWork.js create mode 100644 lib/resolve/Worker.js diff --git a/.gitignore b/.gitignore index 04c46376..07ddee34 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ .DS_Store Thumbs.db -node_modules -components \ No newline at end of file +node_modules \ No newline at end of file diff --git a/README.md b/README.md index b6a8fb16..ab1b9a35 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,10 @@ ## Why? -Bower codebase is becoming unmanageable, especially at its core. +Bower code base is becoming unmanageable, especially at its core. Main issues are: -- __No separation of concerns. The overall codebase has grown in a patch fashion, which has lead to a bloated and tight coupled solution.__ +- __No separation of concerns. The overall code base has grown in a patch fashion, which has lead to a bloated and tight coupled solution.__ - __Monolithic Package.js that handles all package types (both local and remote `Git`, URL, local files, etc).__ - __Package.js has a big nesting level of callbacks, causing confusion and making the code hard to read.__ - Some commands, such as install and update, have incorrect behaviour ([#200](https://github.com/twitter/bower/issues/200), [#256](https://github.com/twitter/bower/issues/256)) @@ -21,7 +21,7 @@ Main issues are: - Ease the process of gathering more contributors. - Clear architecture and separation of concerns. -- Installation/update speedup. +- Installation/update speed-up. - Named endpoints on the CLI install. - Offline installation of packages, thanks to the cache. - Ability to easily add package types (`SVN`, etc). @@ -39,13 +39,12 @@ Main issues are: - **Target:** `semver` range, commit hash, branch (indicates a version). - **Endpoint:** source#target - **Named endpoint:** name@endpoint#target -- **UoW:** Unit of Work - **Components folder:** The folder in which components are installed (`bower_components` by default). - **Package meta:** A data structure similar to the one found in `bower.json`, which might also contain additional information. This is usually stored in a `.bower.json` file, inside a canonical package. ### Overall strategy -![Really nicely drawn architecture diagram](http://f.cl.ly/items/44271M0R1O012H2m4234/resolve_diagram.png "Don't over think it! We already did! :P") +![Really nicely drawn architecture diagram](http://f.cl.ly/items/2z0u3B1817341P0q0H3M/bower_diagram2.jpg "Don't over think it! We already did! :P") Bower is composed of the following components: @@ -59,11 +58,9 @@ Bower is composed of the following components: - Requesting the `PackageRepository` to fail-fast, in case it realises there is no resolution for the current dependency tree. - `PackageRepository`: Abstraction to the underlying complexity of heterogeneous source types. Responsible for: - Storing new entries in `ResolveCache`. - - Queueing resolvers into the `UoW`, if no suitable entry is found in the `ResolveCache`. + - Queueing resolvers into the `Worker`, if no suitable entry is found in the `ResolveCache`. - `ResolveCache`: Keeps a cache of previously resolved endpoints. Lookup can be done using an endpoint. -- `UnitOfWork`: Work coordinator, responsible for: - - Keeping track of which resolvers are being resolved. - - Limiting amount of parallel resolutions. +- `Worker`: A service responsible for limiting amount of parallel executions of tasks of the same type. - `ResolverFactory`: Parses an endpoint and returns a `Resolver` capable of resolving the source type. - `Resolver`: Base resolver, which can be extended by concrete resolvers, like `UrlResolver`, `GitRemoteResolver`, etc. @@ -96,7 +93,7 @@ Here's an overview of the dependency resolve process: 6. **CACHE HIT VALIDATION** - At this stage, and only for the cache hits, the `PackageRepository` will question the `Resolver` if there is any version higher than the one fetched from cache that also complies with the endpoint target. Some considerations: - This step is ignored in case a flag like `offline` is passed. - - How the `Resolver` checks this, depends on the `Resolver` type. (e.g. `GitRemoteResolver` would fetch the git refs with `git ls-remote --tags --heads`, and check if there is a higher version that complies with the target). + - How the `Resolver` checks this, depends on the `Resolver` type. (e.g. `GitRemoteResolver` would fetch the git refs with `git ls-remote --tags --heads`, and check if there is a higher version that complies with the target). - This check should be as quick as possible. If the process of checking a new version is too slow, it's preferable to just assume there is a new version. - If there is no way to check if there is a higher version, assume that there is. - If the `Resolver` indicates that the cached version is outdated, then it is treated as a cache miss. @@ -148,7 +145,7 @@ Simple function that takes a *named endpoint*/endpoint with options and creates function createResolver(endpoint, options) -> Promise ``` -This function will perform transformations/normalisations to the endpoint, like expanding shorthand andendpoints. +This function will perform transformations/normalisations to the endpoint, like expanding shorthand endpoints. The function is async to allow querying the Bower registry, etc. @@ -243,7 +240,7 @@ Creates a temporary dir. Reads `bower.json`/`component.json`, possibly by using a dedicated `read-json` node module that will be available in the Bower organisation. -This method also normalises the `package meta`, filling in any missing information, inferring when possible. +This method also generates the `package meta` based on the `json`, filling in any missing information, inferring when possible. `Resolver#_decoratePkgMeta(meta)`: Promise @@ -283,37 +280,48 @@ The `ResolverFactory` knows these types, and is able to fabricate suitable resol This architecture makes it very easy for the community to create others package types, for instance, a `MercurialFsResolver`, `MercurialResolver`, `SvnResolver`, etc. -#### Unit of work +#### Worker -The work coordinator, responsible for keeping track of which resolvers are being resolved. -The number of parallel resolutions may be limited and configured. +A worker responsible for limiting execution of parallel tasks. +The number of parallel tasks may be limited and configured per type. +This component will be a service that can be accessed to perform tasks. ------------ #### Constructor -`UnitOfWork(options)` +`Worker(defaultConcurrency, types)` -Options: +The `defaultConcurrency` is the default maximum concurrent functions being run. +The `types` allows you to specify different concurrencies for different types. +Use `-1` to specify no limits. -- `maxConcurrent`: maximum number of concurrent resolvers running (defaults to 5) +Example: + +```js +var worker = new Worker(15, { + 'network_io': 10, + 'disk_io': 50 +}); +``` ------------ #### Public methods. -`UnitOfWork#enqueue(resolver)`: Promise +`Worker#enqueue(func, type)`: Promise -Enqueues a resolver to be ran. -The promise is fulfilled when the resolver successfully resolved or is rejected if it failed to resolve. +Enqueues a function to be ran. The function is expected to return a promise. +The returned promise is resolved when the function promise is also resolved. -`UnitOfWork#has(resolver)`: Boolean +The `type` argument is optional and can be a `string` or an array of `strings`. +Use it to specify the type(s) associated with the function. +If multiple types are specified, the function will only be ran when a free slot on every type list is found. -Checks if a resolver is already in the unit of work. +`Worker#abort()`: Promise -`UnitOfWork#abort()`: Promise +Aborts all current work being done. +Returns a promise that is resolved when the current running functions finish to execute. +Any function that was in the queue waiting to be ran is removed immediately. -Aborts the current work being done, by removing any resolvers waiting to resolve. -Note that resolvers running can't be aborted. -Returns a promise that is fulfilled when the current running resolvers finish the resolve process. Any `Resolver` that was in queue to be ran is aborted immediately. diff --git a/lib/resolve/UnitOfWork.js b/lib/resolve/UnitOfWork.js deleted file mode 100644 index 87c6ce46..00000000 --- a/lib/resolve/UnitOfWork.js +++ /dev/null @@ -1,144 +0,0 @@ -var Q = require('q'); -var util = require('util'); -var mout = require('mout'); -var events = require('events'); - -var UnitOfWork = function (options) { - // Ensure options defaults - this._options = mout.object.mixIn({ - maxConcurrent: 5 - }, options); - - // Parse some of the options - this._options.maxConcurrent = this._options.maxConcurrent > 0 ? this._options.maxConcurrent : 0; - - // Initialize some needed properties - this._queue = []; - this._beingResolved = []; -}; - -util.inherits(UnitOfWork, events.EventEmitter); - -// ----------------- - -UnitOfWork.prototype.enqueue = function (resolver) { - var deferred; - - if (this.has(resolver)) { - throw new Error('Attempting to enqueue an already enqueued resolver'); - } - - deferred = Q.defer(); - - // Add to the queue - this._queue.push({ - resolver: resolver, - deferred: deferred - }); - - // Process the queue shortly later so that handlers can be attached to the returned promise - Q.fcall(this.doWork.bind(this)); - - return deferred.promise; -}; - -UnitOfWork.prototype.has = function (resolver) { - var index; - - // Check in the queue - index = this._indexOf(this._queue, resolver); - if (index !== -1) { - return true; - } - - // Check in the being resolved list - index = this._indexOf(this._beingResolved, resolver); - if (index !== -1) { - return true; - } - - return false; -}; - -UnitOfWork.prototype.abort = function () { - var promises, - emptyFunc = function () {}; - - // Empty queue - this._queue = []; - - // Wait for pending resolvers to resolve - promises = this._beingResolved.map(function (entry) { - // Please note that the promise resolution/fail is silenced - return entry.deferred.promise.then(emptyFunc, emptyFunc); - }); - - return Q.all(promises); -}; - -// ----------------- - -UnitOfWork.prototype._doWork = function () { - // Check if the number of allowed packages being resolved reached the maximum - if (this._options.maxConcurrent && this._beingResolved.length >= this._options.maxConcurrent) { - return; - } - - // Find candidates for the free slots - var freeSlots = this._options.maxConcurrent ? this._options.maxConcurrent - this._beingResolved.length : -1, - entry, - resolver, - x; - - for (x = 0; x < this._queue.length && freeSlots; ++x) { - entry = this._queue[x]; - resolver = entry.resolver; - - // Remove from the queue and - this._queue.splice(x--, 1); - - // Put it in the being resolved list - this._beingResolved.push(entry); - freeSlots--; - - // Resolve it, waiting for it to be done - this.emit('pre_resolve', resolver); - resolver.resolve().then(this._onResolveSuccess.bind(this, entry), this._onResolveFailure.bind(this, entry)); - } -}; - -UnitOfWork.prototype._onResolveSuccess = function (entry, result) { - var index; - - // Remove the package from the being resolved list - index = this._beingResolved.indexOf(entry); - this._beingResolved.splice(index, 1); - - entry.deferred.resolve(result); - this.emit('post_resolve', entry.resolver, result); - - // A free spot became available, so let's do some more work - this._doWork(); -}; - -UnitOfWork.prototype._onResolveFailure = function (entry, err) { - var index; - - // Remove the package from the being resolved list - index = this._beingResolved.indexOf(entry); - this._beingResolved.splice(index, 1); - - entry.deferred.reject(err); - this.emit('fail', entry.resolver, err); - - // A free spot became available, so let's do some more work - this._doWork(); -}; - -UnitOfWork.prototype._indexOf = function (arr, pkg) { - return mout.array.findIndex(arr, function (item) { - return item.pkg === pkg; - }); -}; - -module.exports = UnitOfWork; \ No newline at end of file diff --git a/lib/resolve/Worker.js b/lib/resolve/Worker.js new file mode 100644 index 00000000..cfe10aa3 --- /dev/null +++ b/lib/resolve/Worker.js @@ -0,0 +1,150 @@ +var Q = require('q'); +var util = require('util'); +var events = require('events'); +var mout = require('mout'); + +var Worker = function (defaultConcurrency, types) { + this._defaultConcurrency = defaultConcurrency != null ? defaultConcurrency : 10; + + // Initialize some needed properties + this._queue = {}; + this._slots = types || {}; + this._executing = []; +}; + +util.inherits(Worker, events.EventEmitter); + +// ----------------- + +Worker.prototype.enqueue = function (func, type) { + var deferred = Q.defer(), + types, + entry; + + type = type || ''; + types = Array.isArray(type) ? type : [type]; + entry = { + func: func, + types: types, + deferred: deferred + }; + + // Add the entry to all the types queues + types.forEach(function (type) { + this._queue[type] = this._queue[type] || []; + this._queue.push(entry); + }, this); + + // Process the entry shortly later so that handlers can be attached to the returned promise + Q.fcall(this._processEntry.bind(this, entry)); + + return deferred.promise; +}; + +Worker.prototype.abort = function (type) { + var promises = [], + types; + + type = type || ''; + types = Array.isArray(type) ? type : [type]; + + // Empty the queue of each specified types + types.forEach(function (type) { + this._queue[type] = []; + }, this); + + + // Wait for all pending functions of the specified type to finish + promises = this._executing.map(function (entry) { + return entry.deferred.promise; + }); + + return Q.allResolved(promises) + .then(function () {}); // Resolve with no value +}; + +// ----------------- + +Worker.prottoype._processQueue = function (type) { + var queue = this._queue[type], + length = queue ? queue.length : 0, + x; + + for (x = 0; x < length; x += 1) { + if (!this._processEntry(this._queue[x])) { + break; + } + } +}; + +Worker.prototype._processEntry = function (entry) { + var allFree = entry.types.every(this._hasSlot, this); + + // If there is a free slot for every tag + if (allFree) { + // Foreach type + entry.types.forEach(function (type) { + // Remove entry from the queue + mout.utils.remove(this._queue[type], entry); + // Take slot + this._takeSlot(type); + }, this); + + // Execute the function + this._executing.push(entry); + entry.func().then( + this._onResolve.bind(this, entry, true), + this._onResolve.bind(this, entry, false) + ); + } + + return allFree; +}; + + +Worker.prototype._onResolve = function (entry, ok, result) { + // Resolve/reject the deferred based on sucess/error of the promise + if (ok) { + entry.deferred.resolve(result); + } else { + entry.deferred.reject(result); + } + + // Remove it from the executing list + mout.array.remove(this._executing, entry); + + // Free up slots for every type + entry.types.forEach(this._freeSlot, this); + + // Find candidates for the free slots of each type + entry.types.forEach(this._processQueue, this); +}; + +Worker.prototype._hasSlot = function (type) { + var freeSlots = this._slots[type]; + + if (freeSlots == null) { + freeSlots = this._defaultConcurrency; + } + + return freeSlots > 0; +}; + +Worker.prototype._takeSlot = function (type) { + if (this._slots[type] == null) { + this._slots[type] = this._defaultConcurrency; + } else if (!this._slots[type]) { + throw new Error('No free slots'); + } + + // Decrement the free slots + --this._slots[type]; +}; + +Worker.prototype._freeSlot = function (type) { + if (this._slots[type] != null) { + ++this._slots[type]; + } +}; + +module.exports = Worker; \ No newline at end of file diff --git a/package.json b/package.json index 2828deef..38865d7b 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,7 @@ "name": "bower", "version": "0.0.0", "description": "The browser package manager.", + "author": "Twitter", "licenses": [ { "type": "MIT", @@ -9,7 +10,7 @@ } ], "main": "lib", - "homepage": "http://twitter.github.com/bower", + "homepage": "http://bower.io", "engines": { "node": ">=0.8.0" }, @@ -30,8 +31,6 @@ "scripts": { "test": "mocha -R spec" }, - "author": "Twitter", - "license": "MIT", "bin": { "bower": "bin/bower" }, diff --git a/resolve_diagram.graphml b/resolve_diagram.graphml index d9cf540f..8355d8a2 100644 --- a/resolve_diagram.graphml +++ b/resolve_diagram.graphml @@ -95,22 +95,6 @@ - - - - - - UoW - - - - - - - - - - @@ -126,7 +110,7 @@ - + @@ -142,7 +126,7 @@ - + @@ -158,7 +142,7 @@ - + @@ -174,7 +158,7 @@ - + @@ -190,7 +174,7 @@ - + @@ -206,7 +190,7 @@ - + @@ -365,49 +349,7 @@ endpoint - - - - - - - - - Enqueue -Resolver - - - - - - - - - - - - - - - - - - - - Resolved -Resolver - - - - - - - - - - - - + @@ -428,7 +370,7 @@ package - + @@ -441,7 +383,7 @@ package - + @@ -451,7 +393,7 @@ package - + @@ -464,7 +406,7 @@ package - + @@ -477,7 +419,7 @@ package - +