diff --git a/tools/catalog-local.js b/tools/catalog-local.js index 8430adcdd6..0dd8a752fe 100644 --- a/tools/catalog-local.js +++ b/tools/catalog-local.js @@ -85,8 +85,6 @@ _.extend(LocalCatalog.prototype, { self.localPackageDirs = _.filter(options.localPackageDirs || [], utils.isDirectory); - // Lastly, let's read through the data.json file and then put through the - // local overrides. self.refresh({initializing: true}); }, @@ -113,38 +111,6 @@ _.extend(LocalCatalog.prototype, { throw new Error("catalog not initialized yet?"); }, - // Accessor methods below. The primary function of both catalogs is to provide - // data about the existence of various packages, versions, etc, which it does - // through these methods. - - // We have a pattern, where we try to retrieve something and if we fail, call - // refresh to check if there is new data. That makes sense to me, so let's do - // that all the time. (In the future, we should continue some rate-limiting - // on refresh, but for now, most of the time, refreshing on an unknown - // will just cause us to crash if it doesn't exist, so we are just delaying - // the inevitable rather than slowing down normal operations) - _recordOrRefresh: function (recordFinder) { - var self = this; - buildmessage.assertInCapture(); - var record = recordFinder(); - // If we cannot find it maybe refresh. - if (!record && self._refreshingIsProductive()) { - if (! catalog.official.refreshInProgress()) { - catalog.official.refresh(); - } - record = recordFinder(); - } - // If we still cannot find it, give the user a null. - if (!record) { - return null; - } - return record; - }, - - _refreshingIsProductive: function () { - return true; - }, - // Return an array with the names of all of the packages that we // know about, in no particular order. getAllPackageNames: function () { @@ -162,11 +128,7 @@ _.extend(LocalCatalog.prototype, { self._requireInitialized(); options = options || {}; - var get = function () { - return _.findWhere(self.packages, { name: name }); - }; - - return options.noRefresh ? get() : self._recordOrRefresh(get); + return _.findWhere(self.packages, { name: name }); }, // Given a package, returns an array of the versions available for @@ -213,7 +175,7 @@ _.extend(LocalCatalog.prototype, { return lookupVersion() || null; } - return self._recordOrRefresh(lookupVersion); + return lookupVersion(); }, // Overridden by CompleteCatalog. @@ -243,24 +205,23 @@ _.extend(LocalCatalog.prototype, { // one, in case our local cache says "version exists but only for the wrong // arch" and the right arch has been recently published. // XXX should ensure at most one refresh - return self._recordOrRefresh(function () { - var allBuilds = _.where(self.builds, { versionId: versionInfo._id }); - var solution = null; - utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) { - // This build subset works if for all the arches we need, at least one - // build in the subset satisfies it. It is guaranteed to be minimal, - // because we look at subsets in increasing order of size. - var satisfied = _.all(arches, function (neededArch) { - return _.any(buildSubset, function (build) { - var buildArches = build.buildArchitectures.split('+'); - return !!archinfo.mostSpecificMatch(neededArch, buildArches); - }); + // PASCAL - check with Ekate + var allBuilds = _.where(self.builds, { versionId: versionInfo._id }); + var solution = null; + utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) { + // This build subset works if for all the arches we need, at least one + // build in the subset satisfies it. It is guaranteed to be minimal, + // because we look at subsets in increasing order of size. + var satisfied = _.all(arches, function (neededArch) { + return _.any(buildSubset, function (build) { + var buildArches = build.buildArchitectures.split('+'); + return !!archinfo.mostSpecificMatch(neededArch, buildArches); }); - if (satisfied) { - solution = buildSubset; - return true; // stop the iteration - } }); + if (satisfied) { + solution = buildSubset; + return true; // stop the iteration + } return solution; // might be null! }); }, @@ -273,11 +234,9 @@ _.extend(LocalCatalog.prototype, { buildmessage.assertInCapture(); self._requireInitialized(); - return self._recordOrRefresh(function () { - return _.findWhere(self.builds, - { versionId: versionRecord._id, - buildArchitectures: buildArchitectures }); - }); + return _.findWhere(self.builds, + { versionId: versionRecord._id, + buildArchitectures: buildArchitectures }); }, getAllBuilds: function (name, version) { @@ -306,33 +265,13 @@ _.extend(LocalCatalog.prototype, { options = options || {}; buildmessage.assertInCapture(); - //PASCAL Review the this option checking logic - // We need to limit the rate of refresh, or, at least, prevent any sort of - // loops. ForceRefresh will override either one. - if (!options.forceRefresh && !options.initializing && self.refreshing) { - return; - } + self.reset(); + self._recomputeEffectiveLocalPackages(); + var allOK = self._loadLocalPackages({ watchSet: options.watchSet }); + self.initialized = true; + // Rebuild the resolver, since packages may have changed. + self.resolver = null; - if (options.initializing) { - // If we are doing the top level initialization in main.js, everything - // sure had better be in a relaxed state, since we're about to hackily - // steal some data from catalog.official. - if (self.refreshing) - throw Error("initializing local catalog re-entrantly?"); - } - - self.refreshing = true; - // console.log("refreshing the local catalog"); - try { - self.reset(); - self._recomputeEffectiveLocalPackages(); - var allOK = self._loadLocalPackages({ watchSet: options.watchSet }); - self.initialized = true; - // Rebuild the resolver, since packages may have changed. - self.resolver = null; - } finally { - self.refreshing = false; - } }, // Compute self.effectiveLocalPackages from self.localPackageDirs diff --git a/tools/catalog-remote.js b/tools/catalog-remote.js index dbf11e1e33..e0628fbd08 100644 --- a/tools/catalog-remote.js +++ b/tools/catalog-remote.js @@ -17,13 +17,10 @@ var semver = require('semver'); var packageClient = require('./package-client.js'); var sqlite3 = require('../dev_bundle/bin/node_modules/sqlite3'); -//PASCAL NOTES -//What are the options that are passed to the initialize method in catalog? - // A RemoteCatalog is a local cache of the content of troposphere. // A default instance of this catalog is registered by the layered catalog and is available // under the variable "official" from the catalog.js - +// // The remote catalog is backed by a db to make things easier on the memory and for faster queries var RemoteCatalog = function (options) { var self = this; @@ -127,11 +124,11 @@ _.extend(RemoteCatalog.prototype, { }, getAllReleaseTracks: function () { - return _.pluck(this._justQuery("SELECT name FROM releaseTracks"), 'name'); + return _.pluck(this._queryWithRetry("SELECT name FROM releaseTracks"), 'name'); }, getAllPackageNames: function () { - return _.pluck(this._justQuery("SELECT name FROM packages")); + return _.pluck(this._queryWithRetry("SELECT name FROM packages")); }, initialize: function (options) { @@ -192,12 +189,29 @@ _.extend(RemoteCatalog.prototype, { future.wait(); }, - refresh: function (overrides) { + refresh: function () { var self = this; if (self.offline) return; - var updateResult = packageClient.updateServerPackageData(this); + var patience = new utils.Patience({ + messageAfterMs: 2000, + message: function () { + if (self._currentRefreshIsLoud) { + console.log("Refreshing package metadata. This may take a moment."); + } + } + }); + + var updateResult = {}; + try { + packageClient.updateServerPackageData(this); + } finally { + patience.stop(); + } + if (!updateResult.data) { + process.stderr.write("Warning: could not connect to package server\n"); + } if (updateResult.resetData) { tropohouse.default.wipeAllPackages(); self.purgeDB(); @@ -205,12 +219,6 @@ _.extend(RemoteCatalog.prototype, { }, - refreshInProgress: function () { - return false; - // var self = this; - // return self._refreshFiber === Fiber.current; - }, - // Given a release track, return all recommended versions for this track, sorted // by their orderKey. Returns the empty array if the release track does not // exist or does not have any recommended versions. @@ -253,6 +261,15 @@ _.extend(RemoteCatalog.prototype, { return false; }, + _queryWithRetry: function (query, values) { + var self = this; + var result = self._justQuery(query, values); + if (result.length !== 0) + return result; + self.refresh(); + return self._justQuery(query, values); + }, + // Runs a query synchronously. // Query is the sql query to be executed and values are the parameters of the query _justQuery: function (query, values) { @@ -265,13 +282,17 @@ _.extend(RemoteCatalog.prototype, { future.return(rows); }); - return future.wait(); + var results = future.wait(); + if (results !== []) + return results; + self.refresh(); + }, // Execute a query using the values as arguments of the query and return the result as JSON. // This code assumes that the table being queried always have a column called "entity" _queryAsJSON: function (query, values) { - var rows = this._justQuery(query, values); + var rows = this._queryWithRetry(query, values); return _.map(rows, function(entity) { return JSON.parse(entity.content); }); diff --git a/tools/catalog.js b/tools/catalog.js index 473739a847..77728d1b5d 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -15,6 +15,11 @@ var utils = require('./utils.js'); var config = require('./config.js'); var packageClient = require('./package-client.js'); + +// The LayeredCatalog provides a way to query multiple catalogs in a uniform way +// A LayeredCatalog typically contains: +// - a local catalog referencing the packages of the project +// - a reference to the official catalog var LayeredCatalog = function() { var self = this; @@ -77,7 +82,7 @@ _.extend(LayeredCatalog.prototype, { }, getLoadPathForPackage: function (name, version, constraintSolverOpts) { - var self = this; //PASCAL check with Ekate + var self = this; return self.localCatalog.getLoadPathForPackage(name, version, constraintSolverOpts); }, @@ -95,7 +100,6 @@ _.extend(LayeredCatalog.prototype, { getReleaseWithTool: function (toolSpec) { var self = this; buildmessage.assertInCapture(); - self._requireInitialized(); return self._recordOrRefresh(function () { return _.findWhere(self.releaseVersions, { tool: toolSpec }); }); @@ -136,12 +140,6 @@ _.extend(LayeredCatalog.prototype, { return this.localCatalog.rebuildLocalPackages(namedPackages); }, - refreshInProgress: function () { - var self = this; - // console.log("refresh in progress the LayeredCatalog"); - //PASCAL Deal with refresh properly - }, - reset: function () { this.localCatalog.reset(); }, @@ -151,7 +149,6 @@ _.extend(LayeredCatalog.prototype, { // It does not include prereleases (with dashes in the version); getLatestMainlineVersion: function (name) { var self = this; - self._requireInitialized(); buildmessage.assertInCapture(); var versions = self.getSortedVersions(name); @@ -164,18 +161,9 @@ _.extend(LayeredCatalog.prototype, { return self.getVersion(name, latest); }, - // Throw if the catalog's self.initialized value has not been set to true. - _requireInitialized: function () { - var self = this; - //PASCAL - // if (! self.initialized) - // throw new Error("catalog not initialized yet?"); - }, - resolveConstraints : function (constraints, resolverOpts, opts) { var self = this; opts = opts || {}; - self._requireInitialized(); buildmessage.assertInCapture(); // OK, since we are the complete catalog, the uniload catalog must be fully @@ -244,7 +232,6 @@ _.extend(LayeredCatalog.prototype, { } catch (e) { // Maybe we only failed because we need to refresh. Try to refresh // (unless we already are) and retry. - //PASCAL review if (!self._refreshingIsProductive() || exports.official.refreshInProgress()) { throw e; @@ -285,10 +272,7 @@ _.extend(LayeredCatalog.prototype, { return ret.answer; }, - // Refresh the packages in the catalog. - // - // Reread server data from data.json on disk, then load local overrides on top - // of that information. Sets initialized to true. + // Refresh the catalogs referenced by this catalog. // options: // - forceRefresh: even if there is a future in progress, refresh the catalog // anyway. When we are using hot code push, we may be restarting the app @@ -298,83 +282,10 @@ _.extend(LayeredCatalog.prototype, { // to this set. refresh: function (options) { var self = this; - console.log("refresh layered catalo"); self.localCatalog.refresh(options); self.otherCatalog.refresh(options); self.packageCache.refresh(); self.resolver = null; - // options = options || {}; - // buildmessage.assertInCapture(); - - // // We need to limit the rate of refresh, or, at least, prevent any sort of - // // loops. ForceRefresh will override either one. - // if (!options.forceRefresh && !options.initializing && - // (catalog.official._refreshFutures || self.refreshing)) { - - // return; - // } - - // if (options.initializing && !self.forUniload) { - // // If we are doing the top level initialization in main.js, everything - // // sure had better be in a relaxed state, since we're about to hackily - // // steal some data from catalog.official. - // if (self.refreshing) - // throw Error("initializing catalog.complete re-entrantly?"); - // if (catalog.official._refreshFutures) - // throw Error("initializing catalog.complete during official refresh?"); - // } - - // if (self.refreshing) { - // // We're being asked to refresh re-entrantly, maybe because we just - // // updated the official catalog. Let's not do this now, but make the - // // outer call do it instead. - // // XXX refactoring the catalogs so that the two catalogs share their - // // data and this one is just an overlay would reduce this wackiness - // self.needRefresh = true; - // return; - // } - - // self.refreshing = true; - - // try { - // self.reset(); - - // if (!self.forUniload) { - // if (options.initializing) { - // // It's our first time! Everything ought to be at rest. Let's just - // // steal data (without even a deep clone!) from catalog.official. - // // XXX this is horrible. restructure to have a reference to - // // catalog.official instead. - // self.packages = _.clone(catalog.official.packages); - // self.builds = _.clone(catalog.official.builds); - // _.each(catalog.official.versions, function (versions, name) { - // self.versions[name] = _.clone(versions); - // }); - // } else { - // // Not the first time. Slowly load data from disk. - // // XXX restructure this class to just have a reference to - // // catalog.official instead of a copy of its data. - // var localData = packageClient.loadCachedServerData(); - // self._insertServerPackages(localData); - // } - // } - - // self._recomputeEffectiveLocalPackages(); - // var allOK = self._addLocalPackageOverrides( - // { watchSet: options.watchSet }); - // self.initialized = true; - // // Rebuild the resolver, since packages may have changed. - // self.resolver = null; - // } finally { - // self.refreshing = false; - // }f - - // // If we got a re-entrant refresh request, do it now. (But not if we - // // encountered build errors building the packages, since in that case - // // we'd probably just get the same build errors again.) - // if (self.needRefresh && allOK) { - // self.refresh(options); - // } }, _initializeResolver: function () { @@ -396,13 +307,7 @@ _.extend(LayeredCatalog.prototype, { watchLocalPackageDirs: function (watchSet) { var self = this; self.localCatalog.watchLocalPackageDirs(watchSet); - }, - - _refreshingIsProductive: function() { - //PASCAL REVIEW - return true; - } - + } }); exports.DEFAULT_TRACK = remoteCatalog.DEFAULT_TRACK;