github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

cleanup the refresh logic

Browse Source
This commit is contained in:
Pascal Rapicault
2014-09-19 21:25:53 -04:00
parent 9149dd9fc8
commit 01428bc2a0
3 changed files with 71 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
tools/catalog-local.js
View File

@@ -85,8 +85,6 @@ _.extend(LocalCatalog.prototype, {
self.localPackageDirs =
_.filter(options.localPackageDirs || [], utils.isDirectory);
// Lastly, let's read through the data.json file and then put through the
// local overrides.
self.refresh({initializing: true});
},
@@ -113,38 +111,6 @@ _.extend(LocalCatalog.prototype, {
throw new Error("catalog not initialized yet?");
},
// Accessor methods below. The primary function of both catalogs is to provide
// data about the existence of various packages, versions, etc, which it does
// through these methods.
// We have a pattern, where we try to retrieve something and if we fail, call
// refresh to check if there is new data. That makes sense to me, so let's do
// that all the time. (In the future, we should continue some rate-limiting
// on refresh, but for now, most of the time, refreshing on an unknown <stuff>
// will just cause us to crash if it doesn't exist, so we are just delaying
// the inevitable rather than slowing down normal operations)
_recordOrRefresh: function (recordFinder) {
var self = this;
buildmessage.assertInCapture();
var record = recordFinder();
// If we cannot find it maybe refresh.
if (!record && self._refreshingIsProductive()) {
if (! catalog.official.refreshInProgress()) {
catalog.official.refresh();
}
record = recordFinder();
}
// If we still cannot find it, give the user a null.
if (!record) {
return null;
}
return record;
},
_refreshingIsProductive: function () {
return true;
},
// Return an array with the names of all of the packages that we
// know about, in no particular order.
getAllPackageNames: function () {
@@ -162,11 +128,7 @@ _.extend(LocalCatalog.prototype, {
self._requireInitialized();
options = options || {};
var get = function () {
return _.findWhere(self.packages, { name: name });
};
return options.noRefresh ? get() : self._recordOrRefresh(get);
return _.findWhere(self.packages, { name: name });
},
// Given a package, returns an array of the versions available for
@@ -213,7 +175,7 @@ _.extend(LocalCatalog.prototype, {
return lookupVersion() || null;
}
return self._recordOrRefresh(lookupVersion);
return lookupVersion();
},
// Overridden by CompleteCatalog.
@@ -243,24 +205,23 @@ _.extend(LocalCatalog.prototype, {
// one, in case our local cache says "version exists but only for the wrong
// arch" and the right arch has been recently published.
// XXX should ensure at most one refresh
return self._recordOrRefresh(function () {
var allBuilds = _.where(self.builds, { versionId: versionInfo._id });
var solution = null;
utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) {
// This build subset works if for all the arches we need, at least one
// build in the subset satisfies it. It is guaranteed to be minimal,
// because we look at subsets in increasing order of size.
var satisfied = _.all(arches, function (neededArch) {
return _.any(buildSubset, function (build) {
var buildArches = build.buildArchitectures.split('+');
return !!archinfo.mostSpecificMatch(neededArch, buildArches);
});
// PASCAL - check with Ekate
var allBuilds = _.where(self.builds, { versionId: versionInfo._id });
var solution = null;
utils.generateSubsetsOfIncreasingSize(allBuilds, function (buildSubset) {
// This build subset works if for all the arches we need, at least one
// build in the subset satisfies it. It is guaranteed to be minimal,
// because we look at subsets in increasing order of size.
var satisfied = _.all(arches, function (neededArch) {
return _.any(buildSubset, function (build) {
var buildArches = build.buildArchitectures.split('+');
return !!archinfo.mostSpecificMatch(neededArch, buildArches);
});
if (satisfied) {
solution = buildSubset;
return true; // stop the iteration
}
});
if (satisfied) {
solution = buildSubset;
return true; // stop the iteration
}
return solution; // might be null!
});
},
@@ -273,11 +234,9 @@ _.extend(LocalCatalog.prototype, {
buildmessage.assertInCapture();
self._requireInitialized();
return self._recordOrRefresh(function () {
return _.findWhere(self.builds,
{ versionId: versionRecord._id,
buildArchitectures: buildArchitectures });
});
return _.findWhere(self.builds,
{ versionId: versionRecord._id,
buildArchitectures: buildArchitectures });
},
getAllBuilds: function (name, version) {
@@ -306,33 +265,13 @@ _.extend(LocalCatalog.prototype, {
options = options || {};
buildmessage.assertInCapture();
//PASCAL Review the this option checking logic
// We need to limit the rate of refresh, or, at least, prevent any sort of
// loops. ForceRefresh will override either one.
if (!options.forceRefresh && !options.initializing && self.refreshing) {
return;
}
self.reset();
self._recomputeEffectiveLocalPackages();
var allOK = self._loadLocalPackages({ watchSet: options.watchSet });
self.initialized = true;
// Rebuild the resolver, since packages may have changed.
self.resolver = null;
if (options.initializing) {
// If we are doing the top level initialization in main.js, everything
// sure had better be in a relaxed state, since we're about to hackily
// steal some data from catalog.official.
if (self.refreshing)
throw Error("initializing local catalog re-entrantly?");
}
self.refreshing = true;
// console.log("refreshing the local catalog");
try {
self.reset();
self._recomputeEffectiveLocalPackages();
var allOK = self._loadLocalPackages({ watchSet: options.watchSet });
self.initialized = true;
// Rebuild the resolver, since packages may have changed.
self.resolver = null;
} finally {
self.refreshing = false;
}
},
// Compute self.effectiveLocalPackages from self.localPackageDirs

53
tools/catalog-remote.js
View File

@@ -17,13 +17,10 @@ var semver = require('semver');
var packageClient = require('./package-client.js');
var sqlite3 = require('../dev_bundle/bin/node_modules/sqlite3');
//PASCAL NOTES
//What are the options that are passed to the initialize method in catalog?
// A RemoteCatalog is a local cache of the content of troposphere.
// A default instance of this catalog is registered by the layered catalog and is available
// under the variable "official" from the catalog.js
//
// The remote catalog is backed by a db to make things easier on the memory and for faster queries
var RemoteCatalog = function (options) {
var self = this;
@@ -127,11 +124,11 @@ _.extend(RemoteCatalog.prototype, {
},
getAllReleaseTracks: function () {
return _.pluck(this._justQuery("SELECT name FROM releaseTracks"), 'name');
return _.pluck(this._queryWithRetry("SELECT name FROM releaseTracks"), 'name');
},
getAllPackageNames: function () {
return _.pluck(this._justQuery("SELECT name FROM packages"));
return _.pluck(this._queryWithRetry("SELECT name FROM packages"));
},
initialize: function (options) {
@@ -192,12 +189,29 @@ _.extend(RemoteCatalog.prototype, {
future.wait();
},
refresh: function (overrides) {
refresh: function () {
var self = this;
if (self.offline)
return;
var updateResult = packageClient.updateServerPackageData(this);
var patience = new utils.Patience({
messageAfterMs: 2000,
message: function () {
if (self._currentRefreshIsLoud) {
console.log("Refreshing package metadata. This may take a moment.");
}
}
});
var updateResult = {};
try {
packageClient.updateServerPackageData(this);
} finally {
patience.stop();
}
if (!updateResult.data) {
process.stderr.write("Warning: could not connect to package server\n");
}
if (updateResult.resetData) {
tropohouse.default.wipeAllPackages();
self.purgeDB();
@@ -205,12 +219,6 @@ _.extend(RemoteCatalog.prototype, {
},
refreshInProgress: function () {
return false;
// var self = this;
// return self._refreshFiber === Fiber.current;
},
// Given a release track, return all recommended versions for this track, sorted
// by their orderKey. Returns the empty array if the release track does not
// exist or does not have any recommended versions.
@@ -253,6 +261,15 @@ _.extend(RemoteCatalog.prototype, {
return false;
},
_queryWithRetry: function (query, values) {
var self = this;
var result = self._justQuery(query, values);
if (result.length !== 0)
return result;
self.refresh();
return self._justQuery(query, values);
},
// Runs a query synchronously.
// Query is the sql query to be executed and values are the parameters of the query
_justQuery: function (query, values) {
@@ -265,13 +282,17 @@ _.extend(RemoteCatalog.prototype, {
future.return(rows);
});
return future.wait();
var results = future.wait();
if (results !== [])
return results;
self.refresh();
},
// Execute a query using the values as arguments of the query and return the result as JSON.
// This code assumes that the table being queried always have a column called "entity"
_queryAsJSON: function (query, values) {
var rows = this._justQuery(query, values);
var rows = this._queryWithRetry(query, values);
return _.map(rows, function(entity) {
return JSON.parse(entity.content);
});

111
tools/catalog.js
View File

@@ -15,6 +15,11 @@ var utils = require('./utils.js');
var config = require('./config.js');
var packageClient = require('./package-client.js');
// The LayeredCatalog provides a way to query multiple catalogs in a uniform way
// A LayeredCatalog typically contains:
// - a local catalog referencing the packages of the project
// - a reference to the official catalog
var LayeredCatalog = function() {
var self = this;
@@ -77,7 +82,7 @@ _.extend(LayeredCatalog.prototype, {
},
getLoadPathForPackage: function (name, version, constraintSolverOpts) {
var self = this; //PASCAL check with Ekate
var self = this;
return self.localCatalog.getLoadPathForPackage(name, version, constraintSolverOpts);
},
@@ -95,7 +100,6 @@ _.extend(LayeredCatalog.prototype, {
getReleaseWithTool: function (toolSpec) {
var self = this;
buildmessage.assertInCapture();
self._requireInitialized();
return self._recordOrRefresh(function () {
return _.findWhere(self.releaseVersions, { tool: toolSpec });
});
@@ -136,12 +140,6 @@ _.extend(LayeredCatalog.prototype, {
return this.localCatalog.rebuildLocalPackages(namedPackages);
},
refreshInProgress: function () {
var self = this;
// console.log("refresh in progress the LayeredCatalog");
//PASCAL Deal with refresh properly
},
reset: function () {
this.localCatalog.reset();
},
@@ -151,7 +149,6 @@ _.extend(LayeredCatalog.prototype, {
// It does not include prereleases (with dashes in the version);
getLatestMainlineVersion: function (name) {
var self = this;
self._requireInitialized();
buildmessage.assertInCapture();
var versions = self.getSortedVersions(name);
@@ -164,18 +161,9 @@ _.extend(LayeredCatalog.prototype, {
return self.getVersion(name, latest);
},
// Throw if the catalog's self.initialized value has not been set to true.
_requireInitialized: function () {
var self = this;
//PASCAL
// if (! self.initialized)
// throw new Error("catalog not initialized yet?");
},
resolveConstraints : function (constraints, resolverOpts, opts) {
var self = this;
opts = opts || {};
self._requireInitialized();
buildmessage.assertInCapture();
// OK, since we are the complete catalog, the uniload catalog must be fully
@@ -244,7 +232,6 @@ _.extend(LayeredCatalog.prototype, {
} catch (e) {
// Maybe we only failed because we need to refresh. Try to refresh
// (unless we already are) and retry.
//PASCAL review
if (!self._refreshingIsProductive() ||
exports.official.refreshInProgress()) {
throw e;
@@ -285,10 +272,7 @@ _.extend(LayeredCatalog.prototype, {
return ret.answer;
},
// Refresh the packages in the catalog.
//
// Reread server data from data.json on disk, then load local overrides on top
// of that information. Sets initialized to true.
// Refresh the catalogs referenced by this catalog.
// options:
// - forceRefresh: even if there is a future in progress, refresh the catalog
// anyway. When we are using hot code push, we may be restarting the app
@@ -298,83 +282,10 @@ _.extend(LayeredCatalog.prototype, {
// to this set.
refresh: function (options) {
var self = this;
console.log("refresh layered catalo");
self.localCatalog.refresh(options);
self.otherCatalog.refresh(options);
self.packageCache.refresh();
self.resolver = null;
// options = options || {};
// buildmessage.assertInCapture();
// // We need to limit the rate of refresh, or, at least, prevent any sort of
// // loops. ForceRefresh will override either one.
// if (!options.forceRefresh && !options.initializing &&
// (catalog.official._refreshFutures || self.refreshing)) {
// return;
// }
// if (options.initializing && !self.forUniload) {
// // If we are doing the top level initialization in main.js, everything
// // sure had better be in a relaxed state, since we're about to hackily
// // steal some data from catalog.official.
// if (self.refreshing)
// throw Error("initializing catalog.complete re-entrantly?");
// if (catalog.official._refreshFutures)
// throw Error("initializing catalog.complete during official refresh?");
// }
// if (self.refreshing) {
// // We're being asked to refresh re-entrantly, maybe because we just
// // updated the official catalog. Let's not do this now, but make the
// // outer call do it instead.
// // XXX refactoring the catalogs so that the two catalogs share their
// // data and this one is just an overlay would reduce this wackiness
// self.needRefresh = true;
// return;
// }
// self.refreshing = true;
// try {
// self.reset();
// if (!self.forUniload) {
// if (options.initializing) {
// // It's our first time! Everything ought to be at rest. Let's just
// // steal data (without even a deep clone!) from catalog.official.
// // XXX this is horrible. restructure to have a reference to
// // catalog.official instead.
// self.packages = _.clone(catalog.official.packages);
// self.builds = _.clone(catalog.official.builds);
// _.each(catalog.official.versions, function (versions, name) {
// self.versions[name] = _.clone(versions);
// });
// } else {
// // Not the first time. Slowly load data from disk.
// // XXX restructure this class to just have a reference to
// // catalog.official instead of a copy of its data.
// var localData = packageClient.loadCachedServerData();
// self._insertServerPackages(localData);
// }
// }
// self._recomputeEffectiveLocalPackages();
// var allOK = self._addLocalPackageOverrides(
// { watchSet: options.watchSet });
// self.initialized = true;
// // Rebuild the resolver, since packages may have changed.
// self.resolver = null;
// } finally {
// self.refreshing = false;
// }f
// // If we got a re-entrant refresh request, do it now. (But not if we
// // encountered build errors building the packages, since in that case
// // we'd probably just get the same build errors again.)
// if (self.needRefresh && allOK) {
// self.refresh(options);
// }
},
_initializeResolver: function () {
@@ -396,13 +307,7 @@ _.extend(LayeredCatalog.prototype, {
watchLocalPackageDirs: function (watchSet) {
var self = this;
self.localCatalog.watchLocalPackageDirs(watchSet);
},
_refreshingIsProductive: function() {
//PASCAL REVIEW
return true;
}
}
});
exports.DEFAULT_TRACK = remoteCatalog.DEFAULT_TRACK;