github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

'meteor admin make-bootstrap-tarballs'

Browse Source 
other changes:

- defaultReleaseVersion is not synced using 'collections' any more,
  since it is a singleton and has different merge logic
- publish-release --fromCheckout renamed to --from-checkout
- $METEOR_SAVE_TMPDIRS env var to not delete files.mkdtemp stuff
- release.latestDownloaded() now comes from tropohouse

we recognized that the cross-linking stuff in
Tropohouse.maybeDownloadPackageForArchitectures doesn't quite work so we
changed it to not quite work in a different way (which allows us to
leave downloaded-builds out of the bootstrap tarball and decrease its
size by 50%).  will fix later.
This commit is contained in:
David Glasser
2014-05-13 17:03:20 -07:00
parent a9c944c4d9
commit 35bad76bc5
8 changed files with 222 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
tools/catalog.js
View File

@@ -219,13 +219,13 @@ _.extend(Catalog.prototype, {
allPackageData = packageClient.updateServerPackageData(localData);
if (! allPackageData) {
// If we couldn't contact the package server, use our local data.
allPackageData = localData.collections;
allPackageData = localData;
// XXX should do some nicer error handling here (return error to
// caller and let them handle it?)
process.stderr.write("Warning: could not connect to package server\n");
}
} else {
allPackageData = localData.collections;
allPackageData = localData;
}
self.initialized = false;
@@ -235,7 +235,7 @@ _.extend(Catalog.prototype, {
self.releaseTracks = [];
self.releaseVersions = [];
self.defaultReleaseVersion = null;
if (allPackageData) {
if (allPackageData && allPackageData.collections) {
self._insertServerPackages(allPackageData);
}
self._addLocalPackageOverrides(true /* setInitialized */);
@@ -554,14 +554,15 @@ _.extend(Catalog.prototype, {
_insertServerPackages: function (serverPackageData) {
var self = this;
self.packages.push.apply(self.packages, serverPackageData.packages);
self.versions.push.apply(self.versions, serverPackageData.versions);
self.builds.push.apply(self.builds, serverPackageData.builds);
self.releaseTracks.push.apply(self.releaseTracks, serverPackageData.releaseTracks);
self.releaseVersions.push.apply(self.releaseVersions, serverPackageData.releaseVersions);
if (serverPackageData.defaultReleaseVersions &&
serverPackageData.defaultReleaseVersions.length === 1) {
self.defaultReleaseVersion = serverPackageData.defaultReleaseVersions[0];
var collections = serverPackageData.collections;
_.each(
['packages', 'versions', 'builds', 'releaseTracks', 'releaseVersions'],
function (field) {
self[field].push.apply(self[field], collections[field]);
});
if (serverPackageData.defaultReleaseVersion) {
self.defaultReleaseVersion = serverPackageData.defaultReleaseVersion;
}
},
@@ -893,6 +894,27 @@ _.extend(Catalog.prototype, {
return _.findWhere(self.builds,
{ versionId: versionRecord._id,
architecture: archesString });
},
getAllBuilds: function (name, version) {
var self = this;
self._requireInitialized();
var versionRecord = self.getVersion(name, version);
if (!versionRecord)
return null;
return _.where(self.builds, { versionId: versionRecord._id });
},
getDefaultReleaseVersion: function () {
var self = this;
self._requireInitialized();
if (!self.defaultReleaseVersion) {
self.refresh(true);
}
return self.defaultReleaseVersion;
}
});

129
tools/commands.js
View File

@@ -25,6 +25,7 @@ var compiler = require('./compiler.js');
var catalog = require('./catalog.js').catalog;
var serverCatalog = require('./catalog.js').serverCatalog;
var stats = require('./stats.js');
var unipackage = require('./unipackage.js');
// Given a site name passed on the command line (eg, 'mysite'), return
// a fully-qualified hostname ('mysite.meteor.com').
@@ -1573,9 +1574,123 @@ main.registerCommand({
///////////////////////////////////////////////////////////////////////////////
main.registerCommand({
name: 'admin grant'
name: 'admin make-bootstrap-tarballs',
minArgs: 2,
maxArgs: 2,
hidden: true
}, function (options) {
process.stderr.write("'admin grant' command not implemented yet\n");
var releaseNameAndVersion = options.args[0];
var outputDirectory = options.args[1];
serverCatalog.refresh(true);
var parsed = utils.splitConstraint(releaseNameAndVersion);
if (!parsed.constraint)
throw new main.ShowUsage;
var release = serverCatalog.getReleaseVersion(parsed.package,
parsed.constraint);
if (!release) {
// XXX this could also mean package unknown.
process.stderr.write('Release unknown: ' + releaseNameAndVersion + '\n');
return 1;
}
var toolPkg = release.tool && utils.splitConstraint(release.tool);
if (! (toolPkg && toolPkg.constraint))
throw new Error("bad tool in release: " + toolPkg);
var toolPkgBuilds = serverCatalog.getAllBuilds(
toolPkg.package, toolPkg.constraint);
if (!toolPkgBuilds) {
// XXX this could also mean package unknown.
process.stderr.write('Tool version unknown: ' + release.tool + '\n');
return 1;
}
if (!toolPkgBuilds.length) {
process.stderr.write('Tool version has no builds: ' + release.tool + '\n');
return 1;
}
// XXX check to make sure this is the three arches that we want? it's easier
// during 0.9.0 development to allow it to just decide "ok, i just want to
// build the OSX tarball" though.
var buildArches = _.pluck(toolPkgBuilds, 'architecture');
var osArches = _.map(buildArches, function (buildArch) {
var subArches = buildArch.split('+');
var osArches = _.filter(subArches, function (subArch) {
return subArch.substr(0, 3) === 'os.';
});
if (osArches.length !== 1) {
throw Error("build architecture " + buildArch + " lacks unique os.*");
}
return osArches[0];
});
process.stderr.write(
'Building bootstrap tarballs for architectures ' +
osArches.join(', ') + '\n');
// Before downloading anything, check that the catalog contains everything we
// need for the OSes that the tool is built for.
_.each(osArches, function (osArch) {
_.each(release.packages, function (pkgVersion, pkgName) {
if (!serverCatalog.getBuildsForArches(pkgName, pkgVersion, [osArch])) {
throw Error("missing build of " + pkgName + "@" + pkgVersion +
" for " + osArch);
}
});
});
files.mkdir_p(outputDirectory);
_.each(osArches, function (osArch) {
var tmpdir = files.mkdtemp();
// We're going to build and tar up a tropohouse in a temporary directory; we
// don't want to use any of our local packages, so we use serverCatalog
// instead of catalog.
// XXX update to '.meteor' when we combine houses
var tmpTropo = new tropohouse.Tropohouse(
path.join(tmpdir, '.meteor0'), serverCatalog);
tmpTropo.maybeDownloadPackageForArchitectures(
{packageName: toolPkg.package, version: toolPkg.constraint},
[osArch]); // XXX 'browser' too?
_.each(release.packages, function (pkgVersion, pkgName) {
tmpTropo.maybeDownloadPackageForArchitectures(
{packageName: pkgName, version: pkgVersion},
[osArch]); // XXX 'browser' too?
});
// Delete the downloaded-builds directory which basically just has a second
// copy of everything. I think it's OK if the first time we try to deploy
// to Linux from Mac, it has to download a bunch of stuff. (Alternatively,
// we could actually always include Linux64 in the bootstrap tarball, but
// meh.)
// XXX it's not like cross-linking even works yet anyway
files.rm_recursive(path.join(tmpTropo.root, 'downloaded-builds'));
// XXX should we include some sort of preliminary package-metadata as well?
// maybe with a defaultReleaseVersion of the release we are using?
// Create the top-level 'meteor' symlink, which links to the latest tool's
// meteor shell script.
var toolUnipackagePath =
tmpTropo.packagePath(toolPkg.package, toolPkg.constraint);
var toolUnipackage = new unipackage.Unipackage;
toolUnipackage.initFromPath(toolPkg.package, toolUnipackagePath);
var toolRecord = _.findWhere(toolUnipackage.toolsOnDisk, {arch: osArch});
if (!toolRecord)
throw Error("missing tool for " + osArch);
fs.symlinkSync(
path.join(
tmpTropo.packagePath(toolPkg.package, toolPkg.constraint, true),
toolRecord.path,
'meteor'),
path.join(tmpTropo.root, 'meteor'));
files.createTarball(
tmpTropo.root,
path.join(outputDirectory, 'meteor-bootstrap-' + osArch + '.tar.gz'));
});
return 0;
});
@@ -1763,10 +1878,10 @@ main.registerCommand({
// such as the version lock file, something has gone terribly wrong and we
// should throw.
packageSource.initFromPackageDir(options.name, packageDir, true /* immutable */);
var unipackage = compiler.compile(packageSource, {
var unipkg = compiler.compile(packageSource, {
officialBuild: true
}).unipackage;
unipackage.saveToPath(path.join(packageDir, '.build.' + packageSource.name));
unipkg.saveToPath(path.join(packageDir, '.build.' + packageSource.name));
var conn;
try {
@@ -1787,7 +1902,7 @@ main.registerCommand({
options: {
config: { type: String, required: true },
create: { type: Boolean, required: false },
fromCheckout: { type: Boolean, required: false }
'from-checkout': { type: Boolean, required: false }
}
}, function (options) {
@@ -1832,7 +1947,7 @@ main.registerCommand({
// option is not very useful outside of MDG. Right now, to run this option on
// a non-MDG fork of meteor, someone would probably need to go through and
// change the package names to have proper prefixes, etc.
if (options.fromCheckout) {
if (options['from-checkout']) {
// You must be running from checkout to bundle up your checkout as a release.
if (!files.inCheckout()) {
process.stderr.write("Must run from checkout to make release from checkout. \n");
@@ -1856,7 +1971,7 @@ main.registerCommand({
// these by accident. So, we will disallow it for now.
if (relConf.packages || relConf.tool) {
process.stderr.write(
"Setting the --fromCheckout option will use the tool & packages in your meteor " +
"Setting the --from-checkout option will use the tool & packages in your meteor " +
"checkout. \n" +
"Your release configuration file should not contain that information.");
return 1;

10
tools/files.js
View File

@@ -471,10 +471,12 @@ files.mkdtemp = function (prefix) {
return dir;
};
cleanup.onExit(function (sig) {
_.each(tempDirs, files.rm_recursive);
tempDirs = [];
});
if (!process.env.METEOR_SAVE_TMPDIRS) {
cleanup.onExit(function (sig) {
_.each(tempDirs, files.rm_recursive);
tempDirs = [];
});
}
// Takes a buffer containing `.tar.gz` data and extracts the archive
// into a destination directory. destPath should not exist yet, and

8
tools/help.txt
View File

@@ -341,11 +341,11 @@ Options:
--changed A boolean option.
>>> admin grant
Grant a permission on an official service
Usage: meteor admin grant [XXX]
>>> admin make-bootstrap-tarballs
Makes bootstrap tarballs.
Usage: meteor admin make-bootstrap-tarballs release@version /tmp/tarballdir
Not yet implemented
For internal use only.
>>> publish

31
tools/package-client.js
View File

@@ -134,28 +134,15 @@ var mergeCollections = function (sources) {
return ret;
};
// Writes the cached package data to the on-disk cache. Takes in the following
// arguments:
// - syncToken : the token representing our conversation with the server, that
// we can later use to get a diff of this cache and the new server-side data.
// - collectionData : a javascript object representing the data we have about
// packages on the server, with collection names as keys and arrays of those
// collection records as values.
// Writes the cached package data to the on-disk cache.
//
// Returns nothing, but
// XXXX: Does what on errors?
var writePackageDataToDisk = function (syncToken, collectionData) {
var finalWrite = {};
finalWrite.syncToken = syncToken;
finalWrite.formatVersion = "1.0";
finalWrite.collections = {};
_.forEach(collectionData, function(coll, name) {
finalWrite.collections[name] = coll;
});
var writePackageDataToDisk = function (syncToken, data) {
var filename = config.getPackageStorage();
// XXX think about permissions?
files.mkdir_p(path.dirname(filename));
files.writeFileAtomically(filename, JSON.stringify(finalWrite, null, 2));
files.writeFileAtomically(filename, JSON.stringify(data, null, 2));
};
// Contacts the package server to get the latest diff and writes changes to
@@ -185,9 +172,15 @@ exports.updateServerPackageData = function (cachedServerData) {
}
sources.push(remoteData.collections);
var allPackageData = mergeCollections(sources);
writePackageDataToDisk(remoteData.syncToken, allPackageData);
return allPackageData;
var allCollections = mergeCollections(sources);
var data = {
syncToken: remoteData.syncToken,
formatVersion: "1.0",
defaultReleaseVersion: remoteData.defaultReleaseVersion,
collections: allCollections
};
writePackageDataToDisk(remoteData.syncToken, data);
return data;
};
// Returns a logged-in DDP connection to the package server, or null if

11
tools/release.js
View File

@@ -160,16 +160,17 @@ release.usingRightReleaseForApp = function (appDir) {
// Return the name of the latest release that is downloaded and ready
// for use. May not be called when running from a checkout.
release.latestDownloaded = function () {
// XXX update this for tropohouse
if (! files.usesWarehouse())
throw new Error("called from checkout?");
// For self-test only.
if (process.env.METEOR_TEST_LATEST_RELEASE)
return process.env.METEOR_TEST_LATEST_RELEASE;
var ret = warehouse.latestRelease();
if (! ret)
throw new Error("no releases available?");
return ret;
var defaultRelease = catalog.serverCatalog.getDefaultReleaseVersion();
if (!defaultRelease) {
throw new Error("no latest release available?");
}
return defaultRelease.name + '@' + defaultRelease.version;
};
// Load a release and return it as a Release object without setting

70
tools/tropohouse.js
View File

@@ -10,12 +10,13 @@ var httpHelpers = require('./http-helpers.js');
var fiberHelpers = require('./fiber-helpers.js');
var release = require('./release.js');
var archinfo = require('./archinfo.js');
var catalog = require('./catalog.js').catalog;
var catalog = require('./catalog.js');
var Unipackage = require('./unipackage.js').Unipackage;
exports.Tropohouse = function (root) {
exports.Tropohouse = function (root, catalog) {
var self = this;
self.root = root;
self.catalog = catalog;
};
// Return the directory containing our loaded collection of tools, releases and
@@ -34,7 +35,11 @@ var defaultWarehouseDir = function () {
return path.join(warehouseBase, ".meteor0");
};
exports.default = new exports.Tropohouse(defaultWarehouseDir());
// The default tropohouse is on disk at defaultWarehouseDir() and knows not to
// download local packages; you can make your own Tropohouse to override these
// things.
exports.default = new exports.Tropohouse(
defaultWarehouseDir(), catalog.catalog);
_.extend(exports.Tropohouse.prototype, {
// Return the directory within the warehouse that would contain downloaded
@@ -86,14 +91,14 @@ _.extend(exports.Tropohouse.prototype, {
// check for contents.
//
// Returns null if the package name is lexographically invalid.
packagePath: function (packageName, version) {
packagePath: function (packageName, version, relative) {
var self = this;
if (! utils.validPackageName(packageName)) {
return null;
}
var loadPath = path.join(self.root, "packages", packageName, version);
return loadPath;
var relativePath = path.join("packages", packageName, version);
return relative ? relativePath : path.join(self.root, relativePath);
},
// Contacts the package server, downloads and extracts a tarball for a given
@@ -120,8 +125,10 @@ _.extend(exports.Tropohouse.prototype, {
//
// XXX more precise error handling in offline case. maybe throw instead like
// warehouse does.
maybeDownloadPackageForArchitectures: function (versionInfo, requiredArches,
justGetBuilds) {
//
// XXX this is kinda bogus right now and needs to be fixed when we actually
// get around to implement cross-linking (which is the point)
maybeDownloadPackageForArchitectures: function (versionInfo, requiredArches) {
var self = this;
var packageName = versionInfo.packageName;
var version = versionInfo.version;
@@ -129,9 +136,19 @@ _.extend(exports.Tropohouse.prototype, {
// If this package isn't coming from the package server (loaded from a
// checkout, or from an app package directory), don't try to download it (we
// already have it)
if (catalog.isLocalPackage(packageName))
if (self.catalog.isLocalPackage(packageName))
return true;
var packageDir = self.packagePath(packageName, version);
if (fs.existsSync(packageDir)) {
// Package exists for this build, so we are good.
// XXX this doesn't actually work! the point of this whole thing is that
// you can fat-ify a package (eg, at deploy time) but this here assumes
// that once you write a package you'll never write it again.
return true;
}
// Figure out what arches (if any) we have downloaded for this package
// version already.
var downloadedArches = self.downloadedArches(packageName, version);
@@ -140,7 +157,7 @@ _.extend(exports.Tropohouse.prototype, {
});
if (archesToDownload.length) {
var buildsToDownload = catalog.getBuildsForArches(
var buildsToDownload = self.catalog.getBuildsForArches(
packageName, version, archesToDownload);
if (! buildsToDownload) {
// XXX throw a special error instead?
@@ -150,30 +167,21 @@ _.extend(exports.Tropohouse.prototype, {
// XXX how does concurrency work here? we could just get errors if we try
// to rename over the other thing? but that's the same as in warehouse?
_.each(buildsToDownload, function (build) {
self.downloadSpecifiedBuild(build);
self.downloadSpecifiedBuild(build);
});
}
if (justGetBuilds) {
return; // XXX use this return value when we actually call it
}
var packageDir = self.packagePath(packageName, version);
if (fs.existsSync(packageDir)) {
// Package exists for this build, so we are good.
} else {
// We need to turn our builds into a unipackage.
var unipackage = new Unipackage;
var builds = self.downloadedBuilds(packageName, version);
_.each(builds, function (build, i) {
unipackage._loadBuildsFromPath(
packageName,
self.downloadedBuildPath(packageName, version, build),
{firstUnipackage: i === 0});
});
// XXX save new buildinfo stuff so it auto-rebuilds
unipackage.saveToPath(packageDir);
}
// We need to turn our builds into a unipackage.
var unipackage = new Unipackage;
var builds = self.downloadedBuilds(packageName, version);
_.each(builds, function (build, i) {
unipackage._loadBuildsFromPath(
packageName,
self.downloadedBuildPath(packageName, version, build),
{firstUnipackage: i === 0});
});
// XXX save new buildinfo stuff so it auto-rebuilds
unipackage.saveToPath(packageDir);
return true;
}

2
tools/utils.js
View File

@@ -216,8 +216,6 @@ exports.parseConstraint = function (constraintString) {
// XXX should unify this with utils.parseConstraint
exports.splitConstraint = function (constraint) {
var m = constraint.split("@");
if (! m)
throw new Error("Bad package spec: " + constraint);
var ret = { package: m[0] };
if (m.length > 1) {
ret.constraint = m[1];