diff --git a/tools/catalog.js b/tools/catalog.js index 2ac84a0bd2..06606c0297 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -219,13 +219,13 @@ _.extend(Catalog.prototype, { allPackageData = packageClient.updateServerPackageData(localData); if (! allPackageData) { // If we couldn't contact the package server, use our local data. - allPackageData = localData.collections; + allPackageData = localData; // XXX should do some nicer error handling here (return error to // caller and let them handle it?) process.stderr.write("Warning: could not connect to package server\n"); } } else { - allPackageData = localData.collections; + allPackageData = localData; } self.initialized = false; @@ -235,7 +235,7 @@ _.extend(Catalog.prototype, { self.releaseTracks = []; self.releaseVersions = []; self.defaultReleaseVersion = null; - if (allPackageData) { + if (allPackageData && allPackageData.collections) { self._insertServerPackages(allPackageData); } self._addLocalPackageOverrides(true /* setInitialized */); @@ -554,14 +554,15 @@ _.extend(Catalog.prototype, { _insertServerPackages: function (serverPackageData) { var self = this; - self.packages.push.apply(self.packages, serverPackageData.packages); - self.versions.push.apply(self.versions, serverPackageData.versions); - self.builds.push.apply(self.builds, serverPackageData.builds); - self.releaseTracks.push.apply(self.releaseTracks, serverPackageData.releaseTracks); - self.releaseVersions.push.apply(self.releaseVersions, serverPackageData.releaseVersions); - if (serverPackageData.defaultReleaseVersions && - serverPackageData.defaultReleaseVersions.length === 1) { - self.defaultReleaseVersion = serverPackageData.defaultReleaseVersions[0]; + var collections = serverPackageData.collections; + + _.each( + ['packages', 'versions', 'builds', 'releaseTracks', 'releaseVersions'], + function (field) { + self[field].push.apply(self[field], collections[field]); + }); + if (serverPackageData.defaultReleaseVersion) { + self.defaultReleaseVersion = serverPackageData.defaultReleaseVersion; } }, @@ -893,6 +894,27 @@ _.extend(Catalog.prototype, { return _.findWhere(self.builds, { versionId: versionRecord._id, architecture: archesString }); + }, + + getAllBuilds: function (name, version) { + var self = this; + self._requireInitialized(); + + var versionRecord = self.getVersion(name, version); + if (!versionRecord) + return null; + + return _.where(self.builds, { versionId: versionRecord._id }); + }, + + getDefaultReleaseVersion: function () { + var self = this; + self._requireInitialized(); + + if (!self.defaultReleaseVersion) { + self.refresh(true); + } + return self.defaultReleaseVersion; } }); diff --git a/tools/commands.js b/tools/commands.js index 37995d7621..4d54e8f7a0 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -25,6 +25,7 @@ var compiler = require('./compiler.js'); var catalog = require('./catalog.js').catalog; var serverCatalog = require('./catalog.js').serverCatalog; var stats = require('./stats.js'); +var unipackage = require('./unipackage.js'); // Given a site name passed on the command line (eg, 'mysite'), return // a fully-qualified hostname ('mysite.meteor.com'). @@ -1573,9 +1574,123 @@ main.registerCommand({ /////////////////////////////////////////////////////////////////////////////// main.registerCommand({ - name: 'admin grant' + name: 'admin make-bootstrap-tarballs', + minArgs: 2, + maxArgs: 2, + hidden: true }, function (options) { - process.stderr.write("'admin grant' command not implemented yet\n"); + var releaseNameAndVersion = options.args[0]; + var outputDirectory = options.args[1]; + + serverCatalog.refresh(true); + + var parsed = utils.splitConstraint(releaseNameAndVersion); + if (!parsed.constraint) + throw new main.ShowUsage; + + var release = serverCatalog.getReleaseVersion(parsed.package, + parsed.constraint); + if (!release) { + // XXX this could also mean package unknown. + process.stderr.write('Release unknown: ' + releaseNameAndVersion + '\n'); + return 1; + } + + var toolPkg = release.tool && utils.splitConstraint(release.tool); + if (! (toolPkg && toolPkg.constraint)) + throw new Error("bad tool in release: " + toolPkg); + var toolPkgBuilds = serverCatalog.getAllBuilds( + toolPkg.package, toolPkg.constraint); + if (!toolPkgBuilds) { + // XXX this could also mean package unknown. + process.stderr.write('Tool version unknown: ' + release.tool + '\n'); + return 1; + } + if (!toolPkgBuilds.length) { + process.stderr.write('Tool version has no builds: ' + release.tool + '\n'); + return 1; + } + + // XXX check to make sure this is the three arches that we want? it's easier + // during 0.9.0 development to allow it to just decide "ok, i just want to + // build the OSX tarball" though. + var buildArches = _.pluck(toolPkgBuilds, 'architecture'); + var osArches = _.map(buildArches, function (buildArch) { + var subArches = buildArch.split('+'); + var osArches = _.filter(subArches, function (subArch) { + return subArch.substr(0, 3) === 'os.'; + }); + if (osArches.length !== 1) { + throw Error("build architecture " + buildArch + " lacks unique os.*"); + } + return osArches[0]; + }); + + process.stderr.write( + 'Building bootstrap tarballs for architectures ' + + osArches.join(', ') + '\n'); + // Before downloading anything, check that the catalog contains everything we + // need for the OSes that the tool is built for. + _.each(osArches, function (osArch) { + _.each(release.packages, function (pkgVersion, pkgName) { + if (!serverCatalog.getBuildsForArches(pkgName, pkgVersion, [osArch])) { + throw Error("missing build of " + pkgName + "@" + pkgVersion + + " for " + osArch); + } + }); + }); + + files.mkdir_p(outputDirectory); + + _.each(osArches, function (osArch) { + var tmpdir = files.mkdtemp(); + // We're going to build and tar up a tropohouse in a temporary directory; we + // don't want to use any of our local packages, so we use serverCatalog + // instead of catalog. + // XXX update to '.meteor' when we combine houses + var tmpTropo = new tropohouse.Tropohouse( + path.join(tmpdir, '.meteor0'), serverCatalog); + tmpTropo.maybeDownloadPackageForArchitectures( + {packageName: toolPkg.package, version: toolPkg.constraint}, + [osArch]); // XXX 'browser' too? + _.each(release.packages, function (pkgVersion, pkgName) { + tmpTropo.maybeDownloadPackageForArchitectures( + {packageName: pkgName, version: pkgVersion}, + [osArch]); // XXX 'browser' too? + }); + + // Delete the downloaded-builds directory which basically just has a second + // copy of everything. I think it's OK if the first time we try to deploy + // to Linux from Mac, it has to download a bunch of stuff. (Alternatively, + // we could actually always include Linux64 in the bootstrap tarball, but + // meh.) + // XXX it's not like cross-linking even works yet anyway + files.rm_recursive(path.join(tmpTropo.root, 'downloaded-builds')); + + // XXX should we include some sort of preliminary package-metadata as well? + // maybe with a defaultReleaseVersion of the release we are using? + + // Create the top-level 'meteor' symlink, which links to the latest tool's + // meteor shell script. + var toolUnipackagePath = + tmpTropo.packagePath(toolPkg.package, toolPkg.constraint); + var toolUnipackage = new unipackage.Unipackage; + toolUnipackage.initFromPath(toolPkg.package, toolUnipackagePath); + var toolRecord = _.findWhere(toolUnipackage.toolsOnDisk, {arch: osArch}); + if (!toolRecord) + throw Error("missing tool for " + osArch); + fs.symlinkSync( + path.join( + tmpTropo.packagePath(toolPkg.package, toolPkg.constraint, true), + toolRecord.path, + 'meteor'), + path.join(tmpTropo.root, 'meteor')); + + files.createTarball( + tmpTropo.root, + path.join(outputDirectory, 'meteor-bootstrap-' + osArch + '.tar.gz')); + }); + return 0; }); @@ -1763,10 +1878,10 @@ main.registerCommand({ // such as the version lock file, something has gone terribly wrong and we // should throw. packageSource.initFromPackageDir(options.name, packageDir, true /* immutable */); - var unipackage = compiler.compile(packageSource, { + var unipkg = compiler.compile(packageSource, { officialBuild: true }).unipackage; - unipackage.saveToPath(path.join(packageDir, '.build.' + packageSource.name)); + unipkg.saveToPath(path.join(packageDir, '.build.' + packageSource.name)); var conn; try { @@ -1787,7 +1902,7 @@ main.registerCommand({ options: { config: { type: String, required: true }, create: { type: Boolean, required: false }, - fromCheckout: { type: Boolean, required: false } + 'from-checkout': { type: Boolean, required: false } } }, function (options) { @@ -1832,7 +1947,7 @@ main.registerCommand({ // option is not very useful outside of MDG. Right now, to run this option on // a non-MDG fork of meteor, someone would probably need to go through and // change the package names to have proper prefixes, etc. - if (options.fromCheckout) { + if (options['from-checkout']) { // You must be running from checkout to bundle up your checkout as a release. if (!files.inCheckout()) { process.stderr.write("Must run from checkout to make release from checkout. \n"); @@ -1856,7 +1971,7 @@ main.registerCommand({ // these by accident. So, we will disallow it for now. if (relConf.packages || relConf.tool) { process.stderr.write( - "Setting the --fromCheckout option will use the tool & packages in your meteor " + + "Setting the --from-checkout option will use the tool & packages in your meteor " + "checkout. \n" + "Your release configuration file should not contain that information."); return 1; diff --git a/tools/files.js b/tools/files.js index e9b777e6a5..519e248b98 100644 --- a/tools/files.js +++ b/tools/files.js @@ -471,10 +471,12 @@ files.mkdtemp = function (prefix) { return dir; }; -cleanup.onExit(function (sig) { - _.each(tempDirs, files.rm_recursive); - tempDirs = []; -}); +if (!process.env.METEOR_SAVE_TMPDIRS) { + cleanup.onExit(function (sig) { + _.each(tempDirs, files.rm_recursive); + tempDirs = []; + }); +} // Takes a buffer containing `.tar.gz` data and extracts the archive // into a destination directory. destPath should not exist yet, and diff --git a/tools/help.txt b/tools/help.txt index 329779552c..d965065895 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -341,11 +341,11 @@ Options: --changed A boolean option. ->>> admin grant -Grant a permission on an official service -Usage: meteor admin grant [XXX] +>>> admin make-bootstrap-tarballs +Makes bootstrap tarballs. +Usage: meteor admin make-bootstrap-tarballs release@version /tmp/tarballdir -Not yet implemented +For internal use only. >>> publish diff --git a/tools/package-client.js b/tools/package-client.js index 82bb558cbf..d3d64e691b 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -134,28 +134,15 @@ var mergeCollections = function (sources) { return ret; }; -// Writes the cached package data to the on-disk cache. Takes in the following -// arguments: -// - syncToken : the token representing our conversation with the server, that -// we can later use to get a diff of this cache and the new server-side data. -// - collectionData : a javascript object representing the data we have about -// packages on the server, with collection names as keys and arrays of those -// collection records as values. +// Writes the cached package data to the on-disk cache. // // Returns nothing, but // XXXX: Does what on errors? -var writePackageDataToDisk = function (syncToken, collectionData) { - var finalWrite = {}; - finalWrite.syncToken = syncToken; - finalWrite.formatVersion = "1.0"; - finalWrite.collections = {}; - _.forEach(collectionData, function(coll, name) { - finalWrite.collections[name] = coll; - }); +var writePackageDataToDisk = function (syncToken, data) { var filename = config.getPackageStorage(); // XXX think about permissions? files.mkdir_p(path.dirname(filename)); - files.writeFileAtomically(filename, JSON.stringify(finalWrite, null, 2)); + files.writeFileAtomically(filename, JSON.stringify(data, null, 2)); }; // Contacts the package server to get the latest diff and writes changes to @@ -185,9 +172,15 @@ exports.updateServerPackageData = function (cachedServerData) { } sources.push(remoteData.collections); - var allPackageData = mergeCollections(sources); - writePackageDataToDisk(remoteData.syncToken, allPackageData); - return allPackageData; + var allCollections = mergeCollections(sources); + var data = { + syncToken: remoteData.syncToken, + formatVersion: "1.0", + defaultReleaseVersion: remoteData.defaultReleaseVersion, + collections: allCollections + }; + writePackageDataToDisk(remoteData.syncToken, data); + return data; }; // Returns a logged-in DDP connection to the package server, or null if diff --git a/tools/release.js b/tools/release.js index cd9fc9716a..f4d6819fb0 100644 --- a/tools/release.js +++ b/tools/release.js @@ -160,16 +160,17 @@ release.usingRightReleaseForApp = function (appDir) { // Return the name of the latest release that is downloaded and ready // for use. May not be called when running from a checkout. release.latestDownloaded = function () { - // XXX update this for tropohouse if (! files.usesWarehouse()) throw new Error("called from checkout?"); // For self-test only. if (process.env.METEOR_TEST_LATEST_RELEASE) return process.env.METEOR_TEST_LATEST_RELEASE; - var ret = warehouse.latestRelease(); - if (! ret) - throw new Error("no releases available?"); - return ret; + + var defaultRelease = catalog.serverCatalog.getDefaultReleaseVersion(); + if (!defaultRelease) { + throw new Error("no latest release available?"); + } + return defaultRelease.name + '@' + defaultRelease.version; }; // Load a release and return it as a Release object without setting diff --git a/tools/tropohouse.js b/tools/tropohouse.js index 5e8a166e58..24e5fe3fe8 100644 --- a/tools/tropohouse.js +++ b/tools/tropohouse.js @@ -10,12 +10,13 @@ var httpHelpers = require('./http-helpers.js'); var fiberHelpers = require('./fiber-helpers.js'); var release = require('./release.js'); var archinfo = require('./archinfo.js'); -var catalog = require('./catalog.js').catalog; +var catalog = require('./catalog.js'); var Unipackage = require('./unipackage.js').Unipackage; -exports.Tropohouse = function (root) { +exports.Tropohouse = function (root, catalog) { var self = this; self.root = root; + self.catalog = catalog; }; // Return the directory containing our loaded collection of tools, releases and @@ -34,7 +35,11 @@ var defaultWarehouseDir = function () { return path.join(warehouseBase, ".meteor0"); }; -exports.default = new exports.Tropohouse(defaultWarehouseDir()); +// The default tropohouse is on disk at defaultWarehouseDir() and knows not to +// download local packages; you can make your own Tropohouse to override these +// things. +exports.default = new exports.Tropohouse( + defaultWarehouseDir(), catalog.catalog); _.extend(exports.Tropohouse.prototype, { // Return the directory within the warehouse that would contain downloaded @@ -86,14 +91,14 @@ _.extend(exports.Tropohouse.prototype, { // check for contents. // // Returns null if the package name is lexographically invalid. - packagePath: function (packageName, version) { + packagePath: function (packageName, version, relative) { var self = this; if (! utils.validPackageName(packageName)) { return null; } - var loadPath = path.join(self.root, "packages", packageName, version); - return loadPath; + var relativePath = path.join("packages", packageName, version); + return relative ? relativePath : path.join(self.root, relativePath); }, // Contacts the package server, downloads and extracts a tarball for a given @@ -120,8 +125,10 @@ _.extend(exports.Tropohouse.prototype, { // // XXX more precise error handling in offline case. maybe throw instead like // warehouse does. - maybeDownloadPackageForArchitectures: function (versionInfo, requiredArches, - justGetBuilds) { + // + // XXX this is kinda bogus right now and needs to be fixed when we actually + // get around to implement cross-linking (which is the point) + maybeDownloadPackageForArchitectures: function (versionInfo, requiredArches) { var self = this; var packageName = versionInfo.packageName; var version = versionInfo.version; @@ -129,9 +136,19 @@ _.extend(exports.Tropohouse.prototype, { // If this package isn't coming from the package server (loaded from a // checkout, or from an app package directory), don't try to download it (we // already have it) - if (catalog.isLocalPackage(packageName)) + if (self.catalog.isLocalPackage(packageName)) return true; + var packageDir = self.packagePath(packageName, version); + if (fs.existsSync(packageDir)) { + // Package exists for this build, so we are good. + + // XXX this doesn't actually work! the point of this whole thing is that + // you can fat-ify a package (eg, at deploy time) but this here assumes + // that once you write a package you'll never write it again. + return true; + } + // Figure out what arches (if any) we have downloaded for this package // version already. var downloadedArches = self.downloadedArches(packageName, version); @@ -140,7 +157,7 @@ _.extend(exports.Tropohouse.prototype, { }); if (archesToDownload.length) { - var buildsToDownload = catalog.getBuildsForArches( + var buildsToDownload = self.catalog.getBuildsForArches( packageName, version, archesToDownload); if (! buildsToDownload) { // XXX throw a special error instead? @@ -150,30 +167,21 @@ _.extend(exports.Tropohouse.prototype, { // XXX how does concurrency work here? we could just get errors if we try // to rename over the other thing? but that's the same as in warehouse? _.each(buildsToDownload, function (build) { - self.downloadSpecifiedBuild(build); + self.downloadSpecifiedBuild(build); }); } - if (justGetBuilds) { - return; // XXX use this return value when we actually call it - } - - var packageDir = self.packagePath(packageName, version); - if (fs.existsSync(packageDir)) { - // Package exists for this build, so we are good. - } else { - // We need to turn our builds into a unipackage. - var unipackage = new Unipackage; - var builds = self.downloadedBuilds(packageName, version); - _.each(builds, function (build, i) { - unipackage._loadBuildsFromPath( - packageName, - self.downloadedBuildPath(packageName, version, build), - {firstUnipackage: i === 0}); - }); - // XXX save new buildinfo stuff so it auto-rebuilds - unipackage.saveToPath(packageDir); - } + // We need to turn our builds into a unipackage. + var unipackage = new Unipackage; + var builds = self.downloadedBuilds(packageName, version); + _.each(builds, function (build, i) { + unipackage._loadBuildsFromPath( + packageName, + self.downloadedBuildPath(packageName, version, build), + {firstUnipackage: i === 0}); + }); + // XXX save new buildinfo stuff so it auto-rebuilds + unipackage.saveToPath(packageDir); return true; } diff --git a/tools/utils.js b/tools/utils.js index bc3e476f97..9eea3daed6 100644 --- a/tools/utils.js +++ b/tools/utils.js @@ -216,8 +216,6 @@ exports.parseConstraint = function (constraintString) { // XXX should unify this with utils.parseConstraint exports.splitConstraint = function (constraint) { var m = constraint.split("@"); - if (! m) - throw new Error("Bad package spec: " + constraint); var ret = { package: m[0] }; if (m.length > 1) { ret.constraint = m[1];