From c53e85d3330dceebd2bca412d4f528dee258c13b Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Sun, 26 Jan 2014 20:15:34 -0800 Subject: [PATCH] warehouse sandboxing - WIP --- tools/commands.js | 5 +- tools/library.js | 6 ++ tools/selftest.js | 185 ++++++++++++++++++++++++++++++++---- tools/selftests/releases.js | 15 +++ 4 files changed, 192 insertions(+), 19 deletions(-) create mode 100644 tools/selftests/releases.js diff --git a/tools/commands.js b/tools/commands.js index 9d3a0bda62..68ff73c501 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -129,9 +129,8 @@ main.registerCommand({ if (! packages) return 1; // build failed - _.each(packages, function (p) { - p.preheat(); - }); + // XXX we rely on the fact that library.list() forces all of the + // packages to be built. #ListingPackagesImpliesBuildingThem } }); diff --git a/tools/library.js b/tools/library.js index f57bd29480..e215a53e38 100644 --- a/tools/library.js +++ b/tools/library.js @@ -331,6 +331,12 @@ _.extend(Library.prototype, { // // On failure, returns an object with keys: // - messages: a buildmessage.MessageSet with the errors + // + // XXX various callers currently rely on the fact that calling + // list() forces all of the packages in the library to be built! + // They shouldn't do that; they should instead call build() + // themselves if they want the packages + // built. #ListingPackagesImpliesBuildingThem list: function () { var self = this; var names = []; diff --git a/tools/selftest.js b/tools/selftest.js index d815e915cc..74b157b801 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -224,20 +224,41 @@ _.extend(OutputLog.prototype, { // meteor install or checkout, from the user's homedir, and from the // state of any other sandbox. // -// XXX let you pick whether you want it to look like a checkout or a -// release +// This will throw TestFailure if it has to build packages to set up +// the sandbox and the build fails. So, only call it from inside +// tests. // // options: // - app: if omitted, runs created in this sandbox aren't inside an // app directory. if present, a test app is created by copying the // named template from tools/selftests/apps, and runs created inside // the sandbox are inside that test app. XXX implement - +// - warehouse: set to sandbox the warehouse too. If you don't do +// this, the tests are run in the same context (checkout or +// warehouse) as the actual copy of meteor you're running (the +// meteor in 'meteor self-test'. This may only be set when you're +// running 'meteor self-test' from a checkout. If it is set, it +// should look something like this: +// { +// version1: { tools: 'tools1', notices: (...) }, +// version2: { tools: 'tools2', upgraders: ["a"], +// notices: (...), latest: true } +// } +// This would set up a simulated warehouse with two releases in it, +// one called 'version1' and having a tools version of 'tools1', and +// similarly with 'version2'/'tools2', with the latter being marked +// as the latest release, and the latter also having a single +// upgrader named "a". The releases are made by building the +// checkout into a release, and are identical except for their +// version names. If you pass 'notices' (which is optional), set it +// to the verbatim contents of the notices.json file for the +// release, as an object. var Sandbox = function (options) { var self = this; options = options || {}; self.root = files.mkdtemp(); self.cwd = path.join(self.root, 'cwd'); + self.warehouse = null; if (_.has(options, 'app')) { files.cp_r(path.join(__dirname, 'selftests', 'apps', options.app), @@ -245,6 +266,84 @@ var Sandbox = function (options) { } else { fs.mkdirSync(self.cwd, 0755); } + + if (_.has(options, 'warehouse')) { + // Make a directory to hold our new warehouse + self.warehouse = path.join(self.root, 'warehouse'); + console.log(self.warehouse); + fs.mkdirSync(self.warehouse, 0755); + fs.mkdirSync(path.join(self.warehouse, 'releases'), 0755); + fs.mkdirSync(path.join(self.warehouse, 'tools'), 0755); + fs.mkdirSync(path.join(self.warehouse, 'packages'), 0755); + + // Build all packages and symlink them into the warehouse. Make up + // random version names for each one. + var listResult = release.current.library.list(); + var pkgVersions = {}; + if (! listResult.packages) + throw new TestFailure('build-failure', { messages: listResult.messages }); + var packages = listResult.packages; + _.each(packages, function (pkg, name) { + // XXX we rely on the fact that library.list() forces all of the + // packages to be built. #ListingPackagesImpliesBuildingThem + var version = 'v' + ('' + Math.random()).substr(2, 4); // eg, "v5324" + pkgVersions[name] = version; + fs.mkdirSync(path.join(self.warehouse, 'packages', name), 0755); + fs.symlinkSync( + path.resolve(files.getCurrentToolsDir(), 'packages', name, '.build'), + path.join(self.warehouse, 'packages', name, version) + ); + }); + + // Now create each requested release. + var seenLatest = false; + _.each(options.warehouse, function (config, releaseName) { + var toolsVersion = config.tools || releaseName; + + // Release info + var manifest = { + tools: toolsVersion, + packages: pkgVersions, + upgraders: config.upgraders + }; + fs.writeFileSync(path.join(self.warehouse, 'releases', + releaseName + ".release.json"), + JSON.stringify(manifest), 'utf8'); + if (config.notices) { + fs.writeFileSync(path.join(self.warehouse, 'releases', + releaseName + ".notices.json"), + JSON.stringify(config.notices), 'utf8'); + } + + // A copy of our built tools tree with the appropriate version + files.cp_r(buildTools(), + path.join(self.warehouse, 'tools', toolsVersion)); + fs.writeFileSync(path.join(self.warehouse, 'tools', toolsVersion, + ".tools_version.txt"), + toolsVersion, 'utf8'); + + // Latest? + if (config.latest) { + if (seenLatest) + throw new Error("multiple releases marked as latest?"); + fs.symlinkSync( + releaseName + ".release.json", + path.join(self.warehouse, 'releases', 'latest') + ); + fs.symlinkSync(toolsVersion, + path.join(self.warehouse, 'tools', 'latest')); + seenLatest = true; + } + }); + + if (! seenLatest) + throw new Error("no release marked as latest?"); + + // And a cherry on top + fs.symlinkSync("tools/latest/bin/meteor", + path.join(self.warehouse, 'meteor')); + fs.chmodSync(path.join(self.warehouse, 'meteor'), 0755); + } }; _.extend(Sandbox.prototype, { @@ -258,6 +357,52 @@ _.extend(Sandbox.prototype, { }); +// Build a tools release into a temporary directory. Returns that +// directory. This is intended to be a reusable template, copied into +// each sandbox that needs it, so don't modify it. +// +// When you copy the tree, you'll probably want to change +// .tools_version.txt at the root of the tree to a version name of +// your choosing. +var buildTools = _.once(function () { + var outputDir = path.join(files.mkdtemp(), 'tools-build'); + + var child_process = require("child_process"); + var fut = new Future; + + if (! files.inCheckout()) + throw new Error("not in checkout?"); + + var execPath = path.join(files.getCurrentToolsDir(), + 'scripts', 'admin', 'build-tools-tree.sh'); + var env = _.clone(process.env); + env['TARGET_DIR'] = outputDir; + + var proc = child_process.spawn(execPath, [], { + env: env, + stdio: 'ignore' + }); + + proc.on('exit', function (code, signal) { + if (fut) { + fut['return'](code === 0); + } + }); + + proc.on('error', function (err) { + if (fut) { + fut['return'](false); + } + }); + + var success = fut.wait(); + fut = null; + if (! success) + throw new Error("failed to run scripts/admin/build-tools.sh?"); + + return outputDir; +}); + /////////////////////////////////////////////////////////////////////////////// // Run /////////////////////////////////////////////////////////////////////////////// @@ -358,12 +503,12 @@ _.extend(Run.prototype, { env: env }); - self.proc.on('close', function (code, signal) { + self.proc.on('exit', function (code, signal) { if (self.exitStatus === undefined) self._exited({ code: code, signal: signal }); }); - self.proc.on('close', function (code, signal) { + self.proc.on('error', function (err) { if (self.exitStatus === undefined) self._exited(null); }); @@ -632,18 +777,26 @@ var runTests = function (options) { "; actual: " + s(failure.details.actual) + "\n"); } - var lines = failure.details.run.outputLog.get(); - if (! lines.length) { - process.stderr.write(" => No output\n"); - } else { - process.stderr.write(" => Last ten lines:\n"); - _.each(lines.slice(-10), function (line) { - process.stderr.write(" " + - (line.channel === "stderr" ? "2| " : "1| ") + - line.text + - (line.bare ? "%" : "") + "\n"); - }); + if (failure.details.run) { + var lines = failure.details.run.outputLog.get(); + if (! lines.length) { + process.stderr.write(" => No output\n"); + } else { + process.stderr.write(" => Last ten lines:\n"); + _.each(lines.slice(-10), function (line) { + process.stderr.write(" " + + (line.channel === "stderr" ? "2| " : "1| ") + + line.text + + (line.bare ? "%" : "") + "\n"); + }); + } } + + if (failure.details.messages) { + process.stderr.write(" => Errors while building:\n"); + process.stderr.write(failure.details.messages.formatMessages()); + } + failuresInFile[test.file] = true; } else { process.stderr.write("ok\n"); diff --git a/tools/selftests/releases.js b/tools/selftests/releases.js new file mode 100644 index 0000000000..5bc3402b4f --- /dev/null +++ b/tools/selftests/releases.js @@ -0,0 +1,15 @@ +var selftest = require('../selftest.js'); +var Sandbox = selftest.Sandbox; + +selftest.define("smoke", function () { + var s = new Sandbox({ + warehouse: { + v1: { tools: 'tools1', notices: ["kitten"] }, + v2: { tools: 'tools2', notices: ["puppies"], upgraders: ["cats"], + latest: true }} + }); + + console.log("shaZAM"); + while(true) { } + +});