diff --git a/tools/run-all.js b/tools/run-all.js index 471decfca1..44b091a78c 100644 --- a/tools/run-all.js +++ b/tools/run-all.js @@ -46,7 +46,10 @@ var Runner = function (appDir, options) { self.mongoRunner = new MongoRunner({ appDir: self.appDir, port: mongoPort, - onFailure: options.onFailure + onFailure: options.onFailure, + // For testing mongod failover, run with 3 mongod if the env var is + // set. Note that data is not preserved from one run to the next. + multiple: !!process.env.METEOR_TEST_MULTIPLE_MONGOD_REPLSET }); mongoUrl = self.mongoRunner.mongoUrl(); diff --git a/tools/tests/apps/failover-test/.meteor/.gitignore b/tools/tests/apps/failover-test/.meteor/.gitignore new file mode 100644 index 0000000000..4083037423 --- /dev/null +++ b/tools/tests/apps/failover-test/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/tools/tests/apps/failover-test/.meteor/packages b/tools/tests/apps/failover-test/.meteor/packages new file mode 100644 index 0000000000..2334f880ed --- /dev/null +++ b/tools/tests/apps/failover-test/.meteor/packages @@ -0,0 +1,9 @@ +# Meteor packages used by this project, one per line. +# +# 'meteor add' and 'meteor remove' will edit this file for you, +# but you can also edit it by hand. + +meteor +mongo-livedata +underscore +random diff --git a/tools/tests/apps/failover-test/.meteor/release b/tools/tests/apps/failover-test/.meteor/release new file mode 100644 index 0000000000..621e94f0ec --- /dev/null +++ b/tools/tests/apps/failover-test/.meteor/release @@ -0,0 +1 @@ +none diff --git a/tools/tests/apps/failover-test/server/failover-test.js b/tools/tests/apps/failover-test/server/failover-test.js new file mode 100644 index 0000000000..1b704d1e12 --- /dev/null +++ b/tools/tests/apps/failover-test/server/failover-test.js @@ -0,0 +1,91 @@ +// Tests that an observeChanges created before a failover continues to work +// after the failover. Doesn't test anything in particular about writes that +// occur immediately before the failover, but does ensure that a write after the +// failover is observed. +// +// Prints various things. On success, prints SUCCESS and exits 0. On failure, +// exits non-0. + +var C = new Meteor.Collection(Random.id()); + +var steps = {}; +var nextStepTimeout = null; + +var setNextStepTimeout = function () { + nextStepTimeout = Meteor.setTimeout(function () { + console.log('Waited too long and no next step happened.'); + process.exit(1); + }, 30*1000); +}; + +var originalMasterName = null; + +steps.initialized = function () { + // Great, we got the first thing. Let's get another thing. + C.insert({step: 'next'}); + var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; + var master = db.serverConfig._state.master; + if (!master) { + console.log("No master in initialized?"); + process.exit(1); + } + originalMasterName = master.name; + console.log("Master starts as", originalMasterName); +}; + +steps.next = function () { + // Great, we can continue to add things. Now trigger a failover. + var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; + db.admin().command({replSetStepDown: 60, force: true}); + while (true) { + try { + console.log("trying to insert"); + C.insert({step: 'steppedDown'}); + console.log("inserted"); + return; + } catch (e) { + console.log("failed to insert", e); + } + } +}; + +steps.steppedDown = function () { + console.log("Write succeeded after stepdown."); + var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db; + var master = db.serverConfig._state.master; + if (!master) { + console.log("No master in steppedDown?"); + process.exit(1); + } + if (master.name === originalMasterName) { + console.log("Master didn't change?"); + process.exit(1); + } + console.log("Master ended as", master.name); + console.log("SUCCESS"); + process.exit(0); +}; + +C.find().observeChanges({ + added: function (id, fields) { + if (nextStepTimeout) { + Meteor.clearTimeout(nextStepTimeout); + nextStepTimeout = null; + } + if (!fields.step && _.has(steps, fields.step)) { + console.log('Unexpected step:', fields.step); + process.exit(1); + } + console.log("Step", fields.step); + steps[fields.step](); + setNextStepTimeout(); + } +}); + +setNextStepTimeout(); + +C.insert({step: 'initialized'}); + +main = function (argv) { + return 'DAEMON'; +}; diff --git a/tools/tests/mongo.js b/tools/tests/mongo.js new file mode 100644 index 0000000000..bfbf14d99c --- /dev/null +++ b/tools/tests/mongo.js @@ -0,0 +1,21 @@ +var selftest = require('../selftest.js'); +var Sandbox = selftest.Sandbox; +var utils = require('../utils.js'); +var net = require('net'); +var Future = require('fibers/future'); +var _ = require('underscore'); +var files = require('../files.js'); + +// Tests that observeChanges continues to work even over a mongo failover. +selftest.define("mongo failover", ["slow"], function () { + var s = new Sandbox(); + s.set('METEOR_TEST_MULTIPLE_MONGOD_REPLSET', 't'); + s.createApp("failover-test", "failover-test"); + s.cd("failover-test"); + + var run = s.run("--once", "--raw-logs"); + run.waitSecs(120); + run.match("SUCCESS\n"); + run.expectEnd(); + run.expectExit(0); +});