Test that observeChanges works over a failover

tools/run-all.js

@@ -46,7 +46,10 @@ var Runner = function (appDir, options) {
     self.mongoRunner = new MongoRunner({
       appDir: self.appDir,
       port: mongoPort,
-      onFailure: options.onFailure
+      onFailure: options.onFailure,
+      // For testing mongod failover, run with 3 mongod if the env var is
+      // set. Note that data is not preserved from one run to the next.
+      multiple: !!process.env.METEOR_TEST_MULTIPLE_MONGOD_REPLSET
     });
 
     mongoUrl = self.mongoRunner.mongoUrl();

tools/tests/apps/failover-test/.meteor/.gitignore

@@ -0,0 +1 @@
+local

tools/tests/apps/failover-test/.meteor/packages

@@ -0,0 +1,9 @@
+# Meteor packages used by this project, one per line.
+#
+# 'meteor add' and 'meteor remove' will edit this file for you,
+# but you can also edit it by hand.
+
+meteor
+mongo-livedata
+underscore
+random

tools/tests/apps/failover-test/.meteor/release

@@ -0,0 +1 @@
+none

tools/tests/apps/failover-test/server/failover-test.js

@@ -0,0 +1,91 @@
+// Tests that an observeChanges created before a failover continues to work
+// after the failover. Doesn't test anything in particular about writes that
+// occur immediately before the failover, but does ensure that a write after the
+// failover is observed.
+//
+// Prints various things. On success, prints SUCCESS and exits 0. On failure,
+// exits non-0.
+
+var C = new Meteor.Collection(Random.id());
+
+var steps = {};
+var nextStepTimeout = null;
+
+var setNextStepTimeout = function () {
+  nextStepTimeout = Meteor.setTimeout(function () {
+    console.log('Waited too long and no next step happened.');
+    process.exit(1);
+  }, 30*1000);
+};
+
+var originalMasterName = null;
+
+steps.initialized = function () {
+  // Great, we got the first thing. Let's get another thing.
+  C.insert({step: 'next'});
+  var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
+  var master = db.serverConfig._state.master;
+  if (!master) {
+    console.log("No master in initialized?");
+    process.exit(1);
+  }
+  originalMasterName = master.name;
+  console.log("Master starts as", originalMasterName);
+};
+
+steps.next = function () {
+  // Great, we can continue to add things. Now trigger a failover.
+  var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
+  db.admin().command({replSetStepDown: 60, force: true});
+  while (true) {
+    try {
+      console.log("trying to insert");
+      C.insert({step: 'steppedDown'});
+      console.log("inserted");
+      return;
+    } catch (e) {
+      console.log("failed to insert", e);
+    }
+  }
+};
+
+steps.steppedDown = function () {
+  console.log("Write succeeded after stepdown.");
+  var db = MongoInternals.defaultRemoteCollectionDriver().mongo.db;
+  var master = db.serverConfig._state.master;
+  if (!master) {
+    console.log("No master in steppedDown?");
+    process.exit(1);
+  }
+  if (master.name === originalMasterName) {
+    console.log("Master didn't change?");
+    process.exit(1);
+  }
+  console.log("Master ended as", master.name);
+  console.log("SUCCESS");
+  process.exit(0);
+};
+
+C.find().observeChanges({
+  added: function (id, fields) {
+    if (nextStepTimeout) {
+      Meteor.clearTimeout(nextStepTimeout);
+      nextStepTimeout = null;
+    }
+    if (!fields.step && _.has(steps, fields.step)) {
+      console.log('Unexpected step:', fields.step);
+      process.exit(1);
+    }
+    console.log("Step", fields.step);
+    steps[fields.step]();
+    setNextStepTimeout();
+  }
+});
+
+setNextStepTimeout();
+
+C.insert({step: 'initialized'});
+
+main = function (argv) {
+  return 'DAEMON';
+};

tools/tests/mongo.js

@@ -0,0 +1,21 @@
+var selftest = require('../selftest.js');
+var Sandbox = selftest.Sandbox;
+var utils = require('../utils.js');
+var net = require('net');
+var Future = require('fibers/future');
+var _ = require('underscore');
+var files = require('../files.js');
+
+// Tests that observeChanges continues to work even over a mongo failover.
+selftest.define("mongo failover", ["slow"], function () {
+  var s = new Sandbox();
+  s.set('METEOR_TEST_MULTIPLE_MONGOD_REPLSET', 't');
+  s.createApp("failover-test", "failover-test");
+  s.cd("failover-test");
+
+  var run = s.run("--once", "--raw-logs");
+  run.waitSecs(120);
+  run.match("SUCCESS\n");
+  run.expectEnd();
+  run.expectExit(0);
+});