Merge branch 'release-2.16' into 3.0-impact-2.16-quick-gains

# Conflicts:
#	packages/mongo/collection_tests.js
#	packages/mongo/mongo_driver.js
#	packages/mongo/oplog_tests.js

docs/source/api/collections.md

@@ -1082,6 +1082,38 @@ option:
 You can pass any MongoDB valid option, these are just examples using
 certificates configurations.
 
+<h3 id="mongo_oplog_options">Mongo Oplog Options</h3>
+
+> Oplog options were introduced in Meteor 2.15.1
+
+If you set the [`MONGO_OPLOG_URL`](https://docs.meteor.com/environment-variables.html#MONGO-OPLOG-URL) env var, Meteor will use MongoDB's Oplog to show efficient, real time updates to your users via your subscriptions.
+
+Due to how Meteor's Oplog implementation is built behind the scenes, if you have certain collections where you expect **big amounts of write operations**, this might lead to **big CPU spikes on your meteor app server, even if you have no publications/subscriptions on any data/documents of these collections**. For more information on this, please have a look into [this blog post from 2016](https://blog.meteor.com/tuning-meteor-mongo-livedata-for-scalability-13fe9deb8908), [this github discussion from 2022](https://github.com/meteor/meteor/discussions/11842) or [this meteor forums post from 2023](https://forums.meteor.com/t/cpu-spikes-due-to-oplog-updates-without-subscriptions/60028).
+
+To solve this, **2 Oplog settings** have been introduced **to tweak, which collections are *watched* or *ignored* in the oplog**.
+
+**Exclusion**: To *exclude* for example all updates/inserts of documents in the 2 collections called `products` and `prices`, you would need to set the following setting in your Meteor settings file:
+
+```json
+  "packages": {
+    "mongo": {
+      "oplogExcludeCollections": ["products", "prices"]
+    }
+  }
+```
+
+**Inclusion**: vice versa, if you only want to watch/*include* the oplog for changes on documents in the 2 collections `chats` and `messages`, you would use:
+
+```json
+  "packages": {
+    "mongo": {
+      "oplogIncludeCollections": ["chats", "messages"]
+    }
+  }
+```
+
+For obvious reasons, using both `oplogExcludeCollections` and `oplogIncludeCollections` at the same time is not possible and will result in an error.
+
 <h3 id="mongo_connection_options_settings">Mongo.setConnectionOptions</h3>
 
 You can also call `Mongo.setConnectionOptions` to set the connection options but

packages/mongo/collection_tests.js

@@ -475,4 +475,4 @@ Meteor.isServer && Tinytest.addAsync('collection - simple add', async function(t
   id = await collection.insertAsync({a: 2});
   test.equal((await collection.findOneAsync(id)).a, 2);
   await collection.removeAsync({});
-})
+});

packages/mongo/mongo_driver.js

@@ -241,6 +241,11 @@ MongoConnection.prototype.close = function () {
   return this._close();
 };
 
+MongoConnection.prototype._setOplogHandle = function(oplogHandle) {
+  this._oplogHandle = oplogHandle;
+  return this;
+};
+
 // Returns the Mongo Collection object; may yield.
 MongoConnection.prototype.rawCollection = function (collectionName) {
   var self = this;
@@ -930,7 +935,6 @@ Cursor.prototype.getTransform = function () {
 // When you call Meteor.publish() with a function that returns a Cursor, we need
 // to transmute it into the equivalent subscription.  This is the function that
 // does that.
-
 Cursor.prototype._publishCursor = function (sub) {
   var self = this;
   var collection = self._cursorDescription.collectionName;
@@ -1453,10 +1457,13 @@ MongoConnection.prototype.tail = function (cursorDescription, docCallback, timeo
   };
 };
 
+const oplogCollectionWarnings = [];
+
 Object.assign(MongoConnection.prototype, {
   _observeChanges: async function (
       cursorDescription, ordered, callbacks, nonMutatingCallbacks) {
     var self = this;
+    const collectionName = cursorDescription.collectionName;
 
     if (cursorDescription.options.tailable) {
       return self._observeChangesTailable(cursorDescription, ordered, callbacks);
@@ -1499,7 +1506,8 @@ Object.assign(MongoConnection.prototype, {
         nonMutatingCallbacks,
     );
 
-    if (firstHandle) {
+    const oplogOptions = self?._oplogHandle?._oplogOptions || {};
+  const { includeCollections, excludeCollections } = oplogOptions;if (firstHandle) {
       var matcher, sorter;
       var canUseOplog = _.all([
         function () {
@@ -1507,7 +1515,24 @@ Object.assign(MongoConnection.prototype, {
           // want unordered callbacks, and to not want a callback on the polls
           // that won't happen.
           return self._oplogHandle && !ordered &&
-              !callbacks._testOnlyPollCallback;
+              !callbacks._testOnlyPollCallback;}, function () {
+        // We also need to check, if the collection of this Cursor is actually being "watched" by the Oplog handle
+        // if not, we have to fallback to long polling
+        if (excludeCollections?.length && excludeCollections.includes(collectionName)) {
+          if (!oplogCollectionWarnings.includes(collectionName)) {
+            console.warn(`Meteor.settings.packages.mongo.oplogExcludeCollections includes the collection ${collectionName} - your subscriptions will only use long polling!`);
+            oplogCollectionWarnings.push(collectionName); // we only want to show the warnings once per collection!
+          }
+          return false;
+        }
+        if (includeCollections?.length && !includeCollections.includes(collectionName)) {
+          if (!oplogCollectionWarnings.includes(collectionName)) {
+            console.warn(`Meteor.settings.packages.mongo.oplogIncludeCollections does not include the collection ${collectionName} - your subscriptions will only use long polling!`);
+            oplogCollectionWarnings.push(collectionName); // we only want to show the warnings once per collection!
+          }
+          return false;
+        }
+        return true;
         }, function () {
           // We need to be able to compile the selector. Fall back to polling for
           // some newfangled $selector that minimongo doesn't support yet.

packages/mongo/oplog_tailing.js

@@ -27,6 +27,7 @@ OplogHandle = function (oplogUrl, dbName) {
 
   self._oplogLastEntryConnection = null;
   self._oplogTailConnection = null;
+  self._oplogOptions = null;
   self._stopped = false;
   self._tailHandle = null;
   self._readyPromiseResolver = null;
@@ -81,6 +82,8 @@ OplogHandle = function (oplogUrl, dbName) {
   //TODO[fibers] Why wait?
 };
 
+MongoInternals.OplogHandle = OplogHandle;
+
 Object.assign(OplogHandle.prototype, {
   stop: async function () {
     var self = this;
@@ -257,6 +260,40 @@ Object.assign(OplogHandle.prototype, {
       self._lastProcessedTS = lastOplogEntry.ts;
     }
 
+    // These 2 settings allow you to either only watch certain collections (oplogIncludeCollections), or exclude some collections you don't want to watch for oplog updates (oplogExcludeCollections)
+    // Usage:
+    // settings.json = {
+    //   "packages": {
+    //     "mongo": {
+    //       "oplogExcludeCollections": ["products", "prices"] // This would exclude both collections "products" and "prices" from any oplog tailing. 
+    //                                                            Beware! This means, that no subscriptions on these 2 collections will update anymore!
+    //     }
+    //   }
+    // }
+    const includeCollections = Meteor.settings?.packages?.mongo?.oplogIncludeCollections;
+    const excludeCollections = Meteor.settings?.packages?.mongo?.oplogExcludeCollections;
+    if (includeCollections?.length && excludeCollections?.length) {
+      throw new Error("Can't use both mongo oplog settings oplogIncludeCollections and oplogExcludeCollections at the same time.");
+    }
+    if (excludeCollections?.length) {
+      oplogSelector.ns = {
+        $regex: oplogSelector.ns,
+        $nin: excludeCollections.map((collName) => `${self._dbName}.${collName}`)
+      }
+      self._oplogOptions = { excludeCollections };
+    }
+    else if (includeCollections?.length) {
+      oplogSelector = { $and: [
+        { $or: [
+          { ns: /^admin\.\$cmd/ },
+          { ns: { $in: includeCollections.map((collName) => `${self._dbName}.${collName}`) } }
+        ] },
+        { $or: oplogSelector.$or }, // the initial $or to select only certain operations (op)
+        { ts: oplogSelector.ts }
+      ] };
+      self._oplogOptions = { includeCollections };
+    }
+
     var cursorDescription = new CursorDescription(
         OPLOG_COLLECTION, oplogSelector, {tailable: true});
 

packages/mongo/oplog_tests.js

@@ -1,4 +1,5 @@
-var OplogCollection = new Mongo.Collection("oplog-" + Random.id());
+var randomId = Random.id();
+var OplogCollection = new Mongo.Collection("oplog-" + randomId);
 
 Tinytest.addAsync('mongo-livedata - oplog - cursorSupported', async function(
   test
@@ -177,6 +178,115 @@ process.env.MONGO_OPLOG_URL &&
     },
   ]);
 
+const defaultOplogHandle = MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle;
+let previousMongoPackageSettings = {};
+
+async function oplogOptionsTest({
+  test,
+  includeCollectionName,
+  excludeCollectionName,
+  mongoPackageSettings = {}
+}) {
+  try {
+    previousMongoPackageSettings = { ...(Meteor.settings?.packages?.mongo || {}) };
+    if (!Meteor.settings.packages) Meteor.settings.packages = {};
+    Meteor.settings.packages.mongo = mongoPackageSettings;
+
+    const myOplogHandle = new MongoInternals.OplogHandle(process.env.MONGO_OPLOG_URL, 'meteor');
+    MongoInternals.defaultRemoteCollectionDriver().mongo._setOplogHandle(myOplogHandle);
+
+    const IncludeCollection = new Mongo.Collection(includeCollectionName);
+    const ExcludeCollection = new Mongo.Collection(excludeCollectionName);
+
+    const shouldBeTracked = new Promise((resolve) => {
+      IncludeCollection.find({ include: 'yes' }).observeChanges({
+        added(id, fields) { resolve(true) }
+      });
+    });
+    const shouldBeIgnored = new Promise((resolve, reject) => {
+      ExcludeCollection.find({ include: 'no' }).observeChanges({
+        added(id, fields) {
+          // should NOT fire, because this is an excluded collection:
+          reject(false);
+        }
+      });
+      // we give it just 2 seconds until we resolve this promise:
+      setTimeout(() => {
+        resolve(true);
+      }, 2000);
+    });
+
+    // do the inserts:
+    await IncludeCollection.rawCollection().insertOne({ include: 'yes', foo: 'bar' });
+    await ExcludeCollection.rawCollection().insertOne({ include: 'no', foo: 'bar' });
+
+    test.equal(await shouldBeTracked, true);
+    test.equal(await shouldBeIgnored, true);
+  } finally {
+    // Reset:
+    Meteor.settings.packages.mongo = { ...previousMongoPackageSettings };
+    MongoInternals.defaultRemoteCollectionDriver().mongo._setOplogHandle(defaultOplogHandle);
+  }
+}
+
+process.env.MONGO_OPLOG_URL && Tinytest.addAsync(
+  'mongo-livedata - oplog - oplogSettings - oplogExcludeCollections',
+  async test => {
+    const collectionNameA = "oplog-a-" + Random.id();
+    const collectionNameB = "oplog-b-" + Random.id();
+    const mongoPackageSettings = {
+      oplogExcludeCollections: [collectionNameB]
+    };
+    await oplogOptionsTest({
+      test,
+      includeCollectionName: collectionNameA,
+      excludeCollectionName: collectionNameB,
+      mongoPackageSettings
+    });
+  }
+);
+
+process.env.MONGO_OPLOG_URL && Tinytest.addAsync(
+  'mongo-livedata - oplog - oplogSettings - oplogIncludeCollections',
+  async test => {
+    const collectionNameA = "oplog-a-" + Random.id();
+    const collectionNameB = "oplog-b-" + Random.id();
+    const mongoPackageSettings = {
+      oplogIncludeCollections: [collectionNameB]
+    };
+    await oplogOptionsTest({
+      test,
+      includeCollectionName: collectionNameB,
+      excludeCollectionName: collectionNameA,
+      mongoPackageSettings
+    });
+  }
+);
+
+process.env.MONGO_OPLOG_URL && Tinytest.addAsync(
+  'mongo-livedata - oplog - oplogSettings - oplogExcludeCollections & oplogIncludeCollections',
+  async test => {
+    // should fail, because we don't allow including and excluding at the same time!
+    const collectionNameA = "oplog-a-" + Random.id();
+    const collectionNameB = "oplog-b-" + Random.id();
+    const mongoPackageSettings = {
+      oplogIncludeCollections: [collectionNameA],
+      oplogExcludeCollections: [collectionNameB]
+    };
+    try {
+      await oplogOptionsTest({
+        test,
+        includeCollectionName: collectionNameA,
+        excludeCollectionName: collectionNameB,
+        mongoPackageSettings
+      });
+      test.fail();
+    } catch (err) {
+      test.expect_fail();
+    }
+  }
+);
+
 // TODO this is commented for now, but we need to find out the cause
 // PR: https://github.com/meteor/meteor/pull/12057
 // Meteor.isServer && Tinytest.addAsync(