From c19a2471d0dfdfe5d06df78269a1f6b75c900fc7 Mon Sep 17 00:00:00 2001 From: Renan Castro Date: Wed, 16 Feb 2022 10:47:28 -0300 Subject: [PATCH] Fix multiple array operators bug and add support for debug messages --- packages/mongo/oplog_v2_converter.js | 84 +++++++++++++++++----- packages/mongo/oplog_v2_converter_tests.js | 45 ++++++++++-- 2 files changed, 108 insertions(+), 21 deletions(-) diff --git a/packages/mongo/oplog_v2_converter.js b/packages/mongo/oplog_v2_converter.js index 42c67703e7..843b6b25ce 100644 --- a/packages/mongo/oplog_v2_converter.js +++ b/packages/mongo/oplog_v2_converter.js @@ -6,6 +6,18 @@ // diff: { u: { key1: 2022-01-06T18:23:16.131Z, key2: [ObjectID] } } // } +function logConverterCalls(oplogEntry, prefixKey, key) { + if (!process.env.OPLOG_CONVERTER_DEBUG) { + return; + } + console.log('Calling nestedOplogEntryParsers with the following values: '); + console.log( + `Oplog entry: ${JSON.stringify( + oplogEntry + )}, prefixKey: ${prefixKey}, key: ${key}` + ); +} + /* the structure of an entry is: @@ -24,30 +36,64 @@ on mongo 4 const isArrayOperator = possibleArrayOperator => { if (!Object.keys(possibleArrayOperator).length) return false; - return !Object.keys(possibleArrayOperator).find(key => key !== 'a' && !key.match(/u\d+/)); + return !Object.keys(possibleArrayOperator).find( + key => key !== 'a' && !key.match(/u\d+/) + ); }; -const nestedOplogEntryParsers = ( - { i = {}, u = {}, d = {}, ...sFields }, - prefixKey = '' -) => { + +function logOplogEntryError(oplogEntry, prefixKey, key) { + console.log('---'); + console.log( + 'WARNING: Unsupported oplog operation, please fill an issue with this message at github.com/meteor/meteor' + ); + console.log( + `Oplog entry: ${JSON.stringify( + oplogEntry + )}, prefixKey: ${prefixKey}, key: ${key}` + ); + console.log('---'); +} + +const nestedOplogEntryParsers = (oplogEntry, prefixKey = '') => { + const { i = {}, u = {}, d = {}, ...sFields } = oplogEntry; + logConverterCalls(oplogEntry, prefixKey, 'ENTRY_POINT'); const sFieldsOperators = []; Object.entries(sFields).forEach(([key, value]) => { + const actualKeyNameWithoutSPrefix = key.substring(1); if (isArrayOperator(value || {})) { const { a, ...uPosition } = value; - const [positionKey, newArrayIndexValue] = Object.entries(uPosition)[0]; if (uPosition) { - sFieldsOperators.push({ - [newArrayIndexValue === null ? '$unset' : '$set']: { - [`${prefixKey}${key.substring(1)}.${positionKey.substring(1)}`]: - newArrayIndexValue === null ? true : newArrayIndexValue, - }, - }); - }else{ - throw new Error(`Unsupported oplog array entry, please review the input: ${JSON.stringify(value)}`) + for (const [positionKey, newArrayIndexValue] of Object.entries( + uPosition + )) { + sFieldsOperators.push({ + [newArrayIndexValue === null ? '$unset' : '$set']: { + [`${prefixKey}${actualKeyNameWithoutSPrefix}.${positionKey.substring( + 1 + )}`]: newArrayIndexValue === null ? true : newArrayIndexValue, + }, + }); + } + } else { + logOplogEntryError(oplogEntry, prefixKey, key); + throw new Error( + `Unsupported oplog array entry, please review the input: ${JSON.stringify( + value + )}` + ); } } else { + // we are looking at a "sSomething" that is actually a nested object set + if (!actualKeyNameWithoutSPrefix || actualKeyNameWithoutSPrefix === '') { + logOplogEntryError(oplogEntry, prefixKey, key); + return null; + } + logConverterCalls(oplogEntry, prefixKey, key); sFieldsOperators.push( - nestedOplogEntryParsers(value, `${prefixKey}${key.substring(1)}.`) + nestedOplogEntryParsers( + value, + `${prefixKey}${actualKeyNameWithoutSPrefix}.` + ) ); } }); @@ -59,7 +105,8 @@ const nestedOplogEntryParsers = ( const prefixedKey = `${prefixKey}${key}`; return { ...acc, - ...(!Array.isArray(setObjectSource[key]) && typeof setObjectSource[key] === 'object' + ...(!Array.isArray(setObjectSource[key]) && + typeof setObjectSource[key] === 'object' ? flattenObject({ [prefixedKey]: setObjectSource[key] }) : { [prefixedKey]: setObjectSource[key], @@ -85,6 +132,7 @@ const nestedOplogEntryParsers = ( export const oplogV2V1Converter = v2OplogEntry => { if (v2OplogEntry.$v !== 2 || !v2OplogEntry.diff) return v2OplogEntry; + logConverterCalls(v2OplogEntry, 'INITIAL_CALL', 'INITIAL_CALL'); return { $v: 2, ...nestedOplogEntryParsers(v2OplogEntry.diff || {}) }; }; @@ -97,7 +145,9 @@ function flattenObject(ob) { if (!Array.isArray(ob[i]) && typeof ob[i] == 'object' && ob[i] !== null) { const flatObject = flattenObject(ob[i]); let objectKeys = Object.keys(flatObject); - if(objectKeys.length === 0) { return ob; } + if (objectKeys.length === 0) { + return ob; + } for (const x of objectKeys) { toReturn[i + '.' + x] = flatObject[x]; } diff --git a/packages/mongo/oplog_v2_converter_tests.js b/packages/mongo/oplog_v2_converter_tests.js index 5a230989d1..cda6fb3393 100644 --- a/packages/mongo/oplog_v2_converter_tests.js +++ b/packages/mongo/oplog_v2_converter_tests.js @@ -51,13 +51,16 @@ Tinytest.add('oplog - v2/v1 conversion', function(test) { ); //set a new nested field inside an object - const entry51 = { "$v" : 2, "diff" : { "u" : { "count" : 1 }, "i" : { "nested" : { "state" : { } } } } }; + const entry51 = { + $v: 2, + diff: { u: { count: 1 }, i: { nested: { state: {} } } }, + }; // the correct format for this test, inspecting the mongodb oplog, should be "nested" : { "state" : { } } } // but this is a case in which we can flatten the object without collateral, so we are considering // "nested.state" : { } to be valid too test.equal( JSON.stringify(oplogV2V1Converter(entry51)), - JSON.stringify( { "$v" : 2, "$set" : { "nested.state": { }, "count" : 1 } }) + JSON.stringify({ $v: 2, $set: { 'nested.state': {}, count: 1 } }) ); //set an existing nested field inside an object @@ -98,10 +101,30 @@ Tinytest.add('oplog - v2/v1 conversion', function(test) { JSON.stringify({ $v: 2, $set: { 'services.resume.loginTokens': [] } }) ); - const entry91 = { "$v" : 2, "diff" : { "i" : { "tShirt" : { "sizes" : [ "small", "medium", "large" ] } } } }; + const entry91 = { + $v: 2, + diff: { i: { tShirt: { sizes: ['small', 'medium', 'large'] } } }, + }; test.equal( JSON.stringify(oplogV2V1Converter(entry91)), - JSON.stringify({ $v: 2, $set: { "tShirt.sizes" : [ "small", "medium", "large" ] } }) + JSON.stringify({ + $v: 2, + $set: { 'tShirt.sizes': ['small', 'medium', 'large'] }, + }) + ); + + test.equal( + JSON.stringify( + oplogV2V1Converter({ + $v: 2, + diff: { slist: { a: true, u3: 'i', u4: 'h' } }, + }) + ), + JSON.stringify({ + $v: 2, + // oplog v1 outputs the whole list -> list: ['e', 'f', 'g', 'i', 'h', 'j'] + "$set":{"list.3":"i", "list.4":"h"}, + }) ); const entry10 = { @@ -137,4 +160,18 @@ Tinytest.add('oplog - v2/v1 conversion', function(test) { }, }) ); + test.equal( + JSON.stringify( + oplogV2V1Converter({ + $v: 2, + diff: { + sobject: { u: { array: ['2', '2', '4', '3'] } }, + }, + }) + ), + JSON.stringify({ + $v: 2, + $set: { 'object.array': ['2', '2', '4', '3'] }, + }) + ); });