From 7cf9c2116ccf7fc81c79ee4d77069147b38c471c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Miernik?= Date: Tue, 11 Jul 2017 21:58:16 +0200 Subject: [PATCH] Separated Matcher. --- packages/minimongo/common.js | 30 + packages/minimongo/matcher.js | 1259 +++++++++++++++++++++++++++++ packages/minimongo/minimongo.js | 1336 +------------------------------ packages/minimongo/package.js | 3 + 4 files changed, 1317 insertions(+), 1311 deletions(-) create mode 100644 packages/minimongo/common.js create mode 100644 packages/minimongo/matcher.js diff --git a/packages/minimongo/common.js b/packages/minimongo/common.js new file mode 100644 index 0000000000..b471cda7b3 --- /dev/null +++ b/packages/minimongo/common.js @@ -0,0 +1,30 @@ +import {LocalCollection} from './local_collection.js'; + +export function isIndexable (obj) { + return Array.isArray(obj) || LocalCollection._isPlainObject(obj); +} + +export function isNumericKey (s) { + return /^[0-9]+$/.test(s); +} + +// Returns true if this is an object with at least one key and all keys begin +// with $. Unless inconsistentOK is set, throws if some keys begin with $ and +// others don't. +export function isOperatorObject (valueSelector, inconsistentOK) { + if (!LocalCollection._isPlainObject(valueSelector)) + return false; + + var theseAreOperators = undefined; + Object.keys(valueSelector).forEach(function (selKey) { + var thisIsOperator = selKey.substr(0, 1) === '$'; + if (theseAreOperators === undefined) { + theseAreOperators = thisIsOperator; + } else if (theseAreOperators !== thisIsOperator) { + if (!inconsistentOK) + throw new Error(`Inconsistent operator: ${JSON.stringify(valueSelector)}`); + theseAreOperators = false; + } + }); + return !!theseAreOperators; // {} has no operators +} diff --git a/packages/minimongo/matcher.js b/packages/minimongo/matcher.js new file mode 100644 index 0000000000..05746fc776 --- /dev/null +++ b/packages/minimongo/matcher.js @@ -0,0 +1,1259 @@ +import {LocalCollection} from './local_collection.js'; +import { + isIndexable, + isNumericKey, + isOperatorObject, +} from './common.js'; + +// The minimongo selector compiler! + +// Terminology: +// - a 'selector' is the EJSON object representing a selector +// - a 'matcher' is its compiled form (whether a full Minimongo.Matcher +// object or one of the component lambdas that matches parts of it) +// - a 'result object' is an object with a 'result' field and maybe +// distance and arrayIndices. +// - a 'branched value' is an object with a 'value' field and maybe +// 'dontIterate' and 'arrayIndices'. +// - a 'document' is a top-level object that can be stored in a collection. +// - a 'lookup function' is a function that takes in a document and returns +// an array of 'branched values'. +// - a 'branched matcher' maps from an array of branched values to a result +// object. +// - an 'element matcher' maps from a single value to a bool. + +// Main entry point. +// var matcher = new Minimongo.Matcher({a: {$gt: 5}}); +// if (matcher.documentMatches({a: 7})) ... +export class Matcher { + constructor (selector, isUpdate) { + // A set (object mapping string -> *) of all of the document paths looked + // at by the selector. Also includes the empty string if it may look at any + // path (eg, $where). + this._paths = {}; + // Set to true if compilation finds a $near. + this._hasGeoQuery = false; + // Set to true if compilation finds a $where. + this._hasWhere = false; + // Set to false if compilation finds anything other than a simple equality + // or one or more of '$gt', '$gte', '$lt', '$lte', '$ne', '$in', '$nin' used + // with scalars as operands. + this._isSimple = true; + // Set to a dummy document which always matches this Matcher. Or set to null + // if such document is too hard to find. + this._matchingDocument = undefined; + // A clone of the original selector. It may just be a function if the user + // passed in a function; otherwise is definitely an object (eg, IDs are + // translated into {_id: ID} first. Used by canBecomeTrueByModifier and + // Sorter._useWithMatcher. + this._selector = null; + this._docMatcher = this._compileSelector(selector); + // Set to true if selection is done for an update operation + // Default is false + // Used for $near array update (issue #3599) + this._isUpdate = isUpdate; + } + + documentMatches (doc) { + if (doc !== Object(doc)) + throw Error('documentMatches needs a document'); + + return this._docMatcher(doc); + } + + hasGeoQuery () { + return this._hasGeoQuery; + } + + hasWhere () { + return this._hasWhere; + } + + isSimple () { + return this._isSimple; + } + + // Given a selector, return a function that takes one argument, a + // document. It returns a result object. + _compileSelector (selector) { + // you can pass a literal function instead of a selector + if (selector instanceof Function) { + this._isSimple = false; + this._selector = selector; + this._recordPathUsed(''); + return doc => ({result: !!selector.call(doc)}); + } + + // shorthand -- scalars match _id + if (LocalCollection._selectorIsId(selector)) { + this._selector = {_id: selector}; + this._recordPathUsed('_id'); + return doc => ({result: EJSON.equals(doc._id, selector)}); + } + + // protect against dangerous selectors. falsey and {_id: falsey} are both + // likely programmer error, and not what you want, particularly for + // destructive operations. + if (!selector || (selector.hasOwnProperty('_id') && !selector._id)) { + this._isSimple = false; + return nothingMatcher; + } + + // Top level can't be an array or true or binary. + if (Array.isArray(selector) || + EJSON.isBinary(selector) || + typeof selector === 'boolean') + throw new Error(`Invalid selector: ${selector}`); + + this._selector = EJSON.clone(selector); + + return compileDocumentSelector(selector, this, {isRoot: true}); + } + + // Returns a list of key paths the given selector is looking for. It includes + // the empty string if there is a $where. + _getPaths () { + return Object.keys(this._paths); + } + + _recordPathUsed (path) { + this._paths[path] = true; + } +} + +// helpers used by compiled selector code +LocalCollection._f = { + // XXX for _all and _in, consider building 'inquery' at compile time.. + + _type: function (v) { + if (typeof v === "number") + return 1; + if (typeof v === "string") + return 2; + if (typeof v === "boolean") + return 8; + if (Array.isArray(v)) + return 4; + if (v === null) + return 10; + if (v instanceof RegExp) + // note that typeof(/x/) === "object" + return 11; + if (typeof v === "function") + return 13; + if (v instanceof Date) + return 9; + if (EJSON.isBinary(v)) + return 5; + if (v instanceof MongoID.ObjectID) + return 7; + return 3; // object + + // XXX support some/all of these: + // 14, symbol + // 15, javascript code with scope + // 16, 18: 32-bit/64-bit integer + // 17, timestamp + // 255, minkey + // 127, maxkey + }, + + // deep equality test: use for literal document and array matches + _equal: function (a, b) { + return EJSON.equals(a, b, {keyOrderSensitive: true}); + }, + + // maps a type code to a value that can be used to sort values of + // different types + _typeorder: function (t) { + // http://www.mongodb.org/display/DOCS/What+is+the+Compare+Order+for+BSON+Types + // XXX what is the correct sort position for Javascript code? + // ('100' in the matrix below) + // XXX minkey/maxkey + return [-1, // (not a type) + 1, // number + 2, // string + 3, // object + 4, // array + 5, // binary + -1, // deprecated + 6, // ObjectID + 7, // bool + 8, // Date + 0, // null + 9, // RegExp + -1, // deprecated + 100, // JS code + 2, // deprecated (symbol) + 100, // JS code + 1, // 32-bit int + 8, // Mongo timestamp + 1 // 64-bit int + ][t]; + }, + + // compare two values of unknown type according to BSON ordering + // semantics. (as an extension, consider 'undefined' to be less than + // any other value.) return negative if a is less, positive if b is + // less, or 0 if equal + _cmp: function (a, b) { + if (a === undefined) + return b === undefined ? 0 : -1; + if (b === undefined) + return 1; + var ta = LocalCollection._f._type(a); + var tb = LocalCollection._f._type(b); + var oa = LocalCollection._f._typeorder(ta); + var ob = LocalCollection._f._typeorder(tb); + if (oa !== ob) + return oa < ob ? -1 : 1; + if (ta !== tb) + // XXX need to implement this if we implement Symbol or integers, or + // Timestamp + throw Error("Missing type coercion logic in _cmp"); + if (ta === 7) { // ObjectID + // Convert to string. + ta = tb = 2; + a = a.toHexString(); + b = b.toHexString(); + } + if (ta === 9) { // Date + // Convert to millis. + ta = tb = 1; + a = a.getTime(); + b = b.getTime(); + } + + if (ta === 1) // double + return a - b; + if (tb === 2) // string + return a < b ? -1 : (a === b ? 0 : 1); + if (ta === 3) { // Object + // this could be much more efficient in the expected case ... + var to_array = function (obj) { + var ret = []; + for (var key in obj) { + ret.push(key); + ret.push(obj[key]); + } + return ret; + }; + return LocalCollection._f._cmp(to_array(a), to_array(b)); + } + if (ta === 4) { // Array + for (var i = 0; ; i++) { + if (i === a.length) + return (i === b.length) ? 0 : -1; + if (i === b.length) + return 1; + var s = LocalCollection._f._cmp(a[i], b[i]); + if (s !== 0) + return s; + } + } + if (ta === 5) { // binary + // Surprisingly, a small binary blob is always less than a large one in + // Mongo. + if (a.length !== b.length) + return a.length - b.length; + for (i = 0; i < a.length; i++) { + if (a[i] < b[i]) + return -1; + if (a[i] > b[i]) + return 1; + } + return 0; + } + if (ta === 8) { // boolean + if (a) return b ? 0 : 1; + return b ? -1 : 0; + } + if (ta === 10) // null + return 0; + if (ta === 11) // regexp + throw Error("Sorting not supported on regular expression"); // XXX + // 13: javascript code + // 14: symbol + // 15: javascript code with scope + // 16: 32-bit integer + // 17: timestamp + // 18: 64-bit integer + // 255: minkey + // 127: maxkey + if (ta === 13) // javascript code + throw Error("Sorting not supported on Javascript code"); // XXX + throw Error("Unknown type to sort"); + } +}; + +// Each element selector contains: +// - compileElementSelector, a function with args: +// - operand - the "right hand side" of the operator +// - valueSelector - the "context" for the operator (so that $regex can find +// $options) +// - matcher - the Matcher this is going into (so that $elemMatch can compile +// more things) +// returning a function mapping a single value to bool. +// - dontExpandLeafArrays, a bool which prevents expandArraysInBranches from +// being called +// - dontIncludeLeafArrays, a bool which causes an argument to be passed to +// expandArraysInBranches if it is called +const ELEMENT_OPERATORS = { + $lt: makeInequality(function (cmpValue) { + return cmpValue < 0; + }), + $gt: makeInequality(function (cmpValue) { + return cmpValue > 0; + }), + $lte: makeInequality(function (cmpValue) { + return cmpValue <= 0; + }), + $gte: makeInequality(function (cmpValue) { + return cmpValue >= 0; + }), + $mod: { + compileElementSelector: function (operand) { + if (!(Array.isArray(operand) && operand.length === 2 + && typeof(operand[0]) === 'number' + && typeof(operand[1]) === 'number')) { + throw Error("argument to $mod must be an array of two numbers"); + } + // XXX could require to be ints or round or something + var divisor = operand[0]; + var remainder = operand[1]; + return function (value) { + return typeof value === 'number' && value % divisor === remainder; + }; + } + }, + $in: { + compileElementSelector: function (operand) { + if (!Array.isArray(operand)) + throw Error("$in needs an array"); + + var elementMatchers = []; + operand.forEach(function (option) { + if (option instanceof RegExp) + elementMatchers.push(regexpElementMatcher(option)); + else if (isOperatorObject(option)) + throw Error("cannot nest $ under $in"); + else + elementMatchers.push(equalityElementMatcher(option)); + }); + + return function (value) { + // Allow {a: {$in: [null]}} to match when 'a' does not exist. + if (value === undefined) + value = null; + return elementMatchers.some(function (e) { + return e(value); + }); + }; + } + }, + $size: { + // {a: [[5, 5]]} must match {a: {$size: 1}} but not {a: {$size: 2}}, so we + // don't want to consider the element [5,5] in the leaf array [[5,5]] as a + // possible value. + dontExpandLeafArrays: true, + compileElementSelector: function (operand) { + if (typeof operand === 'string') { + // Don't ask me why, but by experimentation, this seems to be what Mongo + // does. + operand = 0; + } else if (typeof operand !== 'number') { + throw Error("$size needs a number"); + } + return function (value) { + return Array.isArray(value) && value.length === operand; + }; + } + }, + $type: { + // {a: [5]} must not match {a: {$type: 4}} (4 means array), but it should + // match {a: {$type: 1}} (1 means number), and {a: [[5]]} must match {$a: + // {$type: 4}}. Thus, when we see a leaf array, we *should* expand it but + // should *not* include it itself. + dontIncludeLeafArrays: true, + compileElementSelector: function (operand) { + if (typeof operand !== 'number') + throw Error("$type needs a number"); + return function (value) { + return value !== undefined + && LocalCollection._f._type(value) === operand; + }; + } + }, + $bitsAllSet: { + compileElementSelector: function (operand) { + var op = getOperandBitmask(operand, '$bitsAllSet') + return function (value) { + var bitmask = getValueBitmask(value, op.length) + return bitmask && op.every(function (byte, idx) { + return ((bitmask[idx] & byte) == byte) + }) + } + } + }, + $bitsAnySet: { + compileElementSelector: function (operand) { + var query = getOperandBitmask(operand, '$bitsAnySet') + return function (value) { + var bitmask = getValueBitmask(value, query.length) + return bitmask && query.some(function (byte, idx) { + return ((~bitmask[idx] & byte) !== byte) + }) + } + } + }, + $bitsAllClear: { + compileElementSelector: function (operand) { + var query = getOperandBitmask(operand, '$bitsAllClear') + return function (value) { + var bitmask = getValueBitmask(value, query.length) + return bitmask && query.every(function (byte, idx) { + return !(bitmask[idx] & byte) + }) + } + } + }, + $bitsAnyClear: { + compileElementSelector: function (operand) { + var query = getOperandBitmask(operand, '$bitsAnyClear') + return function (value) { + var bitmask = getValueBitmask(value, query.length) + return bitmask && query.some(function (byte, idx) { + return ((bitmask[idx] & byte) !== byte) + }) + } + } + }, + $regex: { + compileElementSelector: function (operand, valueSelector) { + if (!(typeof operand === 'string' || operand instanceof RegExp)) + throw Error("$regex has to be a string or RegExp"); + + var regexp; + if (valueSelector.$options !== undefined) { + // Options passed in $options (even the empty string) always overrides + // options in the RegExp object itself. (See also + // Mongo.Collection._rewriteSelector.) + + // Be clear that we only support the JS-supported options, not extended + // ones (eg, Mongo supports x and s). Ideally we would implement x and s + // by transforming the regexp, but not today... + if (/[^gim]/.test(valueSelector.$options)) + throw new Error("Only the i, m, and g regexp options are supported"); + + var regexSource = operand instanceof RegExp ? operand.source : operand; + regexp = new RegExp(regexSource, valueSelector.$options); + } else if (operand instanceof RegExp) { + regexp = operand; + } else { + regexp = new RegExp(operand); + } + return regexpElementMatcher(regexp); + } + }, + $elemMatch: { + dontExpandLeafArrays: true, + compileElementSelector: function (operand, valueSelector, matcher) { + if (!LocalCollection._isPlainObject(operand)) + throw Error("$elemMatch need an object"); + + var subMatcher, isDocMatcher; + if (isOperatorObject(Object.keys(operand) + .filter(function (key) { return !Object.keys(LOGICAL_OPERATORS).includes(key); }) + .reduce(function (a, b) { return Object.assign(a, {[b]: operand[b]}); }, {}), true)) { + subMatcher = compileValueSelector(operand, matcher); + isDocMatcher = false; + } else { + // This is NOT the same as compileValueSelector(operand), and not just + // because of the slightly different calling convention. + // {$elemMatch: {x: 3}} means "an element has a field x:3", not + // "consists only of a field x:3". Also, regexps and sub-$ are allowed. + subMatcher = compileDocumentSelector(operand, matcher, + {inElemMatch: true}); + isDocMatcher = true; + } + + return function (value) { + if (!Array.isArray(value)) + return false; + for (var i = 0; i < value.length; ++i) { + var arrayElement = value[i]; + var arg; + if (isDocMatcher) { + // We can only match {$elemMatch: {b: 3}} against objects. + // (We can also match against arrays, if there's numeric indices, + // eg {$elemMatch: {'0.b': 3}} or {$elemMatch: {0: 3}}.) + if (!isIndexable(arrayElement)) + return false; + arg = arrayElement; + } else { + // dontIterate ensures that {a: {$elemMatch: {$gt: 5}}} matches + // {a: [8]} but not {a: [[8]]} + arg = [{value: arrayElement, dontIterate: true}]; + } + // XXX support $near in $elemMatch by propagating $distance? + if (subMatcher(arg).result) + return i; // specially understood to mean "use as arrayIndices" + } + return false; + }; + } + } +}; + +// Operators that appear at the top level of a document selector. +const LOGICAL_OPERATORS = { + $and: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); + return andDocumentMatchers(matchers); + }, + + $or: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); + + // Special case: if there is only one matcher, use it directly, *preserving* + // any arrayIndices it returns. + if (matchers.length === 1) + return matchers[0]; + + return function (doc) { + var result = matchers.some(function (f) { + return f(doc).result; + }); + // $or does NOT set arrayIndices when it has multiple + // sub-expressions. (Tested against MongoDB.) + return {result: result}; + }; + }, + + $nor: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); + return function (doc) { + var result = matchers.every(function (f) { + return !f(doc).result; + }); + // Never set arrayIndices, because we only match if nothing in particular + // 'matched' (and because this is consistent with MongoDB). + return {result: result}; + }; + }, + + $where: function (selectorValue, matcher) { + // Record that *any* path may be used. + matcher._recordPathUsed(''); + matcher._hasWhere = true; + if (!(selectorValue instanceof Function)) { + // XXX MongoDB seems to have more complex logic to decide where or or not + // to add 'return'; not sure exactly what it is. + selectorValue = Function('obj', `return ${selectorValue}`); + } + return function (doc) { + // We make the document available as both `this` and `obj`. + // XXX not sure what we should do if this throws + return {result: selectorValue.call(doc, doc)}; + }; + }, + + // This is just used as a comment in the query (in MongoDB, it also ends up in + // query logs); it has no effect on the actual selection. + $comment: function () { + return function () { + return {result: true}; + }; + } +}; + +// Operators that (unlike LOGICAL_OPERATORS) pertain to individual paths in a +// document, but (unlike ELEMENT_OPERATORS) do not have a simple definition as +// "match each branched value independently and combine with +// convertElementMatcherToBranchedMatcher". +var VALUE_OPERATORS = { + $eq: function (operand) { + return convertElementMatcherToBranchedMatcher( + equalityElementMatcher(operand)); + }, + $not: function (operand, valueSelector, matcher) { + return invertBranchedMatcher(compileValueSelector(operand, matcher)); + }, + $ne: function (operand) { + return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( + equalityElementMatcher(operand))); + }, + $nin: function (operand) { + return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( + ELEMENT_OPERATORS.$in.compileElementSelector(operand))); + }, + $exists: function (operand) { + var exists = convertElementMatcherToBranchedMatcher(function (value) { + return value !== undefined; + }); + return operand ? exists : invertBranchedMatcher(exists); + }, + // $options just provides options for $regex; its logic is inside $regex + $options: function (operand, valueSelector) { + if (!valueSelector.hasOwnProperty('$regex')) + throw Error("$options needs a $regex"); + return everythingMatcher; + }, + // $maxDistance is basically an argument to $near + $maxDistance: function (operand, valueSelector) { + if (!valueSelector.$near) + throw Error("$maxDistance needs a $near"); + return everythingMatcher; + }, + $all: function (operand, valueSelector, matcher) { + if (!Array.isArray(operand)) + throw Error("$all requires array"); + // Not sure why, but this seems to be what MongoDB does. + if (operand.length === 0) + return nothingMatcher; + + var branchedMatchers = []; + operand.forEach(function (criterion) { + // XXX handle $all/$elemMatch combination + if (isOperatorObject(criterion)) + throw Error("no $ expressions in $all"); + // This is always a regexp or equality selector. + branchedMatchers.push(compileValueSelector(criterion, matcher)); + }); + // andBranchedMatchers does NOT require all selectors to return true on the + // SAME branch. + return andBranchedMatchers(branchedMatchers); + }, + $near: function (operand, valueSelector, matcher, isRoot) { + if (!isRoot) + throw Error("$near can't be inside another $ operator"); + matcher._hasGeoQuery = true; + + // There are two kinds of geodata in MongoDB: legacy coordinate pairs and + // GeoJSON. They use different distance metrics, too. GeoJSON queries are + // marked with a $geometry property, though legacy coordinates can be + // matched using $geometry. + + var maxDistance, point, distance; + if (LocalCollection._isPlainObject(operand) && operand.hasOwnProperty('$geometry')) { + // GeoJSON "2dsphere" mode. + maxDistance = operand.$maxDistance; + point = operand.$geometry; + distance = function (value) { + // XXX: for now, we don't calculate the actual distance between, say, + // polygon and circle. If people care about this use-case it will get + // a priority. + if (!value) + return null; + if(!value.type) + return GeoJSON.pointDistance(point, + { type: "Point", coordinates: pointToArray(value) }); + if (value.type === "Point") { + return GeoJSON.pointDistance(point, value); + } else { + return GeoJSON.geometryWithinRadius(value, point, maxDistance) + ? 0 : maxDistance + 1; + } + }; + } else { + maxDistance = valueSelector.$maxDistance; + if (!isIndexable(operand)) + throw Error("$near argument must be coordinate pair or GeoJSON"); + point = pointToArray(operand); + distance = function (value) { + if (!isIndexable(value)) + return null; + return distanceCoordinatePairs(point, value); + }; + } + + return function (branchedValues) { + // There might be multiple points in the document that match the given + // field. Only one of them needs to be within $maxDistance, but we need to + // evaluate all of them and use the nearest one for the implicit sort + // specifier. (That's why we can't just use ELEMENT_OPERATORS here.) + // + // Note: This differs from MongoDB's implementation, where a document will + // actually show up *multiple times* in the result set, with one entry for + // each within-$maxDistance branching point. + branchedValues = expandArraysInBranches(branchedValues); + var result = {result: false}; + branchedValues.every(function (branch) { + // if operation is an update, don't skip branches, just return the first one (#3599) + if (!matcher._isUpdate){ + if (!(typeof branch.value === "object")){ + return true; + } + var curDistance = distance(branch.value); + // Skip branches that aren't real points or are too far away. + if (curDistance === null || curDistance > maxDistance) + return true; + // Skip anything that's a tie. + if (result.distance !== undefined && result.distance <= curDistance) + return true; + } + result.result = true; + result.distance = curDistance; + if (!branch.arrayIndices) + delete result.arrayIndices; + else + result.arrayIndices = branch.arrayIndices; + if (matcher._isUpdate) + return false; + return true; + }); + return result; + }; + } +}; + +// NB: We are cheating and using this function to implement 'AND' for both +// 'document matchers' and 'branched matchers'. They both return result objects +// but the argument is different: for the former it's a whole doc, whereas for +// the latter it's an array of 'branched values'. +function andSomeMatchers (subMatchers) { + if (subMatchers.length === 0) + return everythingMatcher; + if (subMatchers.length === 1) + return subMatchers[0]; + + return function (docOrBranches) { + var ret = {}; + ret.result = subMatchers.every(function (f) { + var subResult = f(docOrBranches); + // Copy a 'distance' number out of the first sub-matcher that has + // one. Yes, this means that if there are multiple $near fields in a + // query, something arbitrary happens; this appears to be consistent with + // Mongo. + if (subResult.result && subResult.distance !== undefined + && ret.distance === undefined) { + ret.distance = subResult.distance; + } + // Similarly, propagate arrayIndices from sub-matchers... but to match + // MongoDB behavior, this time the *last* sub-matcher with arrayIndices + // wins. + if (subResult.result && subResult.arrayIndices) { + ret.arrayIndices = subResult.arrayIndices; + } + return subResult.result; + }); + + // If we didn't actually match, forget any extra metadata we came up with. + if (!ret.result) { + delete ret.distance; + delete ret.arrayIndices; + } + return ret; + }; +} + +const andDocumentMatchers = andSomeMatchers; +const andBranchedMatchers = andSomeMatchers; + +function compileArrayOfDocumentSelectors (selectors, matcher, inElemMatch) { + if (!Array.isArray(selectors) || selectors.length === 0) + throw Error('$and/$or/$nor must be nonempty array'); + return selectors.map(function (subSelector) { + if (!LocalCollection._isPlainObject(subSelector)) + throw Error('$or/$and/$nor entries need to be full objects'); + return compileDocumentSelector( + subSelector, matcher, {inElemMatch: inElemMatch}); + }); +} + +// Takes in a selector that could match a full document (eg, the original +// selector). Returns a function mapping document->result object. +// +// matcher is the Matcher object we are compiling. +// +// If this is the root document selector (ie, not wrapped in $and or the like), +// then isRoot is true. (This is used by $near.) +function compileDocumentSelector (docSelector, matcher, options = {}) { + let docMatchers = []; + Object.keys(docSelector).forEach(function (key) { + let subSelector = docSelector[key]; + if (key.substr(0, 1) === '$') { + // Outer operators are either logical operators (they recurse back into + // this function), or $where. + if (!LOGICAL_OPERATORS.hasOwnProperty(key)) + throw new Error(`Unrecognized logical operator: ${key}`); + matcher._isSimple = false; + docMatchers.push(LOGICAL_OPERATORS[key](subSelector, matcher, + options.inElemMatch)); + } else { + // Record this path, but only if we aren't in an elemMatcher, since in an + // elemMatch this is a path inside an object in an array, not in the doc + // root. + if (!options.inElemMatch) + matcher._recordPathUsed(key); + let lookUpByIndex = makeLookupFunction(key); + let valueMatcher = + compileValueSelector(subSelector, matcher, options.isRoot); + docMatchers.push(function (doc) { + let branchValues = lookUpByIndex(doc); + return valueMatcher(branchValues); + }); + } + }); + + return andDocumentMatchers(docMatchers); +} + +// Takes in a selector that could match a key-indexed value in a document; eg, +// {$gt: 5, $lt: 9}, or a regular expression, or any non-expression object (to +// indicate equality). Returns a branched matcher: a function mapping +// [branched value]->result object. +function compileValueSelector (valueSelector, matcher, isRoot) { + if (valueSelector instanceof RegExp) { + matcher._isSimple = false; + return convertElementMatcherToBranchedMatcher( + regexpElementMatcher(valueSelector)); + } else if (isOperatorObject(valueSelector)) { + return operatorBranchedMatcher(valueSelector, matcher, isRoot); + } else { + return convertElementMatcherToBranchedMatcher( + equalityElementMatcher(valueSelector)); + } +} + +// Given an element matcher (which evaluates a single value), returns a branched +// value (which evaluates the element matcher on all the branches and returns a +// more structured return value possibly including arrayIndices). +function convertElementMatcherToBranchedMatcher (elementMatcher, options) { + options = options || {}; + return function (branches) { + var expanded = branches; + if (!options.dontExpandLeafArrays) { + expanded = expandArraysInBranches( + branches, options.dontIncludeLeafArrays); + } + var ret = {}; + ret.result = expanded.some(function (element) { + var matched = elementMatcher(element.value); + + // Special case for $elemMatch: it means "true, and use this as an array + // index if I didn't already have one". + if (typeof matched === 'number') { + // XXX This code dates from when we only stored a single array index + // (for the outermost array). Should we be also including deeper array + // indices from the $elemMatch match? + if (!element.arrayIndices) + element.arrayIndices = [matched]; + matched = true; + } + + // If some element matched, and it's tagged with array indices, include + // those indices in our result object. + if (matched && element.arrayIndices) + ret.arrayIndices = element.arrayIndices; + + return matched; + }); + return ret; + }; +} + +// Helpers for $near. +function distanceCoordinatePairs (a, b) { + a = pointToArray(a); + b = pointToArray(b); + var x = a[0] - b[0]; + var y = a[1] - b[1]; + if (Number.isNaN(x) || Number.isNaN(y)) + return null; + return Math.sqrt(x * x + y * y); +} + +// Takes something that is not an operator object and returns an element matcher +// for equality with that thing. +function equalityElementMatcher (elementSelector) { + if (isOperatorObject(elementSelector)) + throw Error("Can't create equalityValueSelector for operator object"); + + // Special-case: null and undefined are equal (if you got undefined in there + // somewhere, or if you got it due to some branch being non-existent in the + // weird special case), even though they aren't with EJSON.equals. + if (elementSelector == null) { // undefined or null + return function (value) { + return value == null; // undefined or null + }; + } + + return function (value) { + return LocalCollection._f._equal(elementSelector, value); + }; +} + +function everythingMatcher (docOrBranchedValues) { + return {result: true}; +} + +function expandArraysInBranches (branches, skipTheArrays) { + var branchesOut = []; + branches.forEach(function (branch) { + var thisIsArray = Array.isArray(branch.value); + // We include the branch itself, *UNLESS* we it's an array that we're going + // to iterate and we're told to skip arrays. (That's right, we include some + // arrays even skipTheArrays is true: these are arrays that were found via + // explicit numerical indices.) + if (!(skipTheArrays && thisIsArray && !branch.dontIterate)) { + branchesOut.push({ + value: branch.value, + arrayIndices: branch.arrayIndices + }); + } + if (thisIsArray && !branch.dontIterate) { + branch.value.forEach(function (leaf, i) { + branchesOut.push({ + value: leaf, + arrayIndices: (branch.arrayIndices || []).concat(i) + }); + }); + } + }); + return branchesOut; +} + +// Helpers for $bitsAllSet/$bitsAnySet/$bitsAllClear/$bitsAnyClear. +function getOperandBitmask (operand, selector) { + // numeric bitmask + // You can provide a numeric bitmask to be matched against the operand field. It must be representable as a non-negative 32-bit signed integer. + // Otherwise, $bitsAllSet will return an error. + if (Number.isInteger(operand) && operand >= 0) { + return new Uint8Array(new Int32Array([operand]).buffer) + } + // bindata bitmask + // You can also use an arbitrarily large BinData instance as a bitmask. + else if (EJSON.isBinary(operand)) { + return new Uint8Array(operand.buffer) + } + // position list + // If querying a list of bit positions, each must be a non-negative integer. Bit positions start at 0 from the least significant bit. + else if (Array.isArray(operand) && operand.every(function (e) { + return Number.isInteger(e) && e >= 0 + })) { + var buffer = new ArrayBuffer((Math.max(...operand) >> 3) + 1) + var view = new Uint8Array(buffer) + operand.forEach(function (x) { + view[x >> 3] |= (1 << (x & 0x7)) + }) + return view + } + // bad operand + else { + throw Error(`operand to ${selector} must be a numeric bitmask (representable as a non-negative 32-bit signed integer), a bindata bitmask or an array with bit positions (non-negative integers)`) + } +} + +function getValueBitmask (value, length) { + // The field value must be either numerical or a BinData instance. Otherwise, $bits... will not match the current document. + // numerical + if (Number.isSafeInteger(value)) { + // $bits... will not match numerical values that cannot be represented as a signed 64-bit integer + // This can be the case if a value is either too large or small to fit in a signed 64-bit integer, or if it has a fractional component. + var buffer = new ArrayBuffer(Math.max(length, 2 * Uint32Array.BYTES_PER_ELEMENT)); + var view = new Uint32Array(buffer, 0, 2) + view[0] = (value % ((1 << 16) * (1 << 16))) | 0 + view[1] = (value / ((1 << 16) * (1 << 16))) | 0 + // sign extension + if (value < 0) { + view = new Uint8Array(buffer, 2) + view.forEach(function (byte, idx) { + view[idx] = 0xff + }) + } + return new Uint8Array(buffer) + } + // bindata + else if (EJSON.isBinary(value)) { + return new Uint8Array(value.buffer) + } + // no match + return false +} + +// Returns a branched matcher that matches iff the given matcher does not. +// Note that this implicitly "deMorganizes" the wrapped function. ie, it +// means that ALL branch values need to fail to match innerBranchedMatcher. +function invertBranchedMatcher (branchedMatcher) { + return function (branchValues) { + var invertMe = branchedMatcher(branchValues); + // We explicitly choose to strip arrayIndices here: it doesn't make sense to + // say "update the array element that does not match something", at least + // in mongo-land. + return {result: !invertMe.result}; + }; +} + +// Helper for $lt/$gt/$lte/$gte. +function makeInequality (cmpValueComparator) { + return { + compileElementSelector: function (operand) { + // Arrays never compare false with non-arrays for any inequality. + // XXX This was behavior we observed in pre-release MongoDB 2.5, but + // it seems to have been reverted. + // See https://jira.mongodb.org/browse/SERVER-11444 + if (Array.isArray(operand)) { + return function () { + return false; + }; + } + + // Special case: consider undefined and null the same (so true with + // $gte/$lte). + if (operand === undefined) + operand = null; + + var operandType = LocalCollection._f._type(operand); + + return function (value) { + if (value === undefined) + value = null; + // Comparisons are never true among things of different type (except + // null vs undefined). + if (LocalCollection._f._type(value) !== operandType) + return false; + return cmpValueComparator(LocalCollection._f._cmp(value, operand)); + }; + } + }; +} + +// makeLookupFunction(key) returns a lookup function. +// +// A lookup function takes in a document and returns an array of matching +// branches. If no arrays are found while looking up the key, this array will +// have exactly one branches (possibly 'undefined', if some segment of the key +// was not found). +// +// If arrays are found in the middle, this can have more than one element, since +// we 'branch'. When we 'branch', if there are more key segments to look up, +// then we only pursue branches that are plain objects (not arrays or scalars). +// This means we can actually end up with no branches! +// +// We do *NOT* branch on arrays that are found at the end (ie, at the last +// dotted member of the key). We just return that array; if you want to +// effectively 'branch' over the array's values, post-process the lookup +// function with expandArraysInBranches. +// +// Each branch is an object with keys: +// - value: the value at the branch +// - dontIterate: an optional bool; if true, it means that 'value' is an array +// that expandArraysInBranches should NOT expand. This specifically happens +// when there is a numeric index in the key, and ensures the +// perhaps-surprising MongoDB behavior where {'a.0': 5} does NOT +// match {a: [[5]]}. +// - arrayIndices: if any array indexing was done during lookup (either due to +// explicit numeric indices or implicit branching), this will be an array of +// the array indices used, from outermost to innermost; it is falsey or +// absent if no array index is used. If an explicit numeric index is used, +// the index will be followed in arrayIndices by the string 'x'. +// +// Note: arrayIndices is used for two purposes. First, it is used to +// implement the '$' modifier feature, which only ever looks at its first +// element. +// +// Second, it is used for sort key generation, which needs to be able to tell +// the difference between different paths. Moreover, it needs to +// differentiate between explicit and implicit branching, which is why +// there's the somewhat hacky 'x' entry: this means that explicit and +// implicit array lookups will have different full arrayIndices paths. (That +// code only requires that different paths have different arrayIndices; it +// doesn't actually 'parse' arrayIndices. As an alternative, arrayIndices +// could contain objects with flags like 'implicit', but I think that only +// makes the code surrounding them more complex.) +// +// (By the way, this field ends up getting passed around a lot without +// cloning, so never mutate any arrayIndices field/var in this package!) +// +// +// At the top level, you may only pass in a plain object or array. +// +// See the test 'minimongo - lookup' for some examples of what lookup functions +// return. +function makeLookupFunction (key, options) { + options = options || {}; + var parts = key.split('.'); + var firstPart = parts.length ? parts[0] : ''; + var firstPartIsNumeric = isNumericKey(firstPart); + var nextPartIsNumeric = parts.length >= 2 && isNumericKey(parts[1]); + var lookupRest; + if (parts.length > 1) { + lookupRest = makeLookupFunction(parts.slice(1).join('.')); + } + + var omitUnnecessaryFields = function (retVal) { + if (!retVal.dontIterate) + delete retVal.dontIterate; + if (retVal.arrayIndices && !retVal.arrayIndices.length) + delete retVal.arrayIndices; + return retVal; + }; + + // Doc will always be a plain object or an array. + // apply an explicit numeric index, an array. + return function (doc, arrayIndices) { + if (!arrayIndices) + arrayIndices = []; + + if (Array.isArray(doc)) { + // If we're being asked to do an invalid lookup into an array (non-integer + // or out-of-bounds), return no results (which is different from returning + // a single undefined result, in that `null` equality checks won't match). + if (!(firstPartIsNumeric && firstPart < doc.length)) + return []; + + // Remember that we used this array index. Include an 'x' to indicate that + // the previous index came from being considered as an explicit array + // index (not branching). + arrayIndices = arrayIndices.concat(+firstPart, 'x'); + } + + // Do our first lookup. + var firstLevel = doc[firstPart]; + + // If there is no deeper to dig, return what we found. + // + // If what we found is an array, most value selectors will choose to treat + // the elements of the array as matchable values in their own right, but + // that's done outside of the lookup function. (Exceptions to this are $size + // and stuff relating to $elemMatch. eg, {a: {$size: 2}} does not match {a: + // [[1, 2]]}.) + // + // That said, if we just did an *explicit* array lookup (on doc) to find + // firstLevel, and firstLevel is an array too, we do NOT want value + // selectors to iterate over it. eg, {'a.0': 5} does not match {a: [[5]]}. + // So in that case, we mark the return value as 'don't iterate'. + if (!lookupRest) { + return [omitUnnecessaryFields({ + value: firstLevel, + dontIterate: Array.isArray(doc) && Array.isArray(firstLevel), + arrayIndices: arrayIndices})]; + } + + // We need to dig deeper. But if we can't, because what we've found is not + // an array or plain object, we're done. If we just did a numeric index into + // an array, we return nothing here (this is a change in Mongo 2.5 from + // Mongo 2.4, where {'a.0.b': null} stopped matching {a: [5]}). Otherwise, + // return a single `undefined` (which can, for example, match via equality + // with `null`). + if (!isIndexable(firstLevel)) { + if (Array.isArray(doc)) + return []; + return [omitUnnecessaryFields({value: undefined, + arrayIndices: arrayIndices})]; + } + + var result = []; + var appendToResult = function (more) { + Array.prototype.push.apply(result, more); + }; + + // Dig deeper: look up the rest of the parts on whatever we've found. + // (lookupRest is smart enough to not try to do invalid lookups into + // firstLevel if it's an array.) + appendToResult(lookupRest(firstLevel, arrayIndices)); + + // If we found an array, then in *addition* to potentially treating the next + // part as a literal integer lookup, we should also 'branch': try to look up + // the rest of the parts on each array element in parallel. + // + // In this case, we *only* dig deeper into array elements that are plain + // objects. (Recall that we only got this far if we have further to dig.) + // This makes sense: we certainly don't dig deeper into non-indexable + // objects. And it would be weird to dig into an array: it's simpler to have + // a rule that explicit integer indexes only apply to an outer array, not to + // an array you find after a branching search. + // + // In the special case of a numeric part in a *sort selector* (not a query + // selector), we skip the branching: we ONLY allow the numeric part to mean + // 'look up this index' in that case, not 'also look up this index in all + // the elements of the array'. + if (Array.isArray(firstLevel) && !(nextPartIsNumeric && options.forSort)) { + firstLevel.forEach(function (branch, arrayIndex) { + if (LocalCollection._isPlainObject(branch)) { + appendToResult(lookupRest( + branch, + arrayIndices.concat(arrayIndex))); + } + }); + } + + return result; + }; +} + +MinimongoTest.makeLookupFunction = makeLookupFunction; + +function nothingMatcher (docOrBranchedValues) { + return {result: false}; +} + +// Takes an operator object (an object with $ keys) and returns a branched +// matcher for it. +function operatorBranchedMatcher (valueSelector, matcher, isRoot) { + // Each valueSelector works separately on the various branches. So one + // operator can match one branch and another can match another branch. This + // is OK. + + var operatorMatchers = []; + Object.keys(valueSelector).forEach(function (operator) { + var operand = valueSelector[operator]; + var simpleRange = ['$lt', '$lte', '$gt', '$gte'].includes(operator) && + typeof operand === 'number'; + var simpleEquality = ['$ne', '$eq'].includes(operator) && operand !== Object(operand); + var simpleInclusion = ['$in', '$nin'].includes(operator) && + Array.isArray(operand) && !operand.some(function (x) { return x === Object(x); }); + + if (! (simpleRange || simpleInclusion || simpleEquality)) { + matcher._isSimple = false; + } + + if (VALUE_OPERATORS.hasOwnProperty(operator)) { + operatorMatchers.push( + VALUE_OPERATORS[operator](operand, valueSelector, matcher, isRoot)); + } else if (ELEMENT_OPERATORS.hasOwnProperty(operator)) { + var options = ELEMENT_OPERATORS[operator]; + operatorMatchers.push( + convertElementMatcherToBranchedMatcher( + options.compileElementSelector( + operand, valueSelector, matcher), + options)); + } else { + throw new Error("Unrecognized operator: " + operator); + } + }); + + return andBranchedMatchers(operatorMatchers); +} + +// Makes sure we get 2 elements array and assume the first one to be x and +// the second one to y no matter what user passes. +// In case user passes { lon: x, lat: y } returns [x, y] +function pointToArray (point) { + return Array.isArray(point) ? point.slice() : [point.x, point.y]; +} + +// Takes a RegExp object and returns an element matcher. +function regexpElementMatcher (regexp) { + return function (value) { + if (value instanceof RegExp) { + return value.toString() === regexp.toString(); + } + // Regexps only work against strings. + if (typeof value !== 'string') + return false; + + // Reset regexp's state to avoid inconsistent matching for objects with the + // same value on consecutive calls of regexp.test. This happens only if the + // regexp has the 'g' flag. Also note that ES6 introduces a new flag 'y' for + // which we should *not* change the lastIndex but MongoDB doesn't support + // either of these flags. + regexp.lastIndex = 0; + + return regexp.test(value); + }; +} diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 027de20b11..1470a57366 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -1,3 +1,10 @@ +import {Matcher} from './matcher'; +import { + isIndexable, + isNumericKey, + isOperatorObject, +} from './common.js'; + // Make sure field names do not contain Mongo restricted // characters ('.', '$', '\0'). // https://docs.mongodb.com/manual/reference/limits/#Restrictions-on-Field-Names @@ -60,7 +67,7 @@ LocalCollection = function (name) { self.paused = false; }; -Minimongo = {}; +Minimongo = {Matcher}; // Object exported only for unit testing. // Use it to export private functions to test in Tinytest. @@ -1187,7 +1194,7 @@ LocalCollection.wrapTransform = function (transform) { return transform(doc); }); - if (!isPlainObject(transformed)) { + if (!LocalCollection._isPlainObject(transformed)) { throw new Error("transform must return object"); } @@ -1204,1308 +1211,14 @@ LocalCollection.wrapTransform = function (transform) { return wrapped; }; -// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as -// arrays. -// XXX maybe this should be EJSON.isArray -isArray = function (x) { - return Array.isArray(x) && !EJSON.isBinary(x); -}; - // XXX maybe this should be EJSON.isObject, though EJSON doesn't know about // RegExp // XXX note that _type(undefined) === 3!!!! -isPlainObject = LocalCollection._isPlainObject = function (x) { +LocalCollection._isPlainObject = function (x) { return x && LocalCollection._f._type(x) === 3; }; -isIndexable = function (x) { - return isArray(x) || isPlainObject(x); -}; - -// Returns true if this is an object with at least one key and all keys begin -// with $. Unless inconsistentOK is set, throws if some keys begin with $ and -// others don't. -isOperatorObject = function (valueSelector, inconsistentOK) { - if (!isPlainObject(valueSelector)) - return false; - - var theseAreOperators = undefined; - Object.keys(valueSelector).forEach(function (selKey) { - var thisIsOperator = selKey.substr(0, 1) === '$'; - if (theseAreOperators === undefined) { - theseAreOperators = thisIsOperator; - } else if (theseAreOperators !== thisIsOperator) { - if (!inconsistentOK) - throw new Error("Inconsistent operator: " + - JSON.stringify(valueSelector)); - theseAreOperators = false; - } - }); - return !!theseAreOperators; // {} has no operators -}; - - -// string can be converted to integer -isNumericKey = function (s) { - return /^[0-9]+$/.test(s); -}; - -// The minimongo selector compiler! - -// Terminology: -// - a "selector" is the EJSON object representing a selector -// - a "matcher" is its compiled form (whether a full Minimongo.Matcher -// object or one of the component lambdas that matches parts of it) -// - a "result object" is an object with a "result" field and maybe -// distance and arrayIndices. -// - a "branched value" is an object with a "value" field and maybe -// "dontIterate" and "arrayIndices". -// - a "document" is a top-level object that can be stored in a collection. -// - a "lookup function" is a function that takes in a document and returns -// an array of "branched values". -// - a "branched matcher" maps from an array of branched values to a result -// object. -// - an "element matcher" maps from a single value to a bool. - -// Main entry point. -// var matcher = new Minimongo.Matcher({a: {$gt: 5}}); -// if (matcher.documentMatches({a: 7})) ... -Minimongo.Matcher = function (selector, isUpdate = false) { - var self = this; - // A set (object mapping string -> *) of all of the document paths looked - // at by the selector. Also includes the empty string if it may look at any - // path (eg, $where). - self._paths = {}; - // Set to true if compilation finds a $near. - self._hasGeoQuery = false; - // Set to true if compilation finds a $where. - self._hasWhere = false; - // Set to false if compilation finds anything other than a simple equality or - // one or more of '$gt', '$gte', '$lt', '$lte', '$ne', '$in', '$nin' used with - // scalars as operands. - self._isSimple = true; - // Set to a dummy document which always matches this Matcher. Or set to null - // if such document is too hard to find. - self._matchingDocument = undefined; - // A clone of the original selector. It may just be a function if the user - // passed in a function; otherwise is definitely an object (eg, IDs are - // translated into {_id: ID} first. Used by canBecomeTrueByModifier and - // Sorter._useWithMatcher. - self._selector = null; - self._docMatcher = self._compileSelector(selector); - // Set to true if selection is done for an update operation - // Default is false - // Used for $near array update (issue #3599) - self._isUpdate = isUpdate; -}; - -Object.assign(Minimongo.Matcher.prototype, { - documentMatches: function (doc) { - if (!doc || typeof doc !== "object") { - throw Error("documentMatches needs a document"); - } - return this._docMatcher(doc); - }, - hasGeoQuery: function () { - return this._hasGeoQuery; - }, - hasWhere: function () { - return this._hasWhere; - }, - isSimple: function () { - return this._isSimple; - }, - - // Given a selector, return a function that takes one argument, a - // document. It returns a result object. - _compileSelector: function (selector) { - var self = this; - // you can pass a literal function instead of a selector - if (selector instanceof Function) { - self._isSimple = false; - self._selector = selector; - self._recordPathUsed(''); - return function (doc) { - return {result: !!selector.call(doc)}; - }; - } - - // shorthand -- scalars match _id - if (LocalCollection._selectorIsId(selector)) { - self._selector = {_id: selector}; - self._recordPathUsed('_id'); - return function (doc) { - return {result: EJSON.equals(doc._id, selector)}; - }; - } - - // protect against dangerous selectors. falsey and {_id: falsey} are both - // likely programmer error, and not what you want, particularly for - // destructive operations. - if (!selector || (('_id' in selector) && !selector._id)) { - self._isSimple = false; - return nothingMatcher; - } - - // Top level can't be an array or true or binary. - if (typeof(selector) === 'boolean' || isArray(selector) || - EJSON.isBinary(selector)) - throw new Error("Invalid selector: " + selector); - - self._selector = EJSON.clone(selector); - return compileDocumentSelector(selector, self, {isRoot: true}); - }, - _recordPathUsed: function (path) { - this._paths[path] = true; - }, - // Returns a list of key paths the given selector is looking for. It includes - // the empty string if there is a $where. - _getPaths: function () { - return Object.keys(this._paths); - } -}); - - -// Takes in a selector that could match a full document (eg, the original -// selector). Returns a function mapping document->result object. -// -// matcher is the Matcher object we are compiling. -// -// If this is the root document selector (ie, not wrapped in $and or the like), -// then isRoot is true. (This is used by $near.) -var compileDocumentSelector = function (docSelector, matcher, options) { - options = options || {}; - var docMatchers = []; - Object.keys(docSelector).forEach(function (key) { - var subSelector = docSelector[key]; - if (key.substr(0, 1) === '$') { - // Outer operators are either logical operators (they recurse back into - // this function), or $where. - if (!LOGICAL_OPERATORS.hasOwnProperty(key)) - throw new Error("Unrecognized logical operator: " + key); - matcher._isSimple = false; - docMatchers.push(LOGICAL_OPERATORS[key](subSelector, matcher, - options.inElemMatch)); - } else { - // Record this path, but only if we aren't in an elemMatcher, since in an - // elemMatch this is a path inside an object in an array, not in the doc - // root. - if (!options.inElemMatch) - matcher._recordPathUsed(key); - var lookUpByIndex = makeLookupFunction(key); - var valueMatcher = - compileValueSelector(subSelector, matcher, options.isRoot); - docMatchers.push(function (doc) { - var branchValues = lookUpByIndex(doc); - return valueMatcher(branchValues); - }); - } - }); - - return andDocumentMatchers(docMatchers); -}; - -// Takes in a selector that could match a key-indexed value in a document; eg, -// {$gt: 5, $lt: 9}, or a regular expression, or any non-expression object (to -// indicate equality). Returns a branched matcher: a function mapping -// [branched value]->result object. -var compileValueSelector = function (valueSelector, matcher, isRoot) { - if (valueSelector instanceof RegExp) { - matcher._isSimple = false; - return convertElementMatcherToBranchedMatcher( - regexpElementMatcher(valueSelector)); - } else if (isOperatorObject(valueSelector)) { - return operatorBranchedMatcher(valueSelector, matcher, isRoot); - } else { - return convertElementMatcherToBranchedMatcher( - equalityElementMatcher(valueSelector)); - } -}; - -// Given an element matcher (which evaluates a single value), returns a branched -// value (which evaluates the element matcher on all the branches and returns a -// more structured return value possibly including arrayIndices). -var convertElementMatcherToBranchedMatcher = function ( - elementMatcher, options) { - options = options || {}; - return function (branches) { - var expanded = branches; - if (!options.dontExpandLeafArrays) { - expanded = expandArraysInBranches( - branches, options.dontIncludeLeafArrays); - } - var ret = {}; - ret.result = expanded.some(function (element) { - var matched = elementMatcher(element.value); - - // Special case for $elemMatch: it means "true, and use this as an array - // index if I didn't already have one". - if (typeof matched === 'number') { - // XXX This code dates from when we only stored a single array index - // (for the outermost array). Should we be also including deeper array - // indices from the $elemMatch match? - if (!element.arrayIndices) - element.arrayIndices = [matched]; - matched = true; - } - - // If some element matched, and it's tagged with array indices, include - // those indices in our result object. - if (matched && element.arrayIndices) - ret.arrayIndices = element.arrayIndices; - - return matched; - }); - return ret; - }; -}; - -// Takes a RegExp object and returns an element matcher. -regexpElementMatcher = function (regexp) { - return function (value) { - if (value instanceof RegExp) { - return value.toString() === regexp.toString(); - } - // Regexps only work against strings. - if (typeof value !== 'string') - return false; - - // Reset regexp's state to avoid inconsistent matching for objects with the - // same value on consecutive calls of regexp.test. This happens only if the - // regexp has the 'g' flag. Also note that ES6 introduces a new flag 'y' for - // which we should *not* change the lastIndex but MongoDB doesn't support - // either of these flags. - regexp.lastIndex = 0; - - return regexp.test(value); - }; -}; - -// Takes something that is not an operator object and returns an element matcher -// for equality with that thing. -equalityElementMatcher = function (elementSelector) { - if (isOperatorObject(elementSelector)) - throw Error("Can't create equalityValueSelector for operator object"); - - // Special-case: null and undefined are equal (if you got undefined in there - // somewhere, or if you got it due to some branch being non-existent in the - // weird special case), even though they aren't with EJSON.equals. - if (elementSelector == null) { // undefined or null - return function (value) { - return value == null; // undefined or null - }; - } - - return function (value) { - return LocalCollection._f._equal(elementSelector, value); - }; -}; - -// Takes an operator object (an object with $ keys) and returns a branched -// matcher for it. -var operatorBranchedMatcher = function (valueSelector, matcher, isRoot) { - // Each valueSelector works separately on the various branches. So one - // operator can match one branch and another can match another branch. This - // is OK. - - var operatorMatchers = []; - Object.keys(valueSelector).forEach(function (operator) { - var operand = valueSelector[operator]; - var simpleRange = ['$lt', '$lte', '$gt', '$gte'].includes(operator) && - typeof operand === 'number'; - var simpleEquality = ['$ne', '$eq'].includes(operator) && operand !== Object(operand); - var simpleInclusion = ['$in', '$nin'].includes(operator) && - Array.isArray(operand) && !operand.some(function (x) { return x === Object(x); }); - - if (! (simpleRange || simpleInclusion || simpleEquality)) { - matcher._isSimple = false; - } - - if (VALUE_OPERATORS.hasOwnProperty(operator)) { - operatorMatchers.push( - VALUE_OPERATORS[operator](operand, valueSelector, matcher, isRoot)); - } else if (ELEMENT_OPERATORS.hasOwnProperty(operator)) { - var options = ELEMENT_OPERATORS[operator]; - operatorMatchers.push( - convertElementMatcherToBranchedMatcher( - options.compileElementSelector( - operand, valueSelector, matcher), - options)); - } else { - throw new Error("Unrecognized operator: " + operator); - } - }); - - return andBranchedMatchers(operatorMatchers); -}; - -var compileArrayOfDocumentSelectors = function ( - selectors, matcher, inElemMatch) { - if (!isArray(selectors) || selectors.length === 0) - throw Error("$and/$or/$nor must be nonempty array"); - return selectors.map(function (subSelector) { - if (!isPlainObject(subSelector)) - throw Error("$or/$and/$nor entries need to be full objects"); - return compileDocumentSelector( - subSelector, matcher, {inElemMatch: inElemMatch}); - }); -}; - -// Operators that appear at the top level of a document selector. -var LOGICAL_OPERATORS = { - $and: function (subSelector, matcher, inElemMatch) { - var matchers = compileArrayOfDocumentSelectors( - subSelector, matcher, inElemMatch); - return andDocumentMatchers(matchers); - }, - - $or: function (subSelector, matcher, inElemMatch) { - var matchers = compileArrayOfDocumentSelectors( - subSelector, matcher, inElemMatch); - - // Special case: if there is only one matcher, use it directly, *preserving* - // any arrayIndices it returns. - if (matchers.length === 1) - return matchers[0]; - - return function (doc) { - var result = matchers.some(function (f) { - return f(doc).result; - }); - // $or does NOT set arrayIndices when it has multiple - // sub-expressions. (Tested against MongoDB.) - return {result: result}; - }; - }, - - $nor: function (subSelector, matcher, inElemMatch) { - var matchers = compileArrayOfDocumentSelectors( - subSelector, matcher, inElemMatch); - return function (doc) { - var result = matchers.every(function (f) { - return !f(doc).result; - }); - // Never set arrayIndices, because we only match if nothing in particular - // "matched" (and because this is consistent with MongoDB). - return {result: result}; - }; - }, - - $where: function (selectorValue, matcher) { - // Record that *any* path may be used. - matcher._recordPathUsed(''); - matcher._hasWhere = true; - if (!(selectorValue instanceof Function)) { - // XXX MongoDB seems to have more complex logic to decide where or or not - // to add "return"; not sure exactly what it is. - selectorValue = Function("obj", "return " + selectorValue); - } - return function (doc) { - // We make the document available as both `this` and `obj`. - // XXX not sure what we should do if this throws - return {result: selectorValue.call(doc, doc)}; - }; - }, - - // This is just used as a comment in the query (in MongoDB, it also ends up in - // query logs); it has no effect on the actual selection. - $comment: function () { - return function () { - return {result: true}; - }; - } -}; - -// Returns a branched matcher that matches iff the given matcher does not. -// Note that this implicitly "deMorganizes" the wrapped function. ie, it -// means that ALL branch values need to fail to match innerBranchedMatcher. -var invertBranchedMatcher = function (branchedMatcher) { - return function (branchValues) { - var invertMe = branchedMatcher(branchValues); - // We explicitly choose to strip arrayIndices here: it doesn't make sense to - // say "update the array element that does not match something", at least - // in mongo-land. - return {result: !invertMe.result}; - }; -}; - -// Operators that (unlike LOGICAL_OPERATORS) pertain to individual paths in a -// document, but (unlike ELEMENT_OPERATORS) do not have a simple definition as -// "match each branched value independently and combine with -// convertElementMatcherToBranchedMatcher". -var VALUE_OPERATORS = { - $eq: function (operand) { - return convertElementMatcherToBranchedMatcher( - equalityElementMatcher(operand)); - }, - $not: function (operand, valueSelector, matcher) { - return invertBranchedMatcher(compileValueSelector(operand, matcher)); - }, - $ne: function (operand) { - return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( - equalityElementMatcher(operand))); - }, - $nin: function (operand) { - return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( - ELEMENT_OPERATORS.$in.compileElementSelector(operand))); - }, - $exists: function (operand) { - var exists = convertElementMatcherToBranchedMatcher(function (value) { - return value !== undefined; - }); - return operand ? exists : invertBranchedMatcher(exists); - }, - // $options just provides options for $regex; its logic is inside $regex - $options: function (operand, valueSelector) { - if (!valueSelector.hasOwnProperty('$regex')) - throw Error("$options needs a $regex"); - return everythingMatcher; - }, - // $maxDistance is basically an argument to $near - $maxDistance: function (operand, valueSelector) { - if (!valueSelector.$near) - throw Error("$maxDistance needs a $near"); - return everythingMatcher; - }, - $all: function (operand, valueSelector, matcher) { - if (!isArray(operand)) - throw Error("$all requires array"); - // Not sure why, but this seems to be what MongoDB does. - if (operand.length === 0) - return nothingMatcher; - - var branchedMatchers = []; - operand.forEach(function (criterion) { - // XXX handle $all/$elemMatch combination - if (isOperatorObject(criterion)) - throw Error("no $ expressions in $all"); - // This is always a regexp or equality selector. - branchedMatchers.push(compileValueSelector(criterion, matcher)); - }); - // andBranchedMatchers does NOT require all selectors to return true on the - // SAME branch. - return andBranchedMatchers(branchedMatchers); - }, - $near: function (operand, valueSelector, matcher, isRoot) { - if (!isRoot) - throw Error("$near can't be inside another $ operator"); - matcher._hasGeoQuery = true; - - // There are two kinds of geodata in MongoDB: legacy coordinate pairs and - // GeoJSON. They use different distance metrics, too. GeoJSON queries are - // marked with a $geometry property, though legacy coordinates can be - // matched using $geometry. - - var maxDistance, point, distance; - if (isPlainObject(operand) && operand.hasOwnProperty('$geometry')) { - // GeoJSON "2dsphere" mode. - maxDistance = operand.$maxDistance; - point = operand.$geometry; - distance = function (value) { - // XXX: for now, we don't calculate the actual distance between, say, - // polygon and circle. If people care about this use-case it will get - // a priority. - if (!value) - return null; - if(!value.type) - return GeoJSON.pointDistance(point, - { type: "Point", coordinates: pointToArray(value) }); - if (value.type === "Point") { - return GeoJSON.pointDistance(point, value); - } else { - return GeoJSON.geometryWithinRadius(value, point, maxDistance) - ? 0 : maxDistance + 1; - } - }; - } else { - maxDistance = valueSelector.$maxDistance; - if (!isArray(operand) && !isPlainObject(operand)) - throw Error("$near argument must be coordinate pair or GeoJSON"); - point = pointToArray(operand); - distance = function (value) { - if (!isArray(value) && !isPlainObject(value)) - return null; - return distanceCoordinatePairs(point, value); - }; - } - - return function (branchedValues) { - // There might be multiple points in the document that match the given - // field. Only one of them needs to be within $maxDistance, but we need to - // evaluate all of them and use the nearest one for the implicit sort - // specifier. (That's why we can't just use ELEMENT_OPERATORS here.) - // - // Note: This differs from MongoDB's implementation, where a document will - // actually show up *multiple times* in the result set, with one entry for - // each within-$maxDistance branching point. - branchedValues = expandArraysInBranches(branchedValues); - var result = {result: false}; - branchedValues.every(function (branch) { - // if operation is an update, don't skip branches, just return the first one (#3599) - if (!matcher._isUpdate){ - if (!(typeof branch.value === "object")){ - return true; - } - var curDistance = distance(branch.value); - // Skip branches that aren't real points or are too far away. - if (curDistance === null || curDistance > maxDistance) - return true; - // Skip anything that's a tie. - if (result.distance !== undefined && result.distance <= curDistance) - return true; - } - result.result = true; - result.distance = curDistance; - if (!branch.arrayIndices) - delete result.arrayIndices; - else - result.arrayIndices = branch.arrayIndices; - if (matcher._isUpdate) - return false; - return true; - }); - return result; - }; - } -}; - -// Helpers for $near. -var distanceCoordinatePairs = function (a, b) { - a = pointToArray(a); - b = pointToArray(b); - var x = a[0] - b[0]; - var y = a[1] - b[1]; - if (Number.isNaN(x) || Number.isNaN(y)) - return null; - return Math.sqrt(x * x + y * y); -}; -// Makes sure we get 2 elements array and assume the first one to be x and -// the second one to y no matter what user passes. -// In case user passes { lon: x, lat: y } returns [x, y] -var pointToArray = function (point) { - return Array.isArray(point) ? point.slice() : [point.x, point.y]; -}; - -// Helper for $lt/$gt/$lte/$gte. -var makeInequality = function (cmpValueComparator) { - return { - compileElementSelector: function (operand) { - // Arrays never compare false with non-arrays for any inequality. - // XXX This was behavior we observed in pre-release MongoDB 2.5, but - // it seems to have been reverted. - // See https://jira.mongodb.org/browse/SERVER-11444 - if (isArray(operand)) { - return function () { - return false; - }; - } - - // Special case: consider undefined and null the same (so true with - // $gte/$lte). - if (operand === undefined) - operand = null; - - var operandType = LocalCollection._f._type(operand); - - return function (value) { - if (value === undefined) - value = null; - // Comparisons are never true among things of different type (except - // null vs undefined). - if (LocalCollection._f._type(value) !== operandType) - return false; - return cmpValueComparator(LocalCollection._f._cmp(value, operand)); - }; - } - }; -}; - -// Helpers for $bitsAllSet/$bitsAnySet/$bitsAllClear/$bitsAnyClear. -var getOperandBitmask = function(operand, selector) { - // numeric bitmask - // You can provide a numeric bitmask to be matched against the operand field. It must be representable as a non-negative 32-bit signed integer. - // Otherwise, $bitsAllSet will return an error. - if (Number.isInteger(operand) && operand >= 0) { - return new Uint8Array(new Int32Array([operand]).buffer) - } - // bindata bitmask - // You can also use an arbitrarily large BinData instance as a bitmask. - else if (EJSON.isBinary(operand)) { - return new Uint8Array(operand.buffer) - } - // position list - // If querying a list of bit positions, each must be a non-negative integer. Bit positions start at 0 from the least significant bit. - else if (isArray(operand) && operand.every(function (e) { - return Number.isInteger(e) && e >= 0 - })) { - var buffer = new ArrayBuffer((Math.max(...operand) >> 3) + 1) - var view = new Uint8Array(buffer) - operand.forEach(function (x) { - view[x >> 3] |= (1 << (x & 0x7)) - }) - return view - } - // bad operand - else { - throw Error(`operand to ${selector} must be a numeric bitmask (representable as a non-negative 32-bit signed integer), a bindata bitmask or an array with bit positions (non-negative integers)`) - } -} -var getValueBitmask = function (value, length) { - // The field value must be either numerical or a BinData instance. Otherwise, $bits... will not match the current document. - // numerical - if (Number.isSafeInteger(value)) { - // $bits... will not match numerical values that cannot be represented as a signed 64-bit integer - // This can be the case if a value is either too large or small to fit in a signed 64-bit integer, or if it has a fractional component. - var buffer = new ArrayBuffer(Math.max(length, 2 * Uint32Array.BYTES_PER_ELEMENT)); - var view = new Uint32Array(buffer, 0, 2) - view[0] = (value % ((1 << 16) * (1 << 16))) | 0 - view[1] = (value / ((1 << 16) * (1 << 16))) | 0 - // sign extension - if (value < 0) { - view = new Uint8Array(buffer, 2) - view.forEach(function (byte, idx) { - view[idx] = 0xff - }) - } - return new Uint8Array(buffer) - } - // bindata - else if (EJSON.isBinary(value)) { - return new Uint8Array(value.buffer) - } - // no match - return false -} - -// Each element selector contains: -// - compileElementSelector, a function with args: -// - operand - the "right hand side" of the operator -// - valueSelector - the "context" for the operator (so that $regex can find -// $options) -// - matcher - the Matcher this is going into (so that $elemMatch can compile -// more things) -// returning a function mapping a single value to bool. -// - dontExpandLeafArrays, a bool which prevents expandArraysInBranches from -// being called -// - dontIncludeLeafArrays, a bool which causes an argument to be passed to -// expandArraysInBranches if it is called -ELEMENT_OPERATORS = { - $lt: makeInequality(function (cmpValue) { - return cmpValue < 0; - }), - $gt: makeInequality(function (cmpValue) { - return cmpValue > 0; - }), - $lte: makeInequality(function (cmpValue) { - return cmpValue <= 0; - }), - $gte: makeInequality(function (cmpValue) { - return cmpValue >= 0; - }), - $mod: { - compileElementSelector: function (operand) { - if (!(isArray(operand) && operand.length === 2 - && typeof(operand[0]) === 'number' - && typeof(operand[1]) === 'number')) { - throw Error("argument to $mod must be an array of two numbers"); - } - // XXX could require to be ints or round or something - var divisor = operand[0]; - var remainder = operand[1]; - return function (value) { - return typeof value === 'number' && value % divisor === remainder; - }; - } - }, - $in: { - compileElementSelector: function (operand) { - if (!isArray(operand)) - throw Error("$in needs an array"); - - var elementMatchers = []; - operand.forEach(function (option) { - if (option instanceof RegExp) - elementMatchers.push(regexpElementMatcher(option)); - else if (isOperatorObject(option)) - throw Error("cannot nest $ under $in"); - else - elementMatchers.push(equalityElementMatcher(option)); - }); - - return function (value) { - // Allow {a: {$in: [null]}} to match when 'a' does not exist. - if (value === undefined) - value = null; - return elementMatchers.some(function (e) { - return e(value); - }); - }; - } - }, - $size: { - // {a: [[5, 5]]} must match {a: {$size: 1}} but not {a: {$size: 2}}, so we - // don't want to consider the element [5,5] in the leaf array [[5,5]] as a - // possible value. - dontExpandLeafArrays: true, - compileElementSelector: function (operand) { - if (typeof operand === 'string') { - // Don't ask me why, but by experimentation, this seems to be what Mongo - // does. - operand = 0; - } else if (typeof operand !== 'number') { - throw Error("$size needs a number"); - } - return function (value) { - return isArray(value) && value.length === operand; - }; - } - }, - $type: { - // {a: [5]} must not match {a: {$type: 4}} (4 means array), but it should - // match {a: {$type: 1}} (1 means number), and {a: [[5]]} must match {$a: - // {$type: 4}}. Thus, when we see a leaf array, we *should* expand it but - // should *not* include it itself. - dontIncludeLeafArrays: true, - compileElementSelector: function (operand) { - if (typeof operand !== 'number') - throw Error("$type needs a number"); - return function (value) { - return value !== undefined - && LocalCollection._f._type(value) === operand; - }; - } - }, - $bitsAllSet: { - compileElementSelector: function (operand) { - var op = getOperandBitmask(operand, '$bitsAllSet') - return function (value) { - var bitmask = getValueBitmask(value, op.length) - return bitmask && op.every(function (byte, idx) { - return ((bitmask[idx] & byte) == byte) - }) - } - } - }, - $bitsAnySet: { - compileElementSelector: function (operand) { - var query = getOperandBitmask(operand, '$bitsAnySet') - return function (value) { - var bitmask = getValueBitmask(value, query.length) - return bitmask && query.some(function (byte, idx) { - return ((~bitmask[idx] & byte) !== byte) - }) - } - } - }, - $bitsAllClear: { - compileElementSelector: function (operand) { - var query = getOperandBitmask(operand, '$bitsAllClear') - return function (value) { - var bitmask = getValueBitmask(value, query.length) - return bitmask && query.every(function (byte, idx) { - return !(bitmask[idx] & byte) - }) - } - } - }, - $bitsAnyClear: { - compileElementSelector: function (operand) { - var query = getOperandBitmask(operand, '$bitsAnyClear') - return function (value) { - var bitmask = getValueBitmask(value, query.length) - return bitmask && query.some(function (byte, idx) { - return ((bitmask[idx] & byte) !== byte) - }) - } - } - }, - $regex: { - compileElementSelector: function (operand, valueSelector) { - if (!(typeof operand === 'string' || operand instanceof RegExp)) - throw Error("$regex has to be a string or RegExp"); - - var regexp; - if (valueSelector.$options !== undefined) { - // Options passed in $options (even the empty string) always overrides - // options in the RegExp object itself. (See also - // Mongo.Collection._rewriteSelector.) - - // Be clear that we only support the JS-supported options, not extended - // ones (eg, Mongo supports x and s). Ideally we would implement x and s - // by transforming the regexp, but not today... - if (/[^gim]/.test(valueSelector.$options)) - throw new Error("Only the i, m, and g regexp options are supported"); - - var regexSource = operand instanceof RegExp ? operand.source : operand; - regexp = new RegExp(regexSource, valueSelector.$options); - } else if (operand instanceof RegExp) { - regexp = operand; - } else { - regexp = new RegExp(operand); - } - return regexpElementMatcher(regexp); - } - }, - $elemMatch: { - dontExpandLeafArrays: true, - compileElementSelector: function (operand, valueSelector, matcher) { - if (!isPlainObject(operand)) - throw Error("$elemMatch need an object"); - - var subMatcher, isDocMatcher; - if (isOperatorObject(Object.keys(operand) - .filter(function (key) { return !Object.keys(LOGICAL_OPERATORS).includes(key); }) - .reduce(function (a, b) { return Object.assign(a, {[b]: operand[b]}); }, {}), true)) { - subMatcher = compileValueSelector(operand, matcher); - isDocMatcher = false; - } else { - // This is NOT the same as compileValueSelector(operand), and not just - // because of the slightly different calling convention. - // {$elemMatch: {x: 3}} means "an element has a field x:3", not - // "consists only of a field x:3". Also, regexps and sub-$ are allowed. - subMatcher = compileDocumentSelector(operand, matcher, - {inElemMatch: true}); - isDocMatcher = true; - } - - return function (value) { - if (!isArray(value)) - return false; - for (var i = 0; i < value.length; ++i) { - var arrayElement = value[i]; - var arg; - if (isDocMatcher) { - // We can only match {$elemMatch: {b: 3}} against objects. - // (We can also match against arrays, if there's numeric indices, - // eg {$elemMatch: {'0.b': 3}} or {$elemMatch: {0: 3}}.) - if (!isPlainObject(arrayElement) && !isArray(arrayElement)) - return false; - arg = arrayElement; - } else { - // dontIterate ensures that {a: {$elemMatch: {$gt: 5}}} matches - // {a: [8]} but not {a: [[8]]} - arg = [{value: arrayElement, dontIterate: true}]; - } - // XXX support $near in $elemMatch by propagating $distance? - if (subMatcher(arg).result) - return i; // specially understood to mean "use as arrayIndices" - } - return false; - }; - } - } -}; - -// makeLookupFunction(key) returns a lookup function. -// -// A lookup function takes in a document and returns an array of matching -// branches. If no arrays are found while looking up the key, this array will -// have exactly one branches (possibly 'undefined', if some segment of the key -// was not found). -// -// If arrays are found in the middle, this can have more than one element, since -// we "branch". When we "branch", if there are more key segments to look up, -// then we only pursue branches that are plain objects (not arrays or scalars). -// This means we can actually end up with no branches! -// -// We do *NOT* branch on arrays that are found at the end (ie, at the last -// dotted member of the key). We just return that array; if you want to -// effectively "branch" over the array's values, post-process the lookup -// function with expandArraysInBranches. -// -// Each branch is an object with keys: -// - value: the value at the branch -// - dontIterate: an optional bool; if true, it means that 'value' is an array -// that expandArraysInBranches should NOT expand. This specifically happens -// when there is a numeric index in the key, and ensures the -// perhaps-surprising MongoDB behavior where {'a.0': 5} does NOT -// match {a: [[5]]}. -// - arrayIndices: if any array indexing was done during lookup (either due to -// explicit numeric indices or implicit branching), this will be an array of -// the array indices used, from outermost to innermost; it is falsey or -// absent if no array index is used. If an explicit numeric index is used, -// the index will be followed in arrayIndices by the string 'x'. -// -// Note: arrayIndices is used for two purposes. First, it is used to -// implement the '$' modifier feature, which only ever looks at its first -// element. -// -// Second, it is used for sort key generation, which needs to be able to tell -// the difference between different paths. Moreover, it needs to -// differentiate between explicit and implicit branching, which is why -// there's the somewhat hacky 'x' entry: this means that explicit and -// implicit array lookups will have different full arrayIndices paths. (That -// code only requires that different paths have different arrayIndices; it -// doesn't actually "parse" arrayIndices. As an alternative, arrayIndices -// could contain objects with flags like "implicit", but I think that only -// makes the code surrounding them more complex.) -// -// (By the way, this field ends up getting passed around a lot without -// cloning, so never mutate any arrayIndices field/var in this package!) -// -// -// At the top level, you may only pass in a plain object or array. -// -// See the test 'minimongo - lookup' for some examples of what lookup functions -// return. -makeLookupFunction = function (key, options) { - options = options || {}; - var parts = key.split('.'); - var firstPart = parts.length ? parts[0] : ''; - var firstPartIsNumeric = isNumericKey(firstPart); - var nextPartIsNumeric = parts.length >= 2 && isNumericKey(parts[1]); - var lookupRest; - if (parts.length > 1) { - lookupRest = makeLookupFunction(parts.slice(1).join('.')); - } - - var omitUnnecessaryFields = function (retVal) { - if (!retVal.dontIterate) - delete retVal.dontIterate; - if (retVal.arrayIndices && !retVal.arrayIndices.length) - delete retVal.arrayIndices; - return retVal; - }; - - // Doc will always be a plain object or an array. - // apply an explicit numeric index, an array. - return function (doc, arrayIndices) { - if (!arrayIndices) - arrayIndices = []; - - if (isArray(doc)) { - // If we're being asked to do an invalid lookup into an array (non-integer - // or out-of-bounds), return no results (which is different from returning - // a single undefined result, in that `null` equality checks won't match). - if (!(firstPartIsNumeric && firstPart < doc.length)) - return []; - - // Remember that we used this array index. Include an 'x' to indicate that - // the previous index came from being considered as an explicit array - // index (not branching). - arrayIndices = arrayIndices.concat(+firstPart, 'x'); - } - - // Do our first lookup. - var firstLevel = doc[firstPart]; - - // If there is no deeper to dig, return what we found. - // - // If what we found is an array, most value selectors will choose to treat - // the elements of the array as matchable values in their own right, but - // that's done outside of the lookup function. (Exceptions to this are $size - // and stuff relating to $elemMatch. eg, {a: {$size: 2}} does not match {a: - // [[1, 2]]}.) - // - // That said, if we just did an *explicit* array lookup (on doc) to find - // firstLevel, and firstLevel is an array too, we do NOT want value - // selectors to iterate over it. eg, {'a.0': 5} does not match {a: [[5]]}. - // So in that case, we mark the return value as "don't iterate". - if (!lookupRest) { - return [omitUnnecessaryFields({ - value: firstLevel, - dontIterate: isArray(doc) && isArray(firstLevel), - arrayIndices: arrayIndices})]; - } - - // We need to dig deeper. But if we can't, because what we've found is not - // an array or plain object, we're done. If we just did a numeric index into - // an array, we return nothing here (this is a change in Mongo 2.5 from - // Mongo 2.4, where {'a.0.b': null} stopped matching {a: [5]}). Otherwise, - // return a single `undefined` (which can, for example, match via equality - // with `null`). - if (!isIndexable(firstLevel)) { - if (isArray(doc)) - return []; - return [omitUnnecessaryFields({value: undefined, - arrayIndices: arrayIndices})]; - } - - var result = []; - var appendToResult = function (more) { - Array.prototype.push.apply(result, more); - }; - - // Dig deeper: look up the rest of the parts on whatever we've found. - // (lookupRest is smart enough to not try to do invalid lookups into - // firstLevel if it's an array.) - appendToResult(lookupRest(firstLevel, arrayIndices)); - - // If we found an array, then in *addition* to potentially treating the next - // part as a literal integer lookup, we should also "branch": try to look up - // the rest of the parts on each array element in parallel. - // - // In this case, we *only* dig deeper into array elements that are plain - // objects. (Recall that we only got this far if we have further to dig.) - // This makes sense: we certainly don't dig deeper into non-indexable - // objects. And it would be weird to dig into an array: it's simpler to have - // a rule that explicit integer indexes only apply to an outer array, not to - // an array you find after a branching search. - // - // In the special case of a numeric part in a *sort selector* (not a query - // selector), we skip the branching: we ONLY allow the numeric part to mean - // "look up this index" in that case, not "also look up this index in all - // the elements of the array". - if (isArray(firstLevel) && !(nextPartIsNumeric && options.forSort)) { - firstLevel.forEach(function (branch, arrayIndex) { - if (isPlainObject(branch)) { - appendToResult(lookupRest( - branch, - arrayIndices.concat(arrayIndex))); - } - }); - } - - return result; - }; -}; -MinimongoTest.makeLookupFunction = makeLookupFunction; - -expandArraysInBranches = function (branches, skipTheArrays) { - var branchesOut = []; - branches.forEach(function (branch) { - var thisIsArray = isArray(branch.value); - // We include the branch itself, *UNLESS* we it's an array that we're going - // to iterate and we're told to skip arrays. (That's right, we include some - // arrays even skipTheArrays is true: these are arrays that were found via - // explicit numerical indices.) - if (!(skipTheArrays && thisIsArray && !branch.dontIterate)) { - branchesOut.push({ - value: branch.value, - arrayIndices: branch.arrayIndices - }); - } - if (thisIsArray && !branch.dontIterate) { - branch.value.forEach(function (leaf, i) { - branchesOut.push({ - value: leaf, - arrayIndices: (branch.arrayIndices || []).concat(i) - }); - }); - } - }); - return branchesOut; -}; - -var nothingMatcher = function (docOrBranchedValues) { - return {result: false}; -}; - -var everythingMatcher = function (docOrBranchedValues) { - return {result: true}; -}; - - -// NB: We are cheating and using this function to implement "AND" for both -// "document matchers" and "branched matchers". They both return result objects -// but the argument is different: for the former it's a whole doc, whereas for -// the latter it's an array of "branched values". -var andSomeMatchers = function (subMatchers) { - if (subMatchers.length === 0) - return everythingMatcher; - if (subMatchers.length === 1) - return subMatchers[0]; - - return function (docOrBranches) { - var ret = {}; - ret.result = subMatchers.every(function (f) { - var subResult = f(docOrBranches); - // Copy a 'distance' number out of the first sub-matcher that has - // one. Yes, this means that if there are multiple $near fields in a - // query, something arbitrary happens; this appears to be consistent with - // Mongo. - if (subResult.result && subResult.distance !== undefined - && ret.distance === undefined) { - ret.distance = subResult.distance; - } - // Similarly, propagate arrayIndices from sub-matchers... but to match - // MongoDB behavior, this time the *last* sub-matcher with arrayIndices - // wins. - if (subResult.result && subResult.arrayIndices) { - ret.arrayIndices = subResult.arrayIndices; - } - return subResult.result; - }); - - // If we didn't actually match, forget any extra metadata we came up with. - if (!ret.result) { - delete ret.distance; - delete ret.arrayIndices; - } - return ret; - }; -}; - -var andDocumentMatchers = andSomeMatchers; -var andBranchedMatchers = andSomeMatchers; - - -// helpers used by compiled selector code -LocalCollection._f = { - // XXX for _all and _in, consider building 'inquery' at compile time.. - - _type: function (v) { - if (typeof v === "number") - return 1; - if (typeof v === "string") - return 2; - if (typeof v === "boolean") - return 8; - if (isArray(v)) - return 4; - if (v === null) - return 10; - if (v instanceof RegExp) - // note that typeof(/x/) === "object" - return 11; - if (typeof v === "function") - return 13; - if (v instanceof Date) - return 9; - if (EJSON.isBinary(v)) - return 5; - if (v instanceof MongoID.ObjectID) - return 7; - return 3; // object - - // XXX support some/all of these: - // 14, symbol - // 15, javascript code with scope - // 16, 18: 32-bit/64-bit integer - // 17, timestamp - // 255, minkey - // 127, maxkey - }, - - // deep equality test: use for literal document and array matches - _equal: function (a, b) { - return EJSON.equals(a, b, {keyOrderSensitive: true}); - }, - - // maps a type code to a value that can be used to sort values of - // different types - _typeorder: function (t) { - // http://www.mongodb.org/display/DOCS/What+is+the+Compare+Order+for+BSON+Types - // XXX what is the correct sort position for Javascript code? - // ('100' in the matrix below) - // XXX minkey/maxkey - return [-1, // (not a type) - 1, // number - 2, // string - 3, // object - 4, // array - 5, // binary - -1, // deprecated - 6, // ObjectID - 7, // bool - 8, // Date - 0, // null - 9, // RegExp - -1, // deprecated - 100, // JS code - 2, // deprecated (symbol) - 100, // JS code - 1, // 32-bit int - 8, // Mongo timestamp - 1 // 64-bit int - ][t]; - }, - - // compare two values of unknown type according to BSON ordering - // semantics. (as an extension, consider 'undefined' to be less than - // any other value.) return negative if a is less, positive if b is - // less, or 0 if equal - _cmp: function (a, b) { - if (a === undefined) - return b === undefined ? 0 : -1; - if (b === undefined) - return 1; - var ta = LocalCollection._f._type(a); - var tb = LocalCollection._f._type(b); - var oa = LocalCollection._f._typeorder(ta); - var ob = LocalCollection._f._typeorder(tb); - if (oa !== ob) - return oa < ob ? -1 : 1; - if (ta !== tb) - // XXX need to implement this if we implement Symbol or integers, or - // Timestamp - throw Error("Missing type coercion logic in _cmp"); - if (ta === 7) { // ObjectID - // Convert to string. - ta = tb = 2; - a = a.toHexString(); - b = b.toHexString(); - } - if (ta === 9) { // Date - // Convert to millis. - ta = tb = 1; - a = a.getTime(); - b = b.getTime(); - } - - if (ta === 1) // double - return a - b; - if (tb === 2) // string - return a < b ? -1 : (a === b ? 0 : 1); - if (ta === 3) { // Object - // this could be much more efficient in the expected case ... - var to_array = function (obj) { - var ret = []; - for (var key in obj) { - ret.push(key); - ret.push(obj[key]); - } - return ret; - }; - return LocalCollection._f._cmp(to_array(a), to_array(b)); - } - if (ta === 4) { // Array - for (var i = 0; ; i++) { - if (i === a.length) - return (i === b.length) ? 0 : -1; - if (i === b.length) - return 1; - var s = LocalCollection._f._cmp(a[i], b[i]); - if (s !== 0) - return s; - } - } - if (ta === 5) { // binary - // Surprisingly, a small binary blob is always less than a large one in - // Mongo. - if (a.length !== b.length) - return a.length - b.length; - for (i = 0; i < a.length; i++) { - if (a[i] < b[i]) - return -1; - if (a[i] > b[i]) - return 1; - } - return 0; - } - if (ta === 8) { // boolean - if (a) return b ? 0 : 1; - return b ? -1 : 0; - } - if (ta === 10) // null - return 0; - if (ta === 11) // regexp - throw Error("Sorting not supported on regular expression"); // XXX - // 13: javascript code - // 14: symbol - // 15: javascript code with scope - // 16: 32-bit integer - // 17: timestamp - // 18: 64-bit integer - // 255: minkey - // 127: maxkey - if (ta === 13) // javascript code - throw Error("Sorting not supported on Javascript code"); // XXX - throw Error("Unknown type to sort"); - } -}; - -const objectOnlyHasDollarKeys = (object) => { +function objectOnlyHasDollarKeys (object) { const keys = Object.keys(object); return keys.length > 0 && keys.every(key => key.charAt(0) === '$'); }; @@ -2947,7 +1660,7 @@ Object.assign(Minimongo.Sorter.prototype, { // (functions (a,b)->(negative or positive or zero)), returns a single // comparator which uses each comparator in order and returns the first // non-zero value. -var composeComparators = function (comparatorArray) { +function composeComparators (comparatorArray) { return function (a, b) { for (var i = 0; i < comparatorArray.length; ++i) { var compare = comparatorArray[i](a, b); @@ -2956,7 +1669,8 @@ var composeComparators = function (comparatorArray) { } return 0; }; -}; +} + // Knows how to compile a fields projection to a predicate function. // @returns - Function: a closure that filters out an object according to the // fields projection rules: @@ -3012,7 +1726,7 @@ LocalCollection._compileProjection = function (fields) { // - tree - Object - tree representation of keys involved in projection // (exception for '_id' as it is a special case handled separately) // - including - Boolean - "take only certain fields" type of projection -projectionDetails = function (fields) { +function projectionDetails (fields) { // Find the non-_id keys (_id is handled specially because it is included unless // explicitly excluded). Sort the keys, so that our code to detect overlaps // like 'foo' and 'foo.bar' can assume that 'foo' comes first. @@ -3073,7 +1787,7 @@ projectionDetails = function (fields) { tree: projectionRulesTree, including: including }; -}; +} // paths - Array: list of mongo style paths // newLeafFn - Function: of form function(path) should return a scalar value to @@ -3084,7 +1798,7 @@ projectionDetails = function (fields) { // conflict resolution. // initial tree - Optional Object: starting tree. // @returns - Object: tree represented as a set of nested objects -pathsToTree = function (paths, newLeafFn, conflictFn, tree) { +function pathsToTree (paths, newLeafFn, conflictFn, tree) { tree = tree || {}; paths.forEach(function (keyPath) { var treePos = tree; @@ -3117,7 +1831,7 @@ pathsToTree = function (paths, newLeafFn, conflictFn, tree) { }); return tree; -}; +} LocalCollection._checkSupportedProjection = function (fields) { if (fields !== Object(fields) || Array.isArray(fields)) @@ -3148,7 +1862,7 @@ LocalCollection._checkSupportedProjection = function (fields) { // out when to set the fields in $setOnInsert, if present. LocalCollection._modify = function (doc, mod, options) { options = options || {}; - if (!isPlainObject(mod)) + if (!LocalCollection._isPlainObject(mod)) throw MinimongoError("Modifier must be an object"); // Make sure the caller can't mutate our data structures. @@ -3238,7 +1952,7 @@ LocalCollection._modify = function (doc, mod, options) { // // if options.arrayIndices is set, use its first element for the (first) '$' in // the path. -var findModTarget = function (doc, keyparts, options) { +function findModTarget (doc, keyparts, options) { options = options || {}; var usedArrayIndex = false; for (var i = 0; i < keyparts.length; i++) { @@ -3304,9 +2018,9 @@ var findModTarget = function (doc, keyparts, options) { } // notreached -}; +} -var NO_CREATE_MODIFIERS = { +const NO_CREATE_MODIFIERS = { $unset: true, $pop: true, $rename: true, @@ -3314,7 +2028,7 @@ var NO_CREATE_MODIFIERS = { $pullAll: true }; -var MODIFIERS = { +const MODIFIERS = { $currentDate: function (target, field, arg) { if (typeof arg === "object" && arg.hasOwnProperty("$type")) { if (arg.$type !== "date") { @@ -3634,6 +2348,7 @@ var MODIFIERS = { throw MinimongoError("$bit is not supported", { field }); } }; + // ordered: bool. // old_results and new_results: collections of documents. // if ordered, they are arrays. @@ -3647,8 +2362,7 @@ LocalCollection._diffQueryUnorderedChanges = function (oldResults, newResults, o }; -LocalCollection._diffQueryOrderedChanges = - function (oldResults, newResults, observer, options) { +LocalCollection._diffQueryOrderedChanges = function (oldResults, newResults, observer, options) { return DiffSequence.diffQueryOrderedChanges(oldResults, newResults, observer, options); }; diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js index 012714325a..9d784276c9 100644 --- a/packages/minimongo/package.js +++ b/packages/minimongo/package.js @@ -24,6 +24,9 @@ Package.onUse(api => { api.addFiles('minimongo.js'); api.addFiles('minimongo_server.js', 'server'); + + // api.mainModule('client_main.js', 'client'); + // api.mainModule('server_main.js', 'server'); }); Package.onTest(api => {