From 51b3cef688571c2e0c8b6d41acfaa690d38d8368 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 6 Dec 2013 22:41:46 -0800 Subject: [PATCH 001/124] Whitespace adjustments to match style guide. --- packages/appcache/appcache-client.js | 5 ++--- packages/appcache/appcache-server.js | 3 +-- packages/livedata/livedata_server_tests.js | 1 - packages/liverange/liverange.js | 3 +-- packages/mongo-livedata/collection.js | 3 +-- packages/spark/spark.js | 3 +-- packages/test-helpers/connection.js | 6 ++---- 7 files changed, 8 insertions(+), 16 deletions(-) diff --git a/packages/appcache/appcache-client.js b/packages/appcache/appcache-client.js index 024b71d01f..1b7297d3aa 100644 --- a/packages/appcache/appcache-client.js +++ b/packages/appcache/appcache-client.js @@ -58,11 +58,10 @@ window.applicationCache.addEventListener('noupdate', cacheIsNowUpToDate, false); window.applicationCache.addEventListener('obsolete', (function() { if (reloadRetry) { cacheIsNowUpToDate(); - } - else { + } else { appcacheUpdated = true; Reload._reload(); } }), false); -} // if window.applicationCache \ No newline at end of file +} // if window.applicationCache diff --git a/packages/appcache/appcache-server.js b/packages/appcache/appcache-server.js index 833acc6880..b389b1e650 100644 --- a/packages/appcache/appcache-server.js +++ b/packages/appcache/appcache-server.js @@ -44,8 +44,7 @@ Meteor.AppCache = { _.each(value, function (urlPrefix) { RoutePolicy.declare(urlPrefix, 'static-online'); }); - } - else { + } else { throw new Error('Invalid AppCache config option: ' + option); } }); diff --git a/packages/livedata/livedata_server_tests.js b/packages/livedata/livedata_server_tests.js index 4fe5dec901..acf6351e5b 100644 --- a/packages/livedata/livedata_server_tests.js +++ b/packages/livedata/livedata_server_tests.js @@ -24,7 +24,6 @@ Tinytest.addAsync( } ); - Tinytest.addAsync( "livedata server - connectionHandle.close()", function (test, onComplete) { diff --git a/packages/liverange/liverange.js b/packages/liverange/liverange.js index cbedb0194f..42def4b4b5 100644 --- a/packages/liverange/liverange.js +++ b/packages/liverange/liverange.js @@ -331,8 +331,7 @@ LiveRange.prototype.visit = function(visitRange, visitNode) { recurse(rangeStart, rangeEnd, startIndex+1); visitRange(false, range); n = rangeEnd; - } - else { + } else { // bare node if (visitNode(true, n) !== false && n.firstChild) recurse(n.firstChild, n.lastChild); diff --git a/packages/mongo-livedata/collection.js b/packages/mongo-livedata/collection.js index ea68480470..b9617a1a6a 100644 --- a/packages/mongo-livedata/collection.js +++ b/packages/mongo-livedata/collection.js @@ -278,8 +278,7 @@ Meteor.Collection._rewriteSelector = function (selector) { ret[key] = _.map(value, function (v) { return Meteor.Collection._rewriteSelector(v); }); - } - else { + } else { ret[key] = value; } }); diff --git a/packages/spark/spark.js b/packages/spark/spark.js index 2055cc7a67..13dfc7f5ff 100644 --- a/packages/spark/spark.js +++ b/packages/spark/spark.js @@ -1055,8 +1055,7 @@ Spark.list = function (cursor, itemFunc, elseFunc) { var frag = itemDict.get(id).liveRange.extract(); if (before === null) { itemDict.lastValue().liveRange.insertAfter(frag); - } - else { + } else { itemDict.get(before).liveRange.insertBefore(frag); } itemDict.moveBefore(id, before); diff --git a/packages/test-helpers/connection.js b/packages/test-helpers/connection.js index 4da2e73f72..a78c3dab46 100644 --- a/packages/test-helpers/connection.js +++ b/packages/test-helpers/connection.js @@ -17,8 +17,7 @@ makeTestConnection = function (test, succeeded, failed) { if (serverConns[serverConn.id]) { test.fail("onConnection callback called multiple times for same session id"); failed(); - } - else { + } else { serverConns[serverConn.id] = serverConn; } }); @@ -30,8 +29,7 @@ makeTestConnection = function (test, succeeded, failed) { if (! serverConn) { test.fail("No onConnection received server side for connected client"); failed(); - } - else { + } else { onConnectionHandle.stop(); succeeded(clientConn, serverConn); } From ead40e18d304d6e10c2f5a18d3471523fefd2bef Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 6 Dec 2013 22:42:36 -0800 Subject: [PATCH 002/124] Hash login tokens before storing them in the database. --- packages/accounts-base/accounts_server.js | 176 +++++++++++++++--- packages/accounts-base/accounts_tests.js | 76 +++++++- packages/accounts-password/password_server.js | 24 ++- packages/accounts-password/password_tests.js | 21 ++- 4 files changed, 261 insertions(+), 36 deletions(-) diff --git a/packages/accounts-base/accounts_server.js b/packages/accounts-base/accounts_server.js index 1dde7a7788..c3392b391c 100644 --- a/packages/accounts-base/accounts_server.js +++ b/packages/accounts-base/accounts_server.js @@ -1,3 +1,5 @@ +var crypto = Npm.require('crypto'); + /// /// CURRENT USER /// @@ -78,14 +80,18 @@ Meteor.methods({ var result = tryAllLoginHandlers(options); if (result !== null) { this.setUserId(result.id); - Accounts._setLoginToken(this.connection.id, result.token); + Accounts._setLoginToken( + result.id, + this.connection, + Accounts._hashLoginToken(result.token) + ); } return result; }, logout: function() { var token = Accounts._getLoginToken(this.connection.id); - Accounts._setLoginToken(this.connection.id, null); + Accounts._setLoginToken(this.userId, this.connection, null); if (token && this.userId) removeLoginToken(this.userId, token); this.setUserId(null); @@ -118,7 +124,7 @@ Meteor.methods({ "services.resume.loginTokensToDelete": tokens, "services.resume.haveLoginTokensToDelete": true }, - $push: { "services.resume.loginTokens": newToken } + $push: { "services.resume.loginTokens": Accounts._hashStampedToken(newToken) } }); Meteor.setTimeout(function () { // The observe on Meteor.users will take care of closing the connections @@ -168,7 +174,10 @@ Accounts._setAccountData = function (connectionId, field, value) { Meteor.server.onConnection(function (connection) { accountData[connection.id] = {connection: connection}; connection.onClose(function () { - removeConnectionFromToken(connection.id); + removeConnectionFromToken( + Accounts._getLoginToken(connection.id), + connection.id + ); delete accountData[connection.id]; }); }); @@ -179,7 +188,23 @@ Meteor.server.onConnection(function (connection) { /// /// support reconnecting using a meteor login token -// token -> list of connection ids +Accounts._hashLoginToken = function (loginToken) { + var hash = crypto.createHash('sha256'); + hash.update(loginToken); + return hash.digest('base64'); +}; + + +// {token, when} => {hashedToken, when} +Accounts._hashStampedToken = function (stampedToken) { + return _.extend( + _.omit(stampedToken, 'token'), + {hashedToken: Accounts._hashLoginToken(stampedToken.token)} + ); +}; + + +// hashed token -> list of connection ids var connectionsByLoginToken = {}; // test hook @@ -188,7 +213,7 @@ Accounts._getTokenConnections = function (token) { }; // Remove the connection from the list of open connections for the token. -var removeConnectionFromToken = function (connectionId) { +var removeConnectionFromToken = function (token, connectionId) { var token = Accounts._getLoginToken(connectionId); if (token) { connectionsByLoginToken[token] = _.without( @@ -204,15 +229,41 @@ Accounts._getLoginToken = function (connectionId) { return Accounts._getAccountData(connectionId, 'loginToken'); }; -Accounts._setLoginToken = function (connectionId, newToken) { - removeConnectionFromToken(connectionId); +// newToken is a hashed token. +Accounts._setLoginToken = function (userId, connection, newToken) { + removeConnectionFromToken( + Accounts._getLoginToken(connection.id), + connection.id + ); - Accounts._setAccountData(connectionId, 'loginToken', newToken); + Accounts._setAccountData(connection.id, 'loginToken', newToken); if (newToken) { if (! _.has(connectionsByLoginToken, newToken)) connectionsByLoginToken[newToken] = []; - connectionsByLoginToken[newToken].push(connectionId); + connectionsByLoginToken[newToken].push(connection.id); + + // Now that we've added the connection to the + // connectionsByLoginToken map for the token, the connection will + // be closed if the token is removed from the database. However + // at this point the token might have already been deleted, which + // wouldn't have closed the connection because it wasn't in the + // map yet. + // + // We also did need to first add the connection to the map above + // (and now remove it here if the token was deleted), because we + // could be getting a response from the database that the token + // still exists, but then it could be deleted in another fiber + // before our `findOne` call returns... and then that other fiber + // would need for the connection to be in the map for it to close + // the connection. + if (! Meteor.users.findOne({ + _id: userId, + "services.resume.loginTokens.hashedToken": newToken + })) { + removeConnectionFromToken(newToken, connection.id); + connection.close(); + } } }; @@ -240,23 +291,84 @@ Accounts.registerLoginHandler(function(options) { return undefined; check(options.resume, String); - var user = Meteor.users.findOne({ - "services.resume.loginTokens.token": ""+options.resume - }); - if (!user) { + var hashedToken = Accounts._hashLoginToken(options.resume); + + // First look for just the new-style hashed login token, to avoid + // sending the unhashed token to the database in a query if we don't + // need to. + var user = Meteor.users.findOne( + {"services.resume.loginTokens.hashedToken": hashedToken}); + + if (! user) { + // If we didn't find the hashed login token, try also looking for + // the old-style unhashed token. But we need to look for either + // the old-style token OR the new-style token, because another + // client connection logging in simultaneously might have already + // converted the token. + user = Meteor.users.findOne({ + $or: [ + {"services.resume.loginTokens.hashedToken": hashedToken}, + {"services.resume.loginTokens.token": options.resume} + ] + }); + } + + if (! user) { throw new Meteor.Error(403, "You've been logged out by the server. " + "Please login again."); } + // Find the token, which will either be an object with fields + // {hashedToken, when} for a hashed token or {token, when} for an + // unhashed token. + var oldUnhashedStyleToken; var token = _.find(user.services.resume.loginTokens, function (token) { - return token.token === options.resume; + return token.hashedToken === hashedToken; }); + if (token) { + oldUnhashedStyleToken = false; + } else { + token = _.find(user.services.resume.loginTokens, function (token) { + return token.token === options.resume; + }); + oldUnhashedStyleToken = true; + } var tokenExpires = Accounts._tokenExpiration(token.when); if (new Date() >= tokenExpires) throw new Meteor.Error(403, "Your session has expired. Please login again."); + // Update to a hashed token when an unhashed token is encountered. + if (oldUnhashedStyleToken) { + // Only add the new hashed token if the old unhashed token still + // exists (this avoids resurrecting the token if it was deleted + // after we read it). Using $addToSet avoids getting an index + // error if another client logging in simultaneously has already + // inserted the new hashed token. + Meteor.users.update( + { + _id: user._id, + "services.resume.loginTokens.token": options.resume + }, + {$addToSet: { + "services.resume.loginTokens": { + "hashedToken": hashedToken, + "when": token.when + } + }} + ); + + // Remove the old token *after* adding the new, since otherwise + // another client trying to login between our removing the old and + // adding the new wouldn't find a token to login with. + Meteor.users.update(user._id, { + $pull: { + "services.resume.loginTokens": { "token": options.resume } + }, + }); + } + return { token: options.resume, tokenExpires: tokenExpires, @@ -270,12 +382,23 @@ Accounts._generateStampedLoginToken = function () { return {token: Random.id(), when: (new Date)}; }; -// Deletes the given loginToken from the database. This will cause all -// connections associated with the token to be closed. +// Deletes the given loginToken from the database. +// +// For new-style hashed token, this will cause all connections +// associated with the token to be closed. +// +// Any connections associated with old-style unhashed tokens will be +// in the process of becoming associated with hashed tokens and then +// they'll get closed. var removeLoginToken = function (userId, loginToken) { Meteor.users.update(userId, { $pull: { - "services.resume.loginTokens": { "token": loginToken } + "services.resume.loginTokens": { + $or: [ + {hashedToken: loginToken }, + {token: loginToken} + ] + } } }); }; @@ -380,11 +503,12 @@ Accounts.insertUserDoc = function (options, user) { var stampedToken = Accounts._generateStampedLoginToken(); result.token = stampedToken.token; result.tokenExpires = Accounts._tokenExpiration(stampedToken.when); + var token = Accounts._hashStampedToken(stampedToken); Meteor._ensure(user, 'services', 'resume'); if (_.has(user.services.resume, 'loginTokens')) - user.services.resume.loginTokens.push(stampedToken); + user.services.resume.loginTokens.push(token); else - user.services.resume.loginTokens = [stampedToken]; + user.services.resume.loginTokens = [token]; } var fullUser; @@ -538,7 +662,7 @@ Accounts.updateOrCreateUserFromExternalService = function( Meteor.users.update( user._id, {$set: setAttrs, - $push: {'services.resume.loginTokens': stampedToken}}); + $push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedToken)}}); return { token: stampedToken.token, id: user._id, @@ -687,6 +811,8 @@ Meteor.users.allow({ /// DEFAULT INDEXES ON USERS Meteor.users._ensureIndex('username', {unique: 1, sparse: 1}); Meteor.users._ensureIndex('emails.address', {unique: 1, sparse: 1}); +Meteor.users._ensureIndex('services.resume.loginTokens.hashedToken', + {unique: 1, sparse: 1}); Meteor.users._ensureIndex('services.resume.loginTokens.token', {unique: 1, sparse: 1}); // For taking care of logoutOtherClients calls that crashed before the tokens @@ -735,8 +861,14 @@ Meteor.startup(function () { /// LOGGING OUT DELETED USERS /// +// When login tokens are removed from the database, close any sessions +// logged in with those tokens. +// +// Because we upgrade unhashed login tokens to hashed tokens at login +// time, sessions will only be logged in with a hashed token. Thus we +// only need to pull out hashed tokens here. var closeTokensForUser = function (userTokens) { - closeConnectionsForTokens(_.pluck(userTokens, "token")); + closeConnectionsForTokens(_.compact(_.pluck(userTokens, "hashedToken"))); }; // Like _.difference, but uses EJSON.equals to compute which values to return. diff --git a/packages/accounts-base/accounts_tests.js b/packages/accounts-base/accounts_tests.js index dcd9cd28ec..8af7577e4b 100644 --- a/packages/accounts-base/accounts_tests.js +++ b/packages/accounts-base/accounts_tests.js @@ -189,10 +189,10 @@ Tinytest.addAsync('accounts - expire numeric token', function (test, onComplete) Meteor.users.update(result.id, { $set: { "services.resume.loginTokens": [{ - token: Random.id(), + hashedToken: Random.id(), when: date }, { - token: Random.id(), + hashedToken: Random.id(), when: +date }] } @@ -210,6 +210,78 @@ Tinytest.addAsync('accounts - expire numeric token', function (test, onComplete) }); +// Login tokens used to be stored unhashed in the database. We want +// to make sure users can still login after upgrading. + +var userWithUnhashedLoginToken = function () { + var result = Accounts.insertUserDoc({}, {username: Random.id()}); + + // Construct an old-style unhashed login token. + var stampedToken = Accounts._generateStampedLoginToken(); + + Meteor.users.update( + result.id, + {$push: {'services.resume.loginTokens': stampedToken}} + ); + + // Return result, as if from Accounts.insertUserDoc. + result.token = stampedToken.token; + result.tokenExpires = Accounts._tokenExpiration(stampedToken.when); + return result; +}; + +Tinytest.addAsync('accounts - login token', function (test, onComplete) { + // Test that we can login when the database contains a leftover + // old style unhashed login token. + var user1 = userWithUnhashedLoginToken(); + var connection = DDP.connect(Meteor.absoluteUrl()); + connection.call('login', {resume: user1.token}); + connection.disconnect(); + + // Steal the unhashed token from the database and use it to login. + // This is a sanity check so that when we *can't* login with a + // stolen *hashed* token, we know it's not a problem with the test. + var user2 = userWithUnhashedLoginToken(); + var stolenToken = Meteor.users.findOne(user2.id).services.resume.loginTokens[0].token; + test.isTrue(stolenToken); + connection = DDP.connect(Meteor.absoluteUrl()); + connection.call('login', {resume: stolenToken}); + connection.disconnect(); + + // Now do the same thing, this time with a stolen hashed token. + var user3 = Accounts.insertUserDoc({generateLoginToken: true}, {username: Random.id()}); + stolenToken = Meteor.users.findOne(user3.id).services.resume.loginTokens[0].hashedToken; + test.isTrue(stolenToken); + connection = DDP.connect(Meteor.absoluteUrl()); + // evil plan foiled + test.throws( + function () { + connection.call('login', {resume: stolenToken}); + }, + /You\'ve been logged out by the server/ + ); + connection.disconnect(); + + // Old style unhashed tokens are replaced by hashed tokens when + // encountered. This means that after someone logins once, the + // old unhashed token is no longer available to be stolen. + var user4 = userWithUnhashedLoginToken(); + connection = DDP.connect(Meteor.absoluteUrl()); + connection.call('login', {resume: user4.token}); + connection.disconnect(); + + // The token is no longer available to be stolen. + stolenToken = Meteor.users.findOne(user4.id).services.resume.loginTokens[0].token; + test.isFalse(stolenToken); + + // After the upgrade, the client can still login with their login token. + connection = DDP.connect(Meteor.absoluteUrl()); + connection.call('login', {resume: user4.token}); + connection.disconnect(); + + onComplete(); +}); + Tinytest.addAsync( 'accounts - connection data cleaned up', function (test, onComplete) { diff --git a/packages/accounts-password/password_server.js b/packages/accounts-password/password_server.js index f65ae8798f..765053f1bb 100644 --- a/packages/accounts-password/password_server.js +++ b/packages/accounts-password/password_server.js @@ -95,7 +95,7 @@ Accounts.registerLoginHandler(function (options) { throw new Meteor.Error(403, "User not found"); var stampedLoginToken = Accounts._generateStampedLoginToken(); Meteor.users.update( - userId, {$push: {'services.resume.loginTokens': stampedLoginToken}}); + userId, {$push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedLoginToken)}}); return { token: stampedLoginToken.token, @@ -139,7 +139,7 @@ Accounts.registerLoginHandler(function (options) { var stampedLoginToken = Accounts._generateStampedLoginToken(); Meteor.users.update( - user._id, {$push: {'services.resume.loginTokens': stampedLoginToken}}); + user._id, {$push: {'services.resume.loginTokens': Accounts._hashStampedToken(stampedLoginToken)}}); return { token: stampedLoginToken.token, @@ -314,13 +314,14 @@ Meteor.methods({resetPassword: function (token, newVerifier) { throw new Meteor.Error(403, "Token has invalid email address"); var stampedLoginToken = Accounts._generateStampedLoginToken(); + var newHashedToken = Accounts._hashStampedToken(stampedLoginToken); // NOTE: We're about to invalidate tokens on the user, who we might be // logged in as. Make sure to avoid logging ourselves out if this // happens. But also make sure not to leave the connection in a state // of having a bad token set if things fail. var oldToken = Accounts._getLoginToken(this.connection.id); - Accounts._setLoginToken(this.connection.id, null); + Accounts._setLoginToken(user._id, this.connection, null); try { // Update the user record by: @@ -331,17 +332,17 @@ Meteor.methods({resetPassword: function (token, newVerifier) { // - Verifying their email, since they got the password reset via email. Meteor.users.update({_id: user._id, 'emails.address': email}, { $set: {'services.password.srp': newVerifier, - 'services.resume.loginTokens': [stampedLoginToken], + 'services.resume.loginTokens': [newHashedToken], 'emails.$.verified': true}, $unset: {'services.password.reset': 1} }); } catch (err) { // update failed somehow. reset to old token. - Accounts._setLoginToken(this.connection.id, oldToken); + Accounts._setLoginToken(user._id, this.connection, oldToken); throw err; } - Accounts._setLoginToken(this.connection.id, stampedLoginToken.token); + Accounts._setLoginToken(user._id, this.connection, newHashedToken.hashedToken); this.setUserId(user._id); return { @@ -421,6 +422,7 @@ Meteor.methods({verifyEmail: function (token) { // Log the user in with a new login token. var stampedLoginToken = Accounts._generateStampedLoginToken(); + var hashedToken = Accounts._hashStampedToken(stampedLoginToken); // By including the address in the query, we can use 'emails.$' in the // modifier to get a reference to the specific object in the emails @@ -432,10 +434,10 @@ Meteor.methods({verifyEmail: function (token) { 'emails.address': tokenRecord.address}, {$set: {'emails.$.verified': true}, $pull: {'services.email.verificationTokens': {token: token}}, - $push: {'services.resume.loginTokens': stampedLoginToken}}); + $push: {'services.resume.loginTokens': hashedToken}}); this.setUserId(user._id); - Accounts._setLoginToken(this.connection.id, stampedLoginToken.token); + Accounts._setLoginToken(user._id, this.connection, hashedToken.hashedToken); return { token: stampedLoginToken.token, tokenExpires: Accounts._tokenExpiration(stampedLoginToken.when), @@ -514,7 +516,11 @@ Meteor.methods({createUser: function (options) { // client gets logged in as the new user afterwards. this.setUserId(result.id); - Accounts._setLoginToken(this.connection.id, result.token); + Accounts._setLoginToken( + result.id, + this.connection, + Accounts._hashLoginToken(result.token) + ); return result; }}); diff --git a/packages/accounts-password/password_tests.js b/packages/accounts-password/password_tests.js index 1da7dff9a4..4c9243525b 100644 --- a/packages/accounts-password/password_tests.js +++ b/packages/accounts-password/password_tests.js @@ -344,7 +344,23 @@ if (Meteor.isClient) (function () { loggedInAs(this.username, test, expect)); }, - function(test, expect) { + function (test, expect) { + // we can't login with an invalid token + var expectLoginError = expect(function (err) { + test.isTrue(err); + }); + Meteor.loginWithToken('invalid', expectLoginError); + }, + + function (test, expect) { + // we can login with a valid token + var expectLoginOK = expect(function (err) { + test.isFalse(err); + }); + Meteor.loginWithToken(Accounts._storedLoginToken(), expectLoginOK); + }, + + function (test, expect) { // test logging out invalidates our token var expectLoginError = expect(function (err) { test.isTrue(err); @@ -356,7 +372,7 @@ if (Meteor.isClient) (function () { }); }, - function(test, expect) { + function (test, expect) { var self = this; // Test that login tokens get expired. We should get logged out when a // token expires, and not be able to log in again with the same token. @@ -568,7 +584,6 @@ if (Meteor.isServer) (function () { test.isTrue(result); var token = Accounts._getAccountData(serverConn.id, 'loginToken'); test.isTrue(token); - test.equal(result.token, token); test.isTrue(_.contains( Accounts._getTokenConnections(token), serverConn.id)); clientConn.disconnect(); From 88d58a0efe9eda8b0df510f6abc500d0e91b8794 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 6 Dec 2013 22:43:05 -0800 Subject: [PATCH 003/124] History entry. --- History.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/History.md b/History.md index 6df859ccae..728bbbce80 100644 --- a/History.md +++ b/History.md @@ -28,6 +28,8 @@ * New 'facts' package publishes internal statistics about Meteor. +* Hash login tokens before storing them in the database. + * Upgraded dependencies: * SockJS server from 0.3.7 to 0.3.8 From 3f12f77a3f6a0bc1309ea30124a36567ddd42bde Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:13:28 -0800 Subject: [PATCH 004/124] Test that localStorage works before uses it Fixes accounts-password login with private-browsing Safari, and probably some IE configurations too. Fixes #1291 (I hope, not tested on IE). Similar to #1688. This still means that multi-tab won't work with private browsing, unfortunately. --- packages/localstorage/localstorage.js | 47 +++++++++++++++++++-------- packages/localstorage/package.js | 1 + 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/packages/localstorage/localstorage.js b/packages/localstorage/localstorage.js index 4dd1e49ba2..ccaefcca08 100644 --- a/packages/localstorage/localstorage.js +++ b/packages/localstorage/localstorage.js @@ -1,20 +1,37 @@ -// This is not an ideal name, but we can change it later. +// Meteor._localStorage is not an ideal name, but we can change it later. if (window.localStorage) { - Meteor._localStorage = { - getItem: function (key) { - return window.localStorage.getItem(key); - }, - setItem: function (key, value) { - window.localStorage.setItem(key, value); - }, - removeItem: function (key) { - window.localStorage.removeItem(key); - } - }; + // Let's test to make sure that localStorage actually works. For example, in + // Safari with private browsing on, window.localStorage exists but actually + // trying to use it throws. + + var key = '_localstorage_test_' + Random.id(); + var retrieved; + try { + window.localStorage.setItem(key, key); + retrieved = window.localStorage.getItem(key); + window.localStorage.removeItem(key); + } catch (e) { + // ... ignore + } + if (key === retrieved) { + Meteor._localStorage = { + getItem: function (key) { + return window.localStorage.getItem(key); + }, + setItem: function (key, value) { + window.localStorage.setItem(key, value); + }, + removeItem: function (key) { + window.localStorage.removeItem(key); + } + }; + } } + // XXX eliminate dependency on jQuery, detect browsers ourselves -else if ($.browser.msie) { // If we are on IE, which support userData +// Else, if we are on IE, which support userData +if (!Meteor._localStorage && $.browser.msie) { var userdata = document.createElement('span'); // could be anything userdata.style.behavior = 'url("#default#userData")'; userdata.id = 'localstorage-helper'; @@ -40,7 +57,9 @@ else if ($.browser.msie) { // If we are on IE, which support userData return userdata.getAttribute(key); } }; -} else { +} + +if (!Meteor._localStorage) { Meteor._debug( "You are running a browser with no localStorage or userData " + "support. Logging in from one tab will not cause another " diff --git a/packages/localstorage/package.js b/packages/localstorage/package.js index c6570323cb..434872dc84 100644 --- a/packages/localstorage/package.js +++ b/packages/localstorage/package.js @@ -5,6 +5,7 @@ Package.describe({ Package.on_use(function (api) { api.use('jquery', 'client'); // XXX only used for browser detection. remove. + api.use('random', 'client'); api.add_files('localstorage.js', 'client'); }); From 7424bb63cd741fb6a75cbb0885a394bc5194fad5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:25:02 -0800 Subject: [PATCH 005/124] mongo_runner: Don't send rs.initiate too early Fixes #1696. Thanks to @Maxpain177 for reporting and providing access to a machine where this was easily reproducible. --- tools/mongo_runner.js | 67 ++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 40d0a39bf8..09a3efbacf 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -234,6 +234,8 @@ exports.launchMongo = function (options) { proc.stdout.setEncoding('utf8'); var listening = false; var replSetReady = false; + var replSetReadyToBeInitiated = false; + var alreadyInitiatedReplSet = false; var maybeCallOnListen = function () { if (listening && replSetReady) { if (createReplSet) @@ -241,29 +243,54 @@ exports.launchMongo = function (options) { onListen(); } }; + + var maybeInitiateReplset = function () { + // We need to want to create a replset, be confident that the server is + // listening, be confident that the server's replset implementation is + // ready to be initiated, and have not already done it. + if (!(createReplSet && listening && replSetReadyToBeInitiated + && !alreadyInitiatedReplSet)) { + return; + } + + alreadyInitiatedReplSet = true; + + // Connect to it and start a replset. + var db = new mongoNpmModule.Db( + 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), + {safe: true}); + db.open(function(err, db) { + if (err) + throw err; + db.admin().command({ + replSetInitiate: { + _id: replSetName, + members: [{_id : 0, host: '127.0.0.1:' + options.port}] + } + }, function (err, result) { + if (err) + throw err; + // why this isn't in the error is unclear. + if (result && result.documents && result.documents[0] + && result.documents[0].errmsg) { + throw result.document[0].errmsg; + } + db.close(true); + }); + }); + }; + proc.stdout.on('data', function (data) { + // note: don't use "else ifs" in this, because 'data' can have multiple + // lines + if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) { + replSetReadyToBeInitiated = true; + maybeInitiateReplset(); + } + if (/ \[initandlisten\] waiting for connections on port/.test(data)) { - if (createReplSet) { - // Connect to it and start a replset. - var db = new mongoNpmModule.Db( - 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), - {safe: true}); - db.open(function(err, db) { - if (err) - throw err; - db.admin().command({ - replSetInitiate: { - _id: replSetName, - members: [{_id : 0, host: '127.0.0.1:' + options.port}] - } - }, function (err, result) { - if (err) - throw err; - db.close(true); - }); - }); - } listening = true; + maybeInitiateReplset(); maybeCallOnListen(); } From 4294d40220d506a55a4fc404ffa1be732b33064f Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Thu, 19 Dec 2013 18:14:22 -0800 Subject: [PATCH 006/124] update docs app to release sso-1 --- docs/.meteor/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index faef31a435..400877328a 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.7.0 +sso-1 From c0626667ec9439f571347ac2287ae70dccf8d474 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:19:15 -0800 Subject: [PATCH 007/124] Don't leave invalid METEOR-PORT files around Could cause mongo startup to hang. Reproduction: $ meteor => Meteor server running on: http://localhost:3000/ # ... wait for server to start, ctrl-c. # leaves '3002' in .meteor/local/db/METEOR-PORT $ meteor -p 5000 # ctrl-c in about a second: once we've wiped the old local db # but before we've configured the new one. # before this commit, '3002' is still in the METEOR-PORT file. $ meteor # before this commit, hangs with: Initializing mongo database... this may take a moment. --- tools/mongo_runner.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 09a3efbacf..b7d0824c99 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -161,9 +161,11 @@ exports.launchMongo = function (options) { } var portFile = path.join(dbPath, 'METEOR-PORT'); + var portFileExists = false; var createReplSet = true; try { createReplSet = +(fs.readFileSync(portFile)) !== options.port; + portFileExists = true; } catch (e) { if (!e || e.code !== 'ENOENT') throw e; @@ -176,6 +178,11 @@ exports.launchMongo = function (options) { // replSet configuration. It's also a little slow to initiate a new replSet, // thus the attempt to not do it unless the port changes.) if (createReplSet) { + // Delete the port file, so we don't mistakenly believe that the DB is + // still configured. + if (portFileExists) + fs.unlinkSync(portFile); + try { var dbFiles = fs.readdirSync(dbPath); } catch (e) { From d84fe821196185e87fa0f87d3d5d73078d705c70 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:54:27 -0800 Subject: [PATCH 008/124] Add some comments to selector compiler --- packages/minimongo/selector.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e077b9363c..44ad5d5a62 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -4,12 +4,25 @@ var isArray = function (x) { return _.isArray(x) && !EJSON.isBinary(x); }; +// If x is an array, true if f(e) is true for some e in x +// (but never try f(x) directly) +// Otherwise, true if f(x) is true. +// +// Use this in cases where f(Array) should never be true... +// for example, equality comparisons to non-arrays, +// ordering comparisons (which should always be false if either side +// is an array), regexps (need string), mod (needs number)... +// XXX ensure comparisons are always false if LHS is an array +// XXX ensure comparisons among different types are false var _anyIfArray = function (x, f) { if (isArray(x)) return _.any(x, f); return f(x); }; +// True if f(x) is true, or x is an array and f(e) is true for some e in x. +// +// Use this for most operators where an array could satisfy the predicate. var _anyIfArrayPlus = function (x, f) { if (f(x)) return true; From b196ba4800745e5d47e231aa1bb16c3c496633af Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 20 Dec 2013 11:06:27 -0800 Subject: [PATCH 009/124] Fix exception while constructing proxy status error. Exception while construction the error string means we never throw into the future, so the process hangs forever. --- packages/ctl-helper/ctl-helper.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index b4698f8b7f..355583cb7e 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -157,9 +157,10 @@ _.extend(Ctl, { var proxyTimeout = Meteor.setTimeout(function () { if (!proxyTagSwitchFuture.isResolved()) proxyTagSwitchFuture['throw']( - new Error("timed out looking for a proxy " + - "or trying to change tags on it " + - proxy.status().status)); + new Error("Timed out looking for a proxy " + + "or trying to change tags on it. Status: " + + (proxy ? proxy.status().status : "no connection")) + ); }, 10*1000); proxyTagSwitchFuture.wait(); Meteor.clearTimeout(proxyTimeout); From 515e6f28158ded457891df3441420d5ab480da5c Mon Sep 17 00:00:00 2001 From: icellan Date: Tue, 17 Dec 2013 23:05:44 +0100 Subject: [PATCH 010/124] Removed extra comma at end of method. Fixes IE7 --- packages/livedata/livedata_common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index f0b6159ec6..1420da5797 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -45,7 +45,7 @@ _.extend(MethodInvocation.prototype, { throw new Error("Can't call setUserId in a method after calling unblock"); self.userId = userId; self._setUserId(userId); - }, + } }); parseDDP = function (stringMessage) { From 995d7282786c899765f90f227f9cc01f0e485f3a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 17:25:02 -0800 Subject: [PATCH 011/124] mongo_runner: Don't send rs.initiate too early Fixes #1696. Thanks to @Maxpain177 for reporting and providing access to a machine where this was easily reproducible. --- tools/mongo_runner.js | 67 ++++++++++++++++++++++++++++++------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 40d0a39bf8..09a3efbacf 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -234,6 +234,8 @@ exports.launchMongo = function (options) { proc.stdout.setEncoding('utf8'); var listening = false; var replSetReady = false; + var replSetReadyToBeInitiated = false; + var alreadyInitiatedReplSet = false; var maybeCallOnListen = function () { if (listening && replSetReady) { if (createReplSet) @@ -241,29 +243,54 @@ exports.launchMongo = function (options) { onListen(); } }; + + var maybeInitiateReplset = function () { + // We need to want to create a replset, be confident that the server is + // listening, be confident that the server's replset implementation is + // ready to be initiated, and have not already done it. + if (!(createReplSet && listening && replSetReadyToBeInitiated + && !alreadyInitiatedReplSet)) { + return; + } + + alreadyInitiatedReplSet = true; + + // Connect to it and start a replset. + var db = new mongoNpmModule.Db( + 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), + {safe: true}); + db.open(function(err, db) { + if (err) + throw err; + db.admin().command({ + replSetInitiate: { + _id: replSetName, + members: [{_id : 0, host: '127.0.0.1:' + options.port}] + } + }, function (err, result) { + if (err) + throw err; + // why this isn't in the error is unclear. + if (result && result.documents && result.documents[0] + && result.documents[0].errmsg) { + throw result.document[0].errmsg; + } + db.close(true); + }); + }); + }; + proc.stdout.on('data', function (data) { + // note: don't use "else ifs" in this, because 'data' can have multiple + // lines + if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) { + replSetReadyToBeInitiated = true; + maybeInitiateReplset(); + } + if (/ \[initandlisten\] waiting for connections on port/.test(data)) { - if (createReplSet) { - // Connect to it and start a replset. - var db = new mongoNpmModule.Db( - 'meteor', new mongoNpmModule.Server('127.0.0.1', options.port), - {safe: true}); - db.open(function(err, db) { - if (err) - throw err; - db.admin().command({ - replSetInitiate: { - _id: replSetName, - members: [{_id : 0, host: '127.0.0.1:' + options.port}] - } - }, function (err, result) { - if (err) - throw err; - db.close(true); - }); - }); - } listening = true; + maybeInitiateReplset(); maybeCallOnListen(); } From 96c907654418b2b9fb12e26575fc4621ba8c5751 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 19 Dec 2013 18:19:15 -0800 Subject: [PATCH 012/124] Don't leave invalid METEOR-PORT files around Could cause mongo startup to hang. Reproduction: $ meteor => Meteor server running on: http://localhost:3000/ # ... wait for server to start, ctrl-c. # leaves '3002' in .meteor/local/db/METEOR-PORT $ meteor -p 5000 # ctrl-c in about a second: once we've wiped the old local db # but before we've configured the new one. # before this commit, '3002' is still in the METEOR-PORT file. $ meteor # before this commit, hangs with: Initializing mongo database... this may take a moment. --- tools/mongo_runner.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index 09a3efbacf..b7d0824c99 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -161,9 +161,11 @@ exports.launchMongo = function (options) { } var portFile = path.join(dbPath, 'METEOR-PORT'); + var portFileExists = false; var createReplSet = true; try { createReplSet = +(fs.readFileSync(portFile)) !== options.port; + portFileExists = true; } catch (e) { if (!e || e.code !== 'ENOENT') throw e; @@ -176,6 +178,11 @@ exports.launchMongo = function (options) { // replSet configuration. It's also a little slow to initiate a new replSet, // thus the attempt to not do it unless the port changes.) if (createReplSet) { + // Delete the port file, so we don't mistakenly believe that the DB is + // still configured. + if (portFileExists) + fs.unlinkSync(portFile); + try { var dbFiles = fs.readdirSync(dbPath); } catch (e) { From 508ad66513cc09ff1b44cf3323d1150c3e62321f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:35:29 -0800 Subject: [PATCH 013/124] Support websockets on Node 0.10.24 too --- docs/client/concepts.html | 8 ++++---- packages/meteor/node-issue-6506-workaround.js | 13 ++++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 57dec594cd..d1e4d30170 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -826,10 +826,10 @@ To get started, run This command will generate a fully-contained Node.js application in the form of a tarball. To run this application, you need to provide Node.js 0.10 and a MongoDB server. (The current release of Meteor has been tested with Node -0.10.21.) You can then run the application by invoking node, specifying the HTTP -port for the application to listen on, and the MongoDB endpoint. If you don't -already have a MongoDB server, we can recommend our friends at -[MongoHQ](http://mongohq.com). +0.10.22, and is recommended for use with 0.10.22 through 0.10.24 only.) You can +then run the application by invoking node, specifying the HTTP port for the +application to listen on, and the MongoDB endpoint. If you don't already have a +MongoDB server, we can recommend our friends at [MongoHQ](http://mongohq.com). $ PORT=3000 MONGO_URL=mongodb://localhost:27017/myapp node bundle/main.js diff --git a/packages/meteor/node-issue-6506-workaround.js b/packages/meteor/node-issue-6506-workaround.js index a08b92a9d8..bd41b7c384 100644 --- a/packages/meteor/node-issue-6506-workaround.js +++ b/packages/meteor/node-issue-6506-workaround.js @@ -1,13 +1,16 @@ // Temporary workaround for https://github.com/joyent/node/issues/6506 -// Our fix involves replicating a bunch of files in order to -// -if (process.version !== 'v0.10.22' && process.version !== 'v0.10.23') { +// Our fix involves replicating a bunch of functions in order to change +// a single line. + +var PATCH_VERSIONS = ['v0.10.22', 'v0.10.23', 'v0.10.24']; + +if (!_.contains(PATCH_VERSIONS, process.version)) { if (!process.env.DISABLE_WEBSOCKETS) { console.error("This version of Meteor contains a patch for a bug in Node v0.10."); - console.error("The patch is against only versions 0.10.22 and 0.10.23."); + console.error("The patch is against only versions 0.10.22 through 0.10.24."); console.error("You are using version " + process.version + " instead, so we cannot apply the patch."); console.error("To mitigate the most common effect of the bug, websockets will be disabled."); - console.error("To enable websockets, use Node v0.10.22 or .23, or upgrade to a later version of Meteor (if available)."); + console.error("To enable websockets, use Node v0.10.22 through v0.10.24, or upgrade to a later version of Meteor (if available)."); process.env.DISABLE_WEBSOCKETS = 't'; } } else { From c5eefd9504ef9bb415d8f7324e988753b9af5b00 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:45:27 -0800 Subject: [PATCH 014/124] History update for 0.7.0.1 --- History.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/History.md b/History.md index 98e61ba25f..43f557d9cb 100644 --- a/History.md +++ b/History.md @@ -1,3 +1,13 @@ +## v0.7.0.1 + +* Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the + message "Initializing mongo database... this may take a moment.". + +* Apply the Node patch to 0.10.24 as well (see the 0.7.0 section for details). + +* Fix gratuitous IE7 incompatibility. + + ## v0.7.0 This version of Meteor contains a patch for a bug in Node 0.10 which From d98b7ed423731874b255ec1c2df682a995181bbb Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:47:13 -0800 Subject: [PATCH 015/124] Don't call onListen more than once eg, maybe the replset loses and regains its PRIMARY. --- tools/mongo_runner.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/mongo_runner.js b/tools/mongo_runner.js index b7d0824c99..d726ad7a16 100644 --- a/tools/mongo_runner.js +++ b/tools/mongo_runner.js @@ -243,10 +243,12 @@ exports.launchMongo = function (options) { var replSetReady = false; var replSetReadyToBeInitiated = false; var alreadyInitiatedReplSet = false; + var alreadyCalledOnListen = false; var maybeCallOnListen = function () { - if (listening && replSetReady) { + if (listening && replSetReady && !alreadyCalledOnListen) { if (createReplSet) fs.writeFileSync(portFile, options.port); + alreadyCalledOnListen = true; onListen(); } }; From 51ad916cafdc78212731345e25d85b61dc66f22d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:53:08 -0800 Subject: [PATCH 016/124] Banner and notice update for 0.7.0.1. --- scripts/admin/banner.txt | 5 ++--- scripts/admin/notices.json | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index def13df77b..1abd8fb7da 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,5 +1,4 @@ -=> Meteor 0.7.0: use MongoDB's operations log to observe equality - queries instead of polling the database server on each update. +=> Meteor 0.7.0.1: Fix hang while setting up local MongoDB. This release is being downloaded in the background. Update your - project to Meteor 0.7.0 by running 'meteor update'. + project to Meteor 0.7.0.1 by running 'meteor update'. diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json index ab41d828c8..7a9375f70a 100644 --- a/scripts/admin/notices.json +++ b/scripts/admin/notices.json @@ -75,6 +75,9 @@ { "release": "0.7.0" }, + { + "release": "0.7.0.1" + }, { "release": "NEXT" } From 42069551405ed7fe9759a8c4a293751f904a75db Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 13:58:30 -0800 Subject: [PATCH 017/124] Better banner text --- scripts/admin/banner.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index 1abd8fb7da..a52ccd54d2 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,4 +1,4 @@ -=> Meteor 0.7.0.1: Fix hang while setting up local MongoDB. +=> Meteor 0.7.0.1: Fix failure to initialize local MongoDB server. This release is being downloaded in the background. Update your project to Meteor 0.7.0.1 by running 'meteor update'. From 5cae2c426a3b599dcd492e2d4514ef22c16b8d84 Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Thu, 19 Dec 2013 18:14:22 -0800 Subject: [PATCH 018/124] update docs app to release sso-1 --- docs/.meteor/release | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index faef31a435..400877328a 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.7.0 +sso-1 From 6fc8332c99872b09d8ef416e0b80436bfe3ef662 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:23:11 -0800 Subject: [PATCH 019/124] Update docs and examples to 0.7.0.1. --- docs/lib/release-override.js | 2 +- examples/leaderboard/.meteor/release | 2 +- examples/parties/.meteor/release | 2 +- examples/todos/.meteor/release | 2 +- examples/wordplay/.meteor/release | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js index 0f61b680aa..60276835d0 100644 --- a/docs/lib/release-override.js +++ b/docs/lib/release-override.js @@ -1,5 +1,5 @@ // While galaxy apps are on their own special meteor releases, override // Meteor.release here. if (Meteor.isClient) { - Meteor.release = Meteor.release ? "0.7.0" : undefined; + Meteor.release = Meteor.release ? "0.7.0.1" : undefined; } diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/leaderboard/.meteor/release +++ b/examples/leaderboard/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/parties/.meteor/release +++ b/examples/parties/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/todos/.meteor/release +++ b/examples/todos/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release index faef31a435..b6e63167d2 100644 --- a/examples/wordplay/.meteor/release +++ b/examples/wordplay/.meteor/release @@ -1 +1 @@ -0.7.0 +0.7.0.1 From caa528733d469d53604abf76a14f0f98368f98cf Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:33:09 -0800 Subject: [PATCH 020/124] Add issue numbers to History --- History.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/History.md b/History.md index 43f557d9cb..d09f53d286 100644 --- a/History.md +++ b/History.md @@ -1,11 +1,11 @@ ## v0.7.0.1 * Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the - message "Initializing mongo database... this may take a moment.". + message "Initializing mongo database... this may take a moment.". #1696 * Apply the Node patch to 0.10.24 as well (see the 0.7.0 section for details). -* Fix gratuitous IE7 incompatibility. +* Fix gratuitous IE7 incompatibility. #1690 ## v0.7.0 From 7341966f49da15302968678446bbbda8181e2db7 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 20 Dec 2013 14:34:06 -0800 Subject: [PATCH 021/124] Fix incorrect Node version number Fixes #1701 --- tools/bundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/bundler.js b/tools/bundler.js index e7acccdf43..9d93ddd75f 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1474,7 +1474,8 @@ var writeSiteArchive = function (targets, outputPath, options) { builder.write('README', { data: new Buffer( "This is a Meteor application bundle. It has only one dependency:\n" + "Node.js 0.10 (with the 'fibers' package). The current release of Meteor\n" + -"has been tested with Node 0.10.21. To run the application:\n" + +"has been tested with Node 0.10.22 and works best with 0.10.22 through\n" + +"0.10.24. To run the application:\n" + "\n" + " $ rm -r programs/server/node_modules/fibers\n" + " $ npm install fibers@1.0.1\n" + From c37d009d96feef10d975c80ee643944f003e42d3 Mon Sep 17 00:00:00 2001 From: Andrew Wilcox Date: Mon, 16 Dec 2013 10:51:51 -0500 Subject: [PATCH 022/124] Extract retry package from livedata --- packages/autoupdate/autoupdate_client.js | 4 +--- packages/autoupdate/package.js | 2 +- packages/livedata/livedata_common.js | 5 ----- packages/livedata/package.js | 4 ++-- packages/retry/.gitignore | 1 + packages/retry/package.js | 10 ++++++++++ packages/{livedata => retry}/retry.js | 23 +++++++++++------------ 7 files changed, 26 insertions(+), 23 deletions(-) create mode 100644 packages/retry/.gitignore create mode 100644 packages/retry/package.js rename packages/{livedata => retry}/retry.js (70%) diff --git a/packages/autoupdate/autoupdate_client.js b/packages/autoupdate/autoupdate_client.js index dfd4063ddd..45179f8ba4 100644 --- a/packages/autoupdate/autoupdate_client.js +++ b/packages/autoupdate/autoupdate_client.js @@ -44,9 +44,7 @@ Autoupdate.newClientAvailable = function () { -// XXX Livedata exporting this via DDP is a hack. See -// packages/livedata/livedata_common.js -var retry = new DDP._Retry({ +var retry = new Retry({ // Unlike the stream reconnect use of Retry, which we want to be instant // in normal operation, this is a wacky failure. We don't want to retry // right away, we can start slowly. diff --git a/packages/autoupdate/package.js b/packages/autoupdate/package.js index f56a572b1a..fded7a3869 100644 --- a/packages/autoupdate/package.js +++ b/packages/autoupdate/package.js @@ -5,7 +5,7 @@ Package.describe({ Package.on_use(function (api) { api.use('webapp', 'server'); - api.use('deps', 'client'); + api.use(['deps', 'retry'], 'client'); api.use(['livedata', 'mongo-livedata'], ['client', 'server']); api.use('deps', 'client'); api.use('reload', 'client', {weak: true}); diff --git a/packages/livedata/livedata_common.js b/packages/livedata/livedata_common.js index 1420da5797..f9a873c6f7 100644 --- a/packages/livedata/livedata_common.js +++ b/packages/livedata/livedata_common.js @@ -115,8 +115,3 @@ stringifyDDP = function (msg) { // state in the DDP session. Meteor.setTimeout and friends clear // it. We can probably find a better way to factor this. DDP._CurrentInvocation = new Meteor.EnvironmentVariable; - - -// This is private and a hack. It is used by autoupdate_client. We -// should refactor. Maybe a separate 'exponential-backoff' package? -DDP._Retry = Retry; diff --git a/packages/livedata/package.js b/packages/livedata/package.js index 6c7778f808..d2b407ae13 100644 --- a/packages/livedata/package.js +++ b/packages/livedata/package.js @@ -6,7 +6,8 @@ Package.describe({ Npm.depends({sockjs: "0.3.8", websocket: "1.0.8"}); Package.on_use(function (api) { - api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps', 'logging'], + api.use(['check', 'random', 'ejson', 'json', 'underscore', 'deps', + 'logging', 'retry'], ['client', 'server']); // It is OK to use this package on a server architecture without making a @@ -33,7 +34,6 @@ Package.on_use(function (api) { // Transport api.use('reload', 'client', {weak: true}); api.add_files('common.js'); - api.add_files('retry.js', ['client', 'server']); api.add_files(['sockjs-0.3.4.js', 'stream_client_sockjs.js'], 'client'); api.add_files('stream_client_nodejs.js', 'server'); api.add_files('stream_client_common.js', ['client', 'server']); diff --git a/packages/retry/.gitignore b/packages/retry/.gitignore new file mode 100644 index 0000000000..677a6fc263 --- /dev/null +++ b/packages/retry/.gitignore @@ -0,0 +1 @@ +.build* diff --git a/packages/retry/package.js b/packages/retry/package.js new file mode 100644 index 0000000000..9c21873dcf --- /dev/null +++ b/packages/retry/package.js @@ -0,0 +1,10 @@ +Package.describe({ + summary: "Retry logic with exponential backoff", + internal: true +}); + +Package.on_use(function (api) { + api.use('underscore', ['client', 'server']); + api.export('Retry'); + api.add_files('retry.js', ['client', 'server']); +}); diff --git a/packages/livedata/retry.js b/packages/retry/retry.js similarity index 70% rename from packages/livedata/retry.js rename to packages/retry/retry.js index d5fdda4d66..a4407b5bdb 100644 --- a/packages/livedata/retry.js +++ b/packages/retry/retry.js @@ -1,24 +1,23 @@ // Retry logic with an exponential backoff. +// +// options: +// baseTimeout: time for initial reconnect attempt (ms). +// exponent: exponential factor to increase timeout each attempt. +// maxTimeout: maximum time between retries (ms). +// minCount: how many times to reconnect "instantly". +// minTimeout: time to wait for the first `minCount` retries (ms). +// fuzz: factor to randomize retry times by (to avoid retry storms). Retry = function (options) { var self = this; _.extend(self, _.defaults(_.clone(options || {}), { - // time for initial reconnect attempt. - baseTimeout: 1000, - // exponential factor to increase timeout each attempt. + baseTimeout: 1000, // 1 second exponent: 2.2, - // maximum time between reconnects. keep this intentionally - // high-ish to ensure a server can recover from a failure caused - // by load + // The default is high-ish to ensure a server can recover from a + // failure caused by load. maxTimeout: 5 * 60000, // 5 minutes - // time to wait for the first 2 retries. this helps page reload - // speed during dev mode restarts, but doesn't hurt prod too - // much (due to CONNECT_TIMEOUT) minTimeout: 10, - // how many times to try to reconnect 'instantly' minCount: 2, - // fuzz factor to randomize reconnect times by. avoid reconnect - // storms. fuzz: 0.5 // +- 25% })); self.retryTimer = null; From 178734b66d7ce9d059cbbe8bdb2788ab541a8632 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 23 Dec 2013 15:17:36 -0800 Subject: [PATCH 023/124] Fix test broken by 0f4a21f (thanks @awwx). --- tools/tests/test_bundler_npm.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/tests/test_bundler_npm.js b/tools/tests/test_bundler_npm.js index e4e1509bc0..188522184e 100644 --- a/tools/tests/test_bundler_npm.js +++ b/tools/tests/test_bundler_npm.js @@ -58,7 +58,7 @@ var _assertCorrectPackageNpmDir = function(deps) { // copy fields with values generated by shrinkwrap that can't be known to the // test author. We set keys on val always in this order so that comparison works well. var val = {}; - _.each(['version', 'from', 'resolved', 'dependencies'], function(key) { + _.each(['version', 'dependencies'], function(key) { if (expected[key]) val[key] = expected[key]; else if (actualMeteorNpmShrinkwrapDependencies[name] && actualMeteorNpmShrinkwrapDependencies[name][key]) @@ -264,7 +264,13 @@ assert.doesNotThrow(function () { var tmpOutputDir = tmpDir(); var result = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip', releaseStamp: 'none', library: lib}); assert.strictEqual(result.errors, false, result.errors && result.errors[0]); + try { _assertCorrectPackageNpmDir(deps); + } catch (e) { + console.log("ACTUAL", e.actual) + console.log("EXPECTED", e.expected) + throw e + } _assertCorrectBundleNpmContents(tmpOutputDir, deps); // Check that a string introduced by our fork is in the source. assert(/clientMaxAge = 604800000/.test( From c0e169dd6217ac11aac80f7cd5e0132d47ded47d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 23 Dec 2013 17:16:27 -0800 Subject: [PATCH 024/124] Place link to _observeDriver on multiplexer Instead of on *only the first* ObserveHandle --- packages/mongo-livedata/mongo_driver.js | 5 ++--- .../mongo-livedata/mongo_livedata_tests.js | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 9d8063fa56..baaa2eeab5 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -999,9 +999,8 @@ MongoConnection.prototype._observeChanges = function ( _testOnlyPollCallback: callbacks._testOnlyPollCallback }); - // This field is only set for the first ObserveHandle in an - // ObserveMultiplexer. It is only there for use tests. - observeHandle._observeDriver = observeDriver; + // This field is only set for use in tests. + multiplexer._observeDriver = observeDriver; } // Blocks until the initial adds have been sent. diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 1f163b4e4b..dbb4a65d36 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -391,13 +391,9 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } }); - // XXX What if there are multiple observe handles on the ObserveMultiplexer? - // There shouldn't be because the collection has a name unique to this - // run. if (Meteor.isServer) { - // For now, has to be polling (not oplog). - test.isTrue(obs._observeDriver); - test.isTrue(obs._observeDriver._suspendPolling); + // For now, has to be polling (not oplog) because it is ordered observe. + test.isTrue(obs._multiplexer._observeDriver._suspendPolling); } var step = 0; @@ -432,7 +428,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, finishObserve(function () { if (Meteor.isServer) - obs._observeDriver._suspendPolling(); + obs._multiplexer._observeDriver._suspendPolling(); // Do a batch of 1-10 operations var batch_count = rnd(10) + 1; @@ -465,7 +461,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } } if (Meteor.isServer) - obs._observeDriver._resumePolling(); + obs._multiplexer._observeDriver._resumePolling(); }); @@ -1886,14 +1882,12 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - _disableOplog", functi if (MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle) { var observeWithOplog = coll.find({x: 5}) .observeChanges({added: function () {}}); - test.isTrue(observeWithOplog._observeDriver); - test.isTrue(observeWithOplog._observeDriver._usesOplog); + test.isTrue(observeWithOplog._multiplexer._observeDriver._usesOplog); observeWithOplog.stop(); } var observeWithoutOplog = coll.find({x: 6}, {_disableOplog: true}) .observeChanges({added: function () {}}); - test.isTrue(observeWithoutOplog._observeDriver); - test.isFalse(observeWithoutOplog._observeDriver._usesOplog); + test.isFalse(observeWithoutOplog._multiplexer._observeDriver._usesOplog); observeWithoutOplog.stop(); }); From 2ed30c5387cb276c8a91d6fcddc56c3a1ab0a46c Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 23 Dec 2013 16:47:31 -0800 Subject: [PATCH 025/124] Update `configureService` and call sites to new service format. The `configure` callback now gets called with a list of endpoints for the service instead of a Services document. --- packages/application-configuration/config.js | 45 +++++++++++++++++--- packages/ctl-helper/ctl-helper.js | 26 +++++------ packages/webapp/webapp_server.js | 7 +-- 3 files changed, 58 insertions(+), 20 deletions(-) diff --git a/packages/application-configuration/config.js b/packages/application-configuration/config.js index 7268dc2ae8..25c3c19a27 100644 --- a/packages/application-configuration/config.js +++ b/packages/application-configuration/config.js @@ -149,15 +149,50 @@ AppConfig.configurePackage = function (packageName, configure) { }; }; -AppConfig.configureService = function (serviceName, configure) { +AppConfig.configureService = function (serviceName, version, configure) { + + // Collect all the endpoints for this service, from both old- and new-format + // documents, and call the `configure` callback with all the service endpoints + // that we know about. + var callConfigure = function (doc) { + var serviceDocs = Services.find({ + name: serviceName, + version: version + }); + var endpoints = []; + serviceDocs.forEach(function (serviceDoc) { + if (serviceDoc.providers) { + _.each(serviceDoc.providers, function (endpoint, app) { + endpoints.push(endpoint); + }); + } else { + endpoints.push(serviceDoc.endpoint); + } + }); + configure(endpoints); + }; + if (ultra) { // there's a Meteor.startup() that produces the various collections, make // sure it runs first before we continue. collectionFuture.wait(); - ultra.subscribe('servicesByName', serviceName); - return Services.find({name: serviceName}).observe({ - added: configure, - changed: configure + // First try to subscribe to the new format service registrations; if that + // sub doesn't exist, then ultraworld hasn't updated to the new format yet, + // so try the old format `servicesByName` sub instead. + ultra.subscribe('services', serviceName, version, { + onError: function (err) { + if (err.error === 404) { + ultra.subscribe('servicesByName', serviceName); + } + } + }); + return Services.find({ + name: serviceName, + version: version + }).observe({ + added: callConfigure, + changed: callConfigure, + removed: callConfigure }); } diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index 355583cb7e..4806ddcc7a 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -139,18 +139,20 @@ _.extend(Ctl, { updateProxyActiveTags: function (tags) { var proxy; var proxyTagSwitchFuture = new Future; - AppConfig.configureService('proxy', function (proxyService) { - try { - proxy = Follower.connect(proxyService.providers.proxy, { - group: "proxy" - }); - proxy.call('updateTags', Ctl.myAppName(), tags); - proxy.disconnect(); - if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['return'](); - } catch (e) { - if (!proxyTagSwitchFuture.isResolved()) - proxyTagSwitchFuture['throw'](e); + AppConfig.configureService('proxy', 'pre0', function (proxyService) { + if (proxyService && ! _.isEmpty(proxyService)) { + try { + proxy = Follower.connect(proxyService, { + group: "proxy" + }); + proxy.call('updateTags', Ctl.myAppName(), tags); + proxy.disconnect(); + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['return'](); + } catch (e) { + if (!proxyTagSwitchFuture.isResolved()) + proxyTagSwitchFuture['throw'](e); + } } }); diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index f647f96746..d8d6f28278 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -580,8 +580,9 @@ var runWebAppServer = function () { // on a per-job basis. Discuss w/ teammates. proxyBinding = AppConfig.configureService( "proxy", + "pre0", function (proxyService) { - if (proxyService.providers.proxy) { + if (proxyService && ! _.isEmpty(proxyService)) { var proxyConf; // XXX Figure out a per-job way to specify bind location // (besides hardcoding the location for ADMIN_APP jobs). @@ -602,9 +603,9 @@ var runWebAppServer = function () { proxyConf = configuration.proxy; } Log("Attempting to bind to proxy at " + - proxyService.providers.proxy); + proxyService); WebAppInternals.bindToProxy(_.extend({ - proxyEndpoint: proxyService.providers.proxy + proxyEndpoint: proxyService }, proxyConf)); } } From 0c975bea03863b105fd6d4d3d7b4ff6364101258 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 26 Dec 2013 17:29:26 -0600 Subject: [PATCH 026/124] Defer the post-login check that the token still exists. --- packages/accounts-base/accounts_server.js | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/packages/accounts-base/accounts_server.js b/packages/accounts-base/accounts_server.js index c3392b391c..79505dbc45 100644 --- a/packages/accounts-base/accounts_server.js +++ b/packages/accounts-base/accounts_server.js @@ -257,13 +257,19 @@ Accounts._setLoginToken = function (userId, connection, newToken) { // before our `findOne` call returns... and then that other fiber // would need for the connection to be in the map for it to close // the connection. - if (! Meteor.users.findOne({ - _id: userId, - "services.resume.loginTokens.hashedToken": newToken - })) { - removeConnectionFromToken(newToken, connection.id); - connection.close(); - } + // + // We defer this check because there's no need for it to be on the critical + // path for login; we just need to ensure that the connection will get + // closed at some point if the token has been deleted. + Meteor.defer(function () { + if (! Meteor.users.findOne({ + _id: userId, + "services.resume.loginTokens.hashedToken": newToken + })) { + removeConnectionFromToken(newToken, connection.id); + connection.close(); + } + }); } }; From bf970344435da2c1c255f50001292d98b11870cc Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 26 Dec 2013 17:48:09 -0600 Subject: [PATCH 027/124] Remove unnecessary first argument on `removeConnectionFromToken` --- packages/accounts-base/accounts_server.js | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/accounts-base/accounts_server.js b/packages/accounts-base/accounts_server.js index 79505dbc45..374e979347 100644 --- a/packages/accounts-base/accounts_server.js +++ b/packages/accounts-base/accounts_server.js @@ -174,10 +174,7 @@ Accounts._setAccountData = function (connectionId, field, value) { Meteor.server.onConnection(function (connection) { accountData[connection.id] = {connection: connection}; connection.onClose(function () { - removeConnectionFromToken( - Accounts._getLoginToken(connection.id), - connection.id - ); + removeConnectionFromToken(connection.id); delete accountData[connection.id]; }); }); @@ -212,8 +209,9 @@ Accounts._getTokenConnections = function (token) { return connectionsByLoginToken[token]; }; -// Remove the connection from the list of open connections for the token. -var removeConnectionFromToken = function (token, connectionId) { +// Remove the connection from the list of open connections for the connection's +// token. +var removeConnectionFromToken = function (connectionId) { var token = Accounts._getLoginToken(connectionId); if (token) { connectionsByLoginToken[token] = _.without( @@ -231,11 +229,7 @@ Accounts._getLoginToken = function (connectionId) { // newToken is a hashed token. Accounts._setLoginToken = function (userId, connection, newToken) { - removeConnectionFromToken( - Accounts._getLoginToken(connection.id), - connection.id - ); - + removeConnectionFromToken(connection.id); Accounts._setAccountData(connection.id, 'loginToken', newToken); if (newToken) { @@ -266,7 +260,7 @@ Accounts._setLoginToken = function (userId, connection, newToken) { _id: userId, "services.resume.loginTokens.hashedToken": newToken })) { - removeConnectionFromToken(newToken, connection.id); + removeConnectionFromToken(connection.id); connection.close(); } }); From 412f89a352e1e55ea036e544d419c7ce0c1800aa Mon Sep 17 00:00:00 2001 From: Matt DeBergalis Date: Thu, 26 Dec 2013 22:57:52 -0800 Subject: [PATCH 028/124] document return value of `Deps.nonreactive` --- docs/client/api.html | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index 5a52591d58..8a63b56bc9 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -2632,9 +2632,10 @@ computation. {{> api_box deps_nonreactive }} -Calls `func()` with `Deps.currentComputation` temporarily set to -`null`. If `func` accesses reactive data sources, these data sources -will never cause a rerun of the enclosing computation. +Calls `func` with `Deps.currentComputation` temporarily set to `null` +and returns `func`'s own return value. If `func` accesses reactive data +sources, these data sources will never cause a rerun of the enclosing +computation. {{> api_box deps_active }} From 484ac965e7a2d1cbc70f0cfd359896ed8c827bad Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 23 Dec 2013 17:16:27 -0800 Subject: [PATCH 029/124] Place link to _observeDriver on multiplexer Instead of on *only the first* ObserveHandle --- packages/mongo-livedata/mongo_driver.js | 5 ++--- .../mongo-livedata/mongo_livedata_tests.js | 18 ++++++------------ 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 9d8063fa56..baaa2eeab5 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -999,9 +999,8 @@ MongoConnection.prototype._observeChanges = function ( _testOnlyPollCallback: callbacks._testOnlyPollCallback }); - // This field is only set for the first ObserveHandle in an - // ObserveMultiplexer. It is only there for use tests. - observeHandle._observeDriver = observeDriver; + // This field is only set for use in tests. + multiplexer._observeDriver = observeDriver; } // Blocks until the initial adds have been sent. diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index 1f163b4e4b..dbb4a65d36 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -391,13 +391,9 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } }); - // XXX What if there are multiple observe handles on the ObserveMultiplexer? - // There shouldn't be because the collection has a name unique to this - // run. if (Meteor.isServer) { - // For now, has to be polling (not oplog). - test.isTrue(obs._observeDriver); - test.isTrue(obs._observeDriver._suspendPolling); + // For now, has to be polling (not oplog) because it is ordered observe. + test.isTrue(obs._multiplexer._observeDriver._suspendPolling); } var step = 0; @@ -432,7 +428,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, finishObserve(function () { if (Meteor.isServer) - obs._observeDriver._suspendPolling(); + obs._multiplexer._observeDriver._suspendPolling(); // Do a batch of 1-10 operations var batch_count = rnd(10) + 1; @@ -465,7 +461,7 @@ Tinytest.addAsync("mongo-livedata - fuzz test, " + idGeneration, function(test, } } if (Meteor.isServer) - obs._observeDriver._resumePolling(); + obs._multiplexer._observeDriver._resumePolling(); }); @@ -1886,14 +1882,12 @@ Meteor.isServer && Tinytest.add("mongo-livedata - oplog - _disableOplog", functi if (MongoInternals.defaultRemoteCollectionDriver().mongo._oplogHandle) { var observeWithOplog = coll.find({x: 5}) .observeChanges({added: function () {}}); - test.isTrue(observeWithOplog._observeDriver); - test.isTrue(observeWithOplog._observeDriver._usesOplog); + test.isTrue(observeWithOplog._multiplexer._observeDriver._usesOplog); observeWithOplog.stop(); } var observeWithoutOplog = coll.find({x: 6}, {_disableOplog: true}) .observeChanges({added: function () {}}); - test.isTrue(observeWithoutOplog._observeDriver); - test.isFalse(observeWithoutOplog._observeDriver._usesOplog); + test.isFalse(observeWithoutOplog._multiplexer._observeDriver._usesOplog); observeWithoutOplog.stop(); }); From 241ff9fa89a16dada67344707d77aa1b3719cb22 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Sat, 28 Dec 2013 11:39:40 -0600 Subject: [PATCH 030/124] Pass through Follower.connect options to DDP.connect. --- packages/follower-livedata/follower.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/follower-livedata/follower.js b/packages/follower-livedata/follower.js index 0e4737522a..1e8f4b2b61 100644 --- a/packages/follower-livedata/follower.js +++ b/packages/follower-livedata/follower.js @@ -104,7 +104,7 @@ Follower = { url: url }]); } else { - conn = DDP.connect(url); + conn = DDP.connect(url, options); prevReconnect = conn.reconnect; prevDisconnect = conn.disconnect; prevApply = conn.apply; From 252caab5c1e4b8ad2c0ed269a63ec90e54e61975 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 29 Dec 2013 16:42:08 -0800 Subject: [PATCH 031/124] Add instructions to https://install.meteor.com/ --- scripts/admin/install-engine.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/admin/install-engine.sh b/scripts/admin/install-engine.sh index ef6eb9b4ed..ebfd904006 100644 --- a/scripts/admin/install-engine.sh +++ b/scripts/admin/install-engine.sh @@ -1,5 +1,19 @@ #!/bin/sh +# This is the Meteor install script! +# Are you looking at this in your web browser, and would like to install Meteor? +# Just open up your terminal and type: +# +# curl https://install.meteor.com/ | sh +# +# Meteor currently supports: +# - Mac: OS X 10.6 and above +# - Linux: x86 and x86_64 systems + + + +# Now, on to the actual installer! + ## NOTE sh NOT bash. This script should be POSIX sh only, since we don't ## know what shell the user has. Debian uses 'dash' for 'sh', for ## example. From 4484a28d629d8cd145b909782743d509ecdd089b Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 30 Dec 2013 14:24:45 -0800 Subject: [PATCH 032/124] Make _subscribeAndWait return a subscription handle. --- packages/livedata/livedata_connection.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/livedata/livedata_connection.js b/packages/livedata/livedata_connection.js index 6142e32dac..c876ffc89c 100644 --- a/packages/livedata/livedata_connection.js +++ b/packages/livedata/livedata_connection.js @@ -533,6 +533,7 @@ _.extend(Connection.prototype, { var self = this; var f = new Future(); var ready = false; + var handle; args = args || []; args.push({ onReady: function () { @@ -547,8 +548,9 @@ _.extend(Connection.prototype, { } }); - self.subscribe.apply(self, [name].concat(args)); + handle = self.subscribe.apply(self, [name].concat(args)); f.wait(); + return handle; }, methods: function (methods) { From f32c1ca419bf71bb37f295056fb5232b6449f67c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 18 Dec 2013 17:36:39 -0800 Subject: [PATCH 033/124] Refactor LocalCollection._makeLookupFunction Based on a lot of experimentation about how Mongo handles arrays, numeric indices, and nulls. Also sets the groundwork for update modifiers with "foo.$.bar" by tracking arrayIndex during lookups. Adds a few expected_fail tests for where I found behavior that differed from Mongo. Some of this behavior seems to have changed between (stable) Mongo 2.4 and (unstable) Mongo 2.5. In the interest of future-proofing, I have preferred the 2.5 behavior. --- packages/minimongo/minimongo_tests.js | 48 ++++++- packages/minimongo/selector.js | 159 ++++++++++++++++++------ packages/minimongo/selector_modifier.js | 5 - 3 files changed, 166 insertions(+), 46 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index efb3d86770..8d96a90c93 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -256,10 +256,11 @@ Tinytest.add("minimongo - lookup", function (test) { [1, [2], undefined]); var lookupA0X = LocalCollection._makeLookupFunction('a.0.x'); - test.equal(lookupA0X({a: [{x: 1}]}), [1]); - test.equal(lookupA0X({a: [{x: [1]}]}), [[1]]); + test.equal(lookupA0X({a: [{x: 1}]}), [1, undefined]); + test.equal(lookupA0X({a: [{x: [1]}]}), [[1], undefined]); test.equal(lookupA0X({a: 5}), [undefined]); - test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), [1]); + test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), + [1, undefined, undefined, undefined]); }); Tinytest.add("minimongo - selector_compiler", function (test) { @@ -600,8 +601,49 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({"a.b": /a/}, {a: {b: "dog"}}); match({"a.b.c": null}, {}); match({"a.b.c": null}, {a: 1}); + match({"a.b": null}, {a: 1}); match({"a.b.c": null}, {a: {b: 4}}); + // dotted keypaths, nulls, numeric indices, arrays + nomatch({"a.b": null}, {a: [1]}); + match({"a.b": []}, {a: {b: []}}); + var big = {a: [{b: 1}, 2, {}, {b: [3, 4]}]}; + match({"a.b": 1}, big); + match({"a.b": [3, 4]}, big); + test.expect_fail(); // XXX fix by observing dontIterate + nomatch({"a.b": 3}, big); + test.expect_fail(); // XXX fix by observing dontIterate + nomatch({"a.b": 4}, big); + match({"a.b": null}, big); // matches on slot 2 + match({'a.1': 8}, {a: [7, 8, 9]}); + nomatch({'a.1': 7}, {a: [7, 8, 9]}); + nomatch({'a.1': null}, {a: [7, 8, 9]}); + match({'a.1': [8, 9]}, {a: [7, [8, 9]]}); + nomatch({'a.1': 6}, {a: [[6, 7], [8, 9]]}); + nomatch({'a.1': 7}, {a: [[6, 7], [8, 9]]}); + test.expect_fail(); // XXX fix by observing dontIterate + nomatch({'a.1': 8}, {a: [[6, 7], [8, 9]]}); + test.expect_fail(); // XXX fix by observing dontIterate + nomatch({'a.1': 9}, {a: [[6, 7], [8, 9]]}); + match({"a.1": 2}, {a: [0, {1: 2}, 3]}); + match({"a.1": {1: 2}}, {a: [0, {1: 2}, 3]}); + match({"x.1.y": 8}, {x: [7, {y: 8}, 9]}); + // comes from trying '1' as key in the plain object + match({"x.1.y": null}, {x: [7, {y: 8}, 9]}); + match({"a.1.b": 9}, {a: [7, {b: 9}, {1: {b: 'foo'}}]}); + match({"a.1.b": 'foo'}, {a: [7, {b: 9}, {1: {b: 'foo'}}]}); + match({"a.1.b": null}, {a: [7, {b: 9}, {1: {b: 'foo'}}]}); + match({"a.1.b": 2}, {a: [1, [{b: 2}], 3]}); + nomatch({"a.1.b": null}, {a: [1, [{b: 2}], 3]}); + // this is new behavior in mongo 2.5 + nomatch({"a.0.b": null}, {a: [5]}); + match({"a.1": 4}, {a: [{1: 4}, 5]}); + match({"a.1": 5}, {a: [{1: 4}, 5]}); + nomatch({"a.1": null}, {a: [{1: 4}, 5]}); + match({"a.1.foo": 4}, {a: [{1: {foo: 4}}, {foo: 5}]}); + match({"a.1.foo": 5}, {a: [{1: {foo: 4}}, {foo: 5}]}); + match({"a.1.foo": null}, {a: [{1: {foo: 4}}, {foo: 5}]}); + // trying to access a dotted field that is undefined at some point // down the chain nomatch({"a.b": 1}, {x: 2}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 44ad5d5a62..96e26e5550 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -1,9 +1,21 @@ // Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as // arrays. +// XXX maybe this should be EJSON.isArray var isArray = function (x) { return _.isArray(x) && !EJSON.isBinary(x); }; +// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about +// RegExp +// XXX note that _type(undefined) === 3!!!! +var isPlainObject = function (x) { + return x && LocalCollection._f._type(x) === 3; +}; + +var isIndexable = function (x) { + return isArray(x) || isPlainObject(x); +}; + // If x is an array, true if f(e) is true for some e in x // (but never try f(x) directly) // Otherwise, true if f(x) is true. @@ -557,61 +569,132 @@ MinimongoTest.matches = function (selector, doc) { return (LocalCollection._compileSelector(selector))(doc); }; + +// string can be converted to integer +numericKey = function (s) { + return /^[0-9]+$/.test(s); +}; + +// XXX redoc // _makeLookupFunction(key) returns a lookup function. // // A lookup function takes in a document and returns an array of matching -// values. This array has more than one element if any segment of the key other -// than the last one is an array. ie, any arrays found when doing non-final -// lookups result in this function "branching"; each element in the returned -// array represents the value found at this branch. If any branch doesn't have a -// final value for the full key, its element in the returned list will be -// undefined. It always returns a non-empty array. +// values. If no arrays are found while looking up the key, this array will +// have exactly one value (possibly 'undefined', if some segment of the key was +// not found). +// +// If arrays are found in the middle, this can have more than one element, since +// we "branch". When we "branch", if there are more key segments to look up, +// then we only pursue branches that are plain objects (not arrays or scalars). +// This means we can actually end up with no entries! +// +// At the top level, you may only pass in a plain object. // // _makeLookupFunction('a.x')({a: {x: 1}}) returns [1] // _makeLookupFunction('a.x')({a: {x: [1]}}) returns [[1]] // _makeLookupFunction('a.x')({a: 5}) returns [undefined] +// _makeLookupFunction('a.x')({a: [5]}) returns [] // _makeLookupFunction('a.x')({a: [{x: 1}, +// [], +// 4, // {x: [2]}, // {y: 3}]}) // returns [1, [2], undefined] -LocalCollection._makeLookupFunction = function (key) { - var dotLocation = key.indexOf('.'); - var first, lookupRest, nextIsNumeric; - if (dotLocation === -1) { - first = key; - } else { - first = key.substr(0, dotLocation); - var rest = key.substr(dotLocation + 1); - lookupRest = LocalCollection._makeLookupFunction(rest); - // Is the next (perhaps final) piece numeric (ie, an array lookup?) - nextIsNumeric = /^\d+(\.|$)/.test(rest); +LocalCollection._makeLookupFunction2 = function (key) { + var parts = key.split('.'); + var firstPart = parts.length ? parts[0] : ''; + var firstPartIsNumeric = numericKey(firstPart); + var lookupRest; + if (parts.length > 1) { + lookupRest = LocalCollection._makeLookupFunction2(parts.slice(1).join('.')); } - return function (doc) { - if (doc == null) // null or undefined - return [undefined]; - var firstLevel = doc[first]; + // Doc will always be a plain object or an array. + // apply an explicit numeric index, an array. + return function (doc, firstArrayIndex) { + if (isArray(doc)) { + // If we're being asked to do an invalid lookup into an array (non-integer + // or out-of-bounds), return no results (which is different from returning + // a single undefined result, in that `null` equality checks won't match). + if (!(firstPartIsNumeric && firstPart < doc.length)) + return []; - // We don't "branch" at the final level. - if (!lookupRest) - return [firstLevel]; + // If this is the first array index we've seen, remember the index. + // (Mongo doesn't support multiple uses of '$', at least not in 2.5. + if (firstArrayIndex === undefined) + firstArrayIndex = +firstPart; + } - // It's an empty array, and we're not done: we won't find anything. - if (isArray(firstLevel) && firstLevel.length === 0) - return [undefined]; + // Do our first lookup. + var firstLevel = doc[firstPart]; - // For each result at this level, finish the lookup on the rest of the key, - // and return everything we find. Also, if the next result is a number, - // don't branch here. + // If there is no deeper to dig, return what we found. // - // Technically, in MongoDB, we should be able to handle the case where - // objects have numeric keys, but Mongo doesn't actually handle this - // consistently yet itself, see eg - // https://jira.mongodb.org/browse/SERVER-2898 - // https://github.com/mongodb/mongo/blob/master/jstests/array_match2.js - if (!isArray(firstLevel) || nextIsNumeric) - firstLevel = [firstLevel]; - return Array.prototype.concat.apply([], _.map(firstLevel, lookupRest)); + // If what we found is an array, most value selectors will choose to treat + // the elements of the array as matchable values in their own right, but + // that's done outside of the lookup function. (Exceptions to this are $size + // and stuff relating to $elemMatch. eg, {a: {$size: 2}} does not match {a: + // [[1, 2]]}.) + // + // That said, if we just did an *explicit* array lookup (on doc) to find + // firstLevel, and firstLevel is an array too, we do NOT want value + // selectors to iterate over it. eg, {'a.0': 5} does not match {a: [[5]]}. + // So in that case, we mark the return value as "don't iterate". + if (!lookupRest) { + return [{value: firstLevel, + dontIterate: isArray(doc) && isArray(firstLevel), + arrayIndex: firstArrayIndex}]; + } + + // We need to dig deeper. But if we can't, because what we've found is not + // an array or plain object, we're done. If we just did a numeric index into + // an array, we return nothing here (this is a change in Mongo 2.5 from + // Mongo 2.4, where {'a.0.b': null} stopped matching {a: [5]}). Otherwise, + // return a single `undefined` (which can, for example, match via equality + // with `null`). + if (!isIndexable(firstLevel)) { + return isArray(doc) ? [] : [{value: undefined, + arrayIndex: firstArrayIndex}]; + } + + var result = []; + var appendToResult = function (more) { + Array.prototype.push.apply(result, more); + }; + + // Dig deeper: look up the rest of the parts on whatever we've found. + // (lookupRest is smart enough to not try to do invalid lookups into + // firstLevel if it's an array.) + appendToResult(lookupRest(firstLevel, firstArrayIndex)); + + // If we found an array, then in *addition* to potentially treating the next + // part as a literal integer lookup, we should also "branch": try to do look + // up the rest of the parts on each array element in parallel. + // + // In this case, we *only* dig deeper into array elements that are plain + // objects. (Recall that we only got this far if we have further to dig.) + // This makes sense: we certainly don't dig deeper into non-indexable + // objects. And it would be weird to dig into an array: it's simpler to have + // a rule that explicit integer indexes only apply to an outer array, not to + // an array you find after a branching search. + if (isArray(firstLevel)) { + _.each(firstLevel, function (branch, arrayIndex) { + if (isPlainObject(branch)) { + appendToResult(lookupRest( + branch, + firstArrayIndex === undefined ? arrayIndex : firstArrayIndex)); + } + }); + } + + return result; + }; +}; + +LocalCollection._makeLookupFunction = function (key) { + var real = LocalCollection._makeLookupFunction2(key); + return function (doc) { + return _.pluck(real(doc), 'value'); }; }; diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index 6e6a65f3b8..c6a9789fdf 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -118,11 +118,6 @@ function pathHasNumericKeys (path) { return _.any(path.split('.'), numericKey); } -// string can be converted to integer -function numericKey (s) { - return /^[0-9]+$/.test(s); -} - function isLiteralSelector (selector) { return _.all(selector, function (subSelector, keyPath) { if (keyPath.substr(0, 1) === "$" || _.isRegExp(subSelector)) From c94760a0f4d8c8b0f47d3fca02f6cd7a55556789 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 24 Dec 2013 19:04:44 -0800 Subject: [PATCH 034/124] Fix an old comment. --- packages/minimongo/selector.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 96e26e5550..906e8965c0 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -414,9 +414,9 @@ LocalCollection._f = { if (v === null) return 10; if (v instanceof RegExp) + // note that typeof(/x/) === "object" return 11; if (typeof v === "function") - // note that typeof(/x/) === "function" return 13; if (v instanceof Date) return 9; From aea6d0cca9b2c2cf4052bfb87ace6d604b27187f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 24 Dec 2013 19:07:19 -0800 Subject: [PATCH 035/124] Factor sort code into its own file. --- packages/minimongo/package.js | 1 + packages/minimongo/selector.js | 104 +-------------------------------- packages/minimongo/sort.js | 100 +++++++++++++++++++++++++++++++ 3 files changed, 102 insertions(+), 103 deletions(-) create mode 100644 packages/minimongo/sort.js diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js index e226b80ce0..69dc2045a4 100644 --- a/packages/minimongo/package.js +++ b/packages/minimongo/package.js @@ -13,6 +13,7 @@ Package.on_use(function (api) { api.add_files([ 'minimongo.js', 'selector.js', + 'sort.js', 'projection.js', 'modify.js', 'diff.js', diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 906e8965c0..7cf0ef62c2 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -1,7 +1,7 @@ // Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as // arrays. // XXX maybe this should be EJSON.isArray -var isArray = function (x) { +isArray = function (x) { return _.isArray(x) && !EJSON.isBinary(x); }; @@ -780,105 +780,3 @@ LocalCollection._compileSelector = function (selector, cursor) { return compileDocumentSelector(selector, cursor); }; - -// Give a sort spec, which can be in any of these forms: -// {"key1": 1, "key2": -1} -// [["key1", "asc"], ["key2", "desc"]] -// ["key1", ["key2", "desc"]] -// -// (.. with the first form being dependent on the key enumeration -// behavior of your javascript VM, which usually does what you mean in -// this case if the key names don't look like integers ..) -// -// return a function that takes two objects, and returns -1 if the -// first object comes first in order, 1 if the second object comes -// first, or 0 if neither object comes before the other. - -LocalCollection._compileSort = function (spec, cursor) { - var sortSpecParts = []; - - if (spec instanceof Array) { - for (var i = 0; i < spec.length; i++) { - if (typeof spec[i] === "string") { - sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(spec[i]), - ascending: true - }); - } else { - sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(spec[i][0]), - ascending: spec[i][1] !== "desc" - }); - } - } - } else if (typeof spec === "object") { - for (var key in spec) { - sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(key), - ascending: spec[key] >= 0 - }); - } - } else { - throw Error("Bad sort specification: ", JSON.stringify(spec)); - } - - // If there are no sorting rules specified, try to sort on _distance hidden - // fields on cursor we may acquire if query involved $near operator. - if (sortSpecParts.length === 0) - return function (a, b) { - if (!cursor || !cursor._distance) - return 0; - return cursor._distance[a._id] - cursor._distance[b._id]; - }; - - // reduceValue takes in all the possible values for the sort key along various - // branches, and returns the min or max value (according to the bool - // findMin). Each value can itself be an array, and we look at its values - // too. (ie, we do a single level of flattening on branchValues, then find the - // min/max.) - var reduceValue = function (branchValues, findMin) { - var reduced; - var first = true; - // Iterate over all the values found in all the branches, and if a value is - // an array itself, iterate over the values in the array separately. - _.each(branchValues, function (branchValue) { - // Value not an array? Pretend it is. - if (!isArray(branchValue)) - branchValue = [branchValue]; - // Value is an empty array? Pretend it was missing, since that's where it - // should be sorted. - if (isArray(branchValue) && branchValue.length === 0) - branchValue = [undefined]; - _.each(branchValue, function (value) { - // We should get here at least once: lookup functions return non-empty - // arrays, so the outer loop runs at least once, and we prevented - // branchValue from being an empty array. - if (first) { - reduced = value; - first = false; - } else { - // Compare the value we found to the value we found so far, saving it - // if it's less (for an ascending sort) or more (for a descending - // sort). - var cmp = LocalCollection._f._cmp(reduced, value); - if ((findMin && cmp > 0) || (!findMin && cmp < 0)) - reduced = value; - } - }); - }); - return reduced; - }; - - return function (a, b) { - for (var i = 0; i < sortSpecParts.length; ++i) { - var specPart = sortSpecParts[i]; - var aValue = reduceValue(specPart.lookup(a), specPart.ascending); - var bValue = reduceValue(specPart.lookup(b), specPart.ascending); - var compare = LocalCollection._f._cmp(aValue, bValue); - if (compare !== 0) - return specPart.ascending ? compare : -compare; - }; - return 0; - }; -}; - diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js new file mode 100644 index 0000000000..fb3b80d881 --- /dev/null +++ b/packages/minimongo/sort.js @@ -0,0 +1,100 @@ +// Give a sort spec, which can be in any of these forms: +// {"key1": 1, "key2": -1} +// [["key1", "asc"], ["key2", "desc"]] +// ["key1", ["key2", "desc"]] +// +// (.. with the first form being dependent on the key enumeration +// behavior of your javascript VM, which usually does what you mean in +// this case if the key names don't look like integers ..) +// +// return a function that takes two objects, and returns -1 if the +// first object comes first in order, 1 if the second object comes +// first, or 0 if neither object comes before the other. + +LocalCollection._compileSort = function (spec, cursor) { + var sortSpecParts = []; + + if (spec instanceof Array) { + for (var i = 0; i < spec.length; i++) { + if (typeof spec[i] === "string") { + sortSpecParts.push({ + lookup: LocalCollection._makeLookupFunction(spec[i]), + ascending: true + }); + } else { + sortSpecParts.push({ + lookup: LocalCollection._makeLookupFunction(spec[i][0]), + ascending: spec[i][1] !== "desc" + }); + } + } + } else if (typeof spec === "object") { + for (var key in spec) { + sortSpecParts.push({ + lookup: LocalCollection._makeLookupFunction(key), + ascending: spec[key] >= 0 + }); + } + } else { + throw Error("Bad sort specification: ", JSON.stringify(spec)); + } + + // If there are no sorting rules specified, try to sort on _distance hidden + // fields on cursor we may acquire if query involved $near operator. + if (sortSpecParts.length === 0) + return function (a, b) { + if (!cursor || !cursor._distance) + return 0; + return cursor._distance[a._id] - cursor._distance[b._id]; + }; + + // reduceValue takes in all the possible values for the sort key along various + // branches, and returns the min or max value (according to the bool + // findMin). Each value can itself be an array, and we look at its values + // too. (ie, we do a single level of flattening on branchValues, then find the + // min/max.) + var reduceValue = function (branchValues, findMin) { + var reduced; + var first = true; + // Iterate over all the values found in all the branches, and if a value is + // an array itself, iterate over the values in the array separately. + _.each(branchValues, function (branchValue) { + // Value not an array? Pretend it is. + if (!isArray(branchValue)) + branchValue = [branchValue]; + // Value is an empty array? Pretend it was missing, since that's where it + // should be sorted. + if (isArray(branchValue) && branchValue.length === 0) + branchValue = [undefined]; + _.each(branchValue, function (value) { + // We should get here at least once: lookup functions return non-empty + // arrays, so the outer loop runs at least once, and we prevented + // branchValue from being an empty array. + if (first) { + reduced = value; + first = false; + } else { + // Compare the value we found to the value we found so far, saving it + // if it's less (for an ascending sort) or more (for a descending + // sort). + var cmp = LocalCollection._f._cmp(reduced, value); + if ((findMin && cmp > 0) || (!findMin && cmp < 0)) + reduced = value; + } + }); + }); + return reduced; + }; + + return function (a, b) { + for (var i = 0; i < sortSpecParts.length; ++i) { + var specPart = sortSpecParts[i]; + var aValue = reduceValue(specPart.lookup(a), specPart.ascending); + var bValue = reduceValue(specPart.lookup(b), specPart.ascending); + var compare = LocalCollection._f._cmp(aValue, bValue); + if (compare !== 0) + return specPart.ascending ? compare : -compare; + }; + return 0; + }; +}; From 447a6b3e0d56810b9da7615f8113734a9d4ec40c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 11:46:25 -0800 Subject: [PATCH 036/124] Some updates to our NOTES file. --- packages/minimongo/NOTES | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/minimongo/NOTES b/packages/minimongo/NOTES index e2db1fb0e2..7e7927d619 100644 --- a/packages/minimongo/NOTES +++ b/packages/minimongo/NOTES @@ -8,7 +8,12 @@ values won't work.) In update, we don't support '$' to indicate the matched array object as in {$set: {'a.$.x': 12}}. -Sort does not support subkeys. You can sort on 'a', but not 'a.b'. +Sort does not correctly implement the Mongo behavior where only array entries +that match the query's selector are used as sort keys. Specifically, when +sorting a document with sort specifier {a:1}, the document {a: [5, 10]} will +usually have sort key 5 (the minimum is used for ascending sorts and the maximum +for descending sorts). But if the query's selector is {a: {$gt: 7}}, then Mongo +will actually use 10 as the sort key because 5 does not match. ## ON TYPES ## @@ -29,9 +34,6 @@ integer type, and we don't support the integer type yet.) ## API ## -find() doesn't support retrieving a subset of fields. It always -returns the whole doc. - find() doesn't support the min and max parameters. findAndModify isn't supported. @@ -43,9 +45,6 @@ update() should have a clear stance on atomicity (both in terms of multiple ops on a single document, and on multi-document update mode.) It just hasn't been looked at/thought about yet. -upsert combined with $-operators might work, but hasn't actually been -looked at or tested. - In general, the API needs tests, especially update. (On the other hand, the underlying selector and mutator code is quite well tested.) @@ -61,5 +60,5 @@ just get a random exception or error. We don't support capped collections. -No performance optimization has been done. In particular, there are no +Little performance optimization has been done. In particular, there are no indexes. From c85c70f0719b2e11e17121e962f1bc56e09b2958 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 12:48:55 -0800 Subject: [PATCH 037/124] refactor: _compileSelector results {result:bool} instead of just a bool. This will give us room to add arrayIndex (and perhaps to get the geoquery distance stuff into a better place). This is just a very surface refactoring: it doesn't even extend to the "value selectors" yet. --- packages/minimongo/minimongo.js | 18 ++++---- packages/minimongo/modify.js | 4 +- packages/minimongo/selector.js | 42 ++++++++++++------- packages/minimongo/selector_modifier.js | 2 +- .../mongo-livedata/oplog_observe_driver.js | 4 +- 5 files changed, 42 insertions(+), 28 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 54870d87b1..b23effb8e0 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -403,7 +403,7 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { // slow path for arbitrary selector, sort, skip, limit for (var id in self.collection.docs) { var doc = self.collection.docs[id]; - if (self.selector_f(doc)) { + if (self.selector_f(doc).result) { if (ordered) results.push(doc); else @@ -477,7 +477,7 @@ LocalCollection.prototype.insert = function (doc, callback) { // trigger live queries that match for (var qid in self.queries) { var query = self.queries[qid]; - if (query.selector_f(doc)) { + if (query.selector_f(doc).result) { if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -514,13 +514,13 @@ LocalCollection.prototype.remove = function (selector, callback) { var strId = LocalCollection._idStringify(id); // We still have to run selector_f, in case it's something like // {_id: "X", a: 42} - if (_.has(self.docs, strId) && selector_f(self.docs[strId])) + if (_.has(self.docs, strId) && selector_f(self.docs[strId]).result) remove.push(strId); }); } else { for (var id in self.docs) { var doc = self.docs[id]; - if (selector_f(doc)) { + if (selector_f(doc).result) { remove.push(id); } } @@ -531,7 +531,7 @@ LocalCollection.prototype.remove = function (selector, callback) { var removeId = remove[i]; var removeDoc = self.docs[removeId]; _.each(self.queries, function (query, qid) { - if (query.selector_f(removeDoc)) { + if (query.selector_f(removeDoc).result) { if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -590,8 +590,10 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { for (var id in self.docs) { var doc = self.docs[id]; - if (selector_f(doc)) { + var queryResult = selector_f(doc); + if (queryResult.result) { // XXX Should we save the original even if mod ends up being a no-op? + // XXX queryResult should have arrayIndex on it, useful for '$' self._saveOriginal(id, doc); self._modifyAndNotify(doc, mod, recomputeQids); ++updateCount; @@ -665,7 +667,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (var qid in self.queries) { var query = self.queries[qid]; if (query.ordered) { - matched_before[qid] = query.selector_f(doc); + matched_before[qid] = query.selector_f(doc).result; } else { // Because we don't support skip or limit (yet) in unordered queries, we // can just do a direct lookup. @@ -681,7 +683,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (qid in self.queries) { query = self.queries[qid]; var before = matched_before[qid]; - var after = query.selector_f(doc); + var after = query.selector_f(doc).result; if (query.cursor.skip || query.cursor.limit) { // We need to recompute any query where the doc may have been in the diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index e52a03b5c3..4ea173d79b 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -317,8 +317,8 @@ LocalCollection._modifiers = { // same issue as $elemMatch possibly? var match = LocalCollection._compileSelector(arg); for (var i = 0; i < x.length; i++) - if (!match(x[i])) - out.push(x[i]) + if (!match(x[i]).result) + out.push(x[i]); } else { for (var i = 0; i < x.length; i++) if (!LocalCollection._f._equal(x[i], arg)) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 7cf0ef62c2..ccdf835551 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -125,10 +125,11 @@ var LOGICAL_OPERATORS = { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); }); + return compileDocumentSelector(selector, cursor); + }); return function (doc, wholeDoc) { return _.all(subSelectorFunctions, function (f) { - return f(doc, wholeDoc); + return f(doc, wholeDoc).result; }); }; }, @@ -137,10 +138,11 @@ var LOGICAL_OPERATORS = { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); }); + return compileDocumentSelector(selector, cursor); + }); return function (doc, wholeDoc) { return _.any(subSelectorFunctions, function (f) { - return f(doc, wholeDoc); + return f(doc, wholeDoc).result; }); }; }, @@ -149,10 +151,11 @@ var LOGICAL_OPERATORS = { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); }); + return compileDocumentSelector(selector, cursor); + }); return function (doc, wholeDoc) { return _.all(subSelectorFunctions, function (f) { - return !f(doc, wholeDoc); + return !f(doc, wholeDoc).result; }); }; }, @@ -327,7 +330,7 @@ var VALUE_OPERATORS = { if (!isArray(value)) return false; return _.any(value, function (x) { - return matcher(x, doc); + return matcher(x, doc).result; }); }; }, @@ -566,7 +569,7 @@ LocalCollection._f = { // For unit tests. True if the given document matches the given // selector. MinimongoTest.matches = function (selector, doc) { - return (LocalCollection._compileSelector(selector))(doc); + return (LocalCollection._compileSelector(selector))(doc).result; }; @@ -576,6 +579,8 @@ numericKey = function (s) { }; // XXX redoc +// XXX be aware that _compileSort currently assumes that lookup functions +// return non-empty arrays but that is no longer the case // _makeLookupFunction(key) returns a lookup function. // // A lookup function takes in a document and returns an array of matching @@ -746,24 +751,27 @@ var compileDocumentSelector = function (docSelector, cursor) { // If called w/o wholeDoc, doc is considered the original by default if (wholeDoc === undefined) wholeDoc = doc; - return _.all(perKeySelectors, function (f) { + return {result: _.all(perKeySelectors, function (f) { return f(doc, wholeDoc); - }); + })}; }; }; // Given a selector, return a function that takes one argument, a -// document, and returns true if the document matches the selector, -// else false. +// document. It returns an object with fields +// - result: bool, true if the document matches the selector +// XXX add "arrayIndex" for use by update with '$' LocalCollection._compileSelector = function (selector, cursor) { // you can pass a literal function instead of a selector if (selector instanceof Function) - return function (doc) {return selector.call(doc);}; + return function (doc) { + return {result: !!selector.call(doc)}; + }; // shorthand -- scalars match _id if (LocalCollection._selectorIsId(selector)) { return function (doc) { - return EJSON.equals(doc._id, selector); + return {result: EJSON.equals(doc._id, selector)}; }; } @@ -771,7 +779,7 @@ LocalCollection._compileSelector = function (selector, cursor) { // likely programmer error, and not what you want, particularly for // destructive operations. if (!selector || (('_id' in selector) && !selector._id)) - return function (doc) {return false;}; + return matchesNothingSelector; // Top level can't be an array or true or binary. if (typeof(selector) === 'boolean' || isArray(selector) || @@ -780,3 +788,7 @@ LocalCollection._compileSelector = function (selector, cursor) { return compileDocumentSelector(selector, cursor); }; + +var matchesNothingSelector = function (doc) { + return {result: false}; +}; diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index c6a9789fdf..0901647b53 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -97,7 +97,7 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) throw e; } - return selectorFn(doc); + return selectorFn(doc).result; }; // Returns a list of key paths the given selector is looking for diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index e4bbaccb73..3319d74951 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -131,7 +131,7 @@ _.extend(OplogObserveDriver.prototype, { var self = this; newDoc = _.clone(newDoc); - var matchesNow = newDoc && self._selectorFn(newDoc); + var matchesNow = newDoc && self._selectorFn(newDoc).result; if (mustMatchNow && !matchesNow) { throw Error("expected " + EJSON.stringify(newDoc) + " to match " + EJSON.stringify(self._cursorDescription)); @@ -238,7 +238,7 @@ _.extend(OplogObserveDriver.prototype, { // XXX what if selector yields? for now it can't but later it could have // $where - if (self._selectorFn(op.o)) + if (self._selectorFn(op.o).result) self._add(op.o); } else if (op.op === 'u') { // Is this a modifier ($set/$unset, which may require us to poll the From 466664c1213b78614456ef107131216eebc409a0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 12:58:56 -0800 Subject: [PATCH 038/124] Minor refactoring in compileDocumentSelector --- packages/minimongo/selector.js | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index ccdf835551..e64dde46ed 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -746,15 +746,7 @@ var compileDocumentSelector = function (docSelector, cursor) { } }); - - return function (doc, wholeDoc) { - // If called w/o wholeDoc, doc is considered the original by default - if (wholeDoc === undefined) - wholeDoc = doc; - return {result: _.all(perKeySelectors, function (f) { - return f(doc, wholeDoc); - })}; - }; + return andCompiledDocumentSelectors(perKeySelectors); }; // Given a selector, return a function that takes one argument, a @@ -786,9 +778,26 @@ LocalCollection._compileSelector = function (selector, cursor) { EJSON.isBinary(selector)) throw new Error("Invalid selector: " + selector); - return compileDocumentSelector(selector, cursor); + // XXX get rid of second argument once _distance refactored + var s = compileDocumentSelector(selector, cursor); + return function (doc) { + return s(doc, doc); + }; }; var matchesNothingSelector = function (doc) { return {result: false}; }; + +var andCompiledDocumentSelectors = function (selectors) { + // XXX simplify to not involve 'arguments' once _distance is refactored + return function (/*doc, sometimes wholeDoc*/) { + var args = _.toArray(arguments); + // XXX take arrayIndex, etc into account + var result = _.all(selectors, function (f) { + // XXX once sub-selectors return structed thing, add '.result' here + return f.apply(null, args); + }); + return {result: result}; + }; +}; From 373413b5501eaba3f86681cc4ddb6fe663c7c0fd Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 13:02:37 -0800 Subject: [PATCH 039/124] Implement $comment It's a top-level logical operator that is ignored. --- packages/minimongo/minimongo_tests.js | 4 ++++ packages/minimongo/selector.js | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 8d96a90c93..94ea0146e2 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -952,6 +952,10 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({dogs: {$elemMatch: {name: /e/, age: 5}}}, {dogs: [{name: "Fido", age: 5}, {name: "Rex", age: 3}]}); + // $comment + match({a: 5, $comment: "asdf"}, {a: 5}); + nomatch({a: 6, $comment: "asdf"}, {a: 5}); + // XXX still needs tests: // - non-scalar arguments to $gt, $lt, etc }); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e64dde46ed..81632754ad 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -167,6 +167,12 @@ var LOGICAL_OPERATORS = { return function (doc) { return selectorValue.call(doc); }; + }, + + "$comment": function () { + return function () { + return true; + }; } }; From 2263cd49b438e2518b4dd7c9c32ef34ae5fc44db Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 13:08:19 -0800 Subject: [PATCH 040/124] Support "obj" name in $where --- packages/minimongo/minimongo_tests.js | 2 ++ packages/minimongo/selector.js | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 94ea0146e2..d939ac541c 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -908,7 +908,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { // $where match({$where: "this.a === 1"}, {a: 1}); + match({$where: "obj.a === 1"}, {a: 1}); nomatch({$where: "this.a !== 1"}, {a: 1}); + nomatch({$where: "obj.a !== 1"}, {a: 1}); nomatch({$where: "this.a === 1", a: 2}, {a: 1}); match({$where: "this.a === 1", b: 2}, {a: 1, b: 2}); match({$where: "this.a === 1 && this.b === 2"}, {a: 1, b: 2}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 81632754ad..42798fecde 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -162,10 +162,14 @@ var LOGICAL_OPERATORS = { "$where": function(selectorValue) { if (!(selectorValue instanceof Function)) { - selectorValue = Function("return " + selectorValue); + // XXX MongoDB seems to have more complex logic to decide where or or not + // to add "return"; not sure exactly what it is. + selectorValue = Function("obj", "return " + selectorValue); } return function (doc) { - return selectorValue.call(doc); + // We make the document available as both `this` and `obj`. + // XXX not sure what we should do if this throws + return selectorValue.call(doc, doc); }; }, From 6eac02e9bc018e0a04edec0829dfeeab9332faf2 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 14:39:01 -0800 Subject: [PATCH 041/124] refactor: logical operators use structured return --- packages/minimongo/selector.js | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 42798fecde..e29936bd3c 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -127,11 +127,7 @@ var LOGICAL_OPERATORS = { var subSelectorFunctions = _.map(subSelector, function (selector) { return compileDocumentSelector(selector, cursor); }); - return function (doc, wholeDoc) { - return _.all(subSelectorFunctions, function (f) { - return f(doc, wholeDoc).result; - }); - }; + return andCompiledDocumentSelectors(subSelectorFunctions); }, "$or": function(subSelector, operators, cursor) { @@ -141,9 +137,11 @@ var LOGICAL_OPERATORS = { return compileDocumentSelector(selector, cursor); }); return function (doc, wholeDoc) { - return _.any(subSelectorFunctions, function (f) { + var result = _.any(subSelectorFunctions, function (f) { return f(doc, wholeDoc).result; }); + // XXX arrayIndex! + return {result: result}; }; }, @@ -154,9 +152,12 @@ var LOGICAL_OPERATORS = { return compileDocumentSelector(selector, cursor); }); return function (doc, wholeDoc) { - return _.all(subSelectorFunctions, function (f) { + var result = _.all(subSelectorFunctions, function (f) { return !f(doc, wholeDoc).result; }); + // Never set arrayIndex, because we only match if nothing in particular + // "matched". + return {result: result}; }; }, @@ -169,13 +170,15 @@ var LOGICAL_OPERATORS = { return function (doc) { // We make the document available as both `this` and `obj`. // XXX not sure what we should do if this throws - return selectorValue.call(doc, doc); + return {result: selectorValue.call(doc, doc)}; }; }, + // This is just used as a comment in the query (in MongoDB, it also ends up in + // query logs); it has no effect on the actual selection. "$comment": function () { return function () { - return true; + return {result: true}; }; } }; @@ -722,6 +725,7 @@ var compileDocumentSelector = function (docSelector, cursor) { // this function), or $where. if (!_.has(LOGICAL_OPERATORS, key)) throw new Error("Unrecognized logical operator: " + key); + // XXX rename perKeySelectors perKeySelectors.push( LOGICAL_OPERATORS[key](subSelector, docSelector, cursor)); } else { @@ -745,13 +749,16 @@ var compileDocumentSelector = function (docSelector, cursor) { // XXX this still isn't right. consider {a: {$ne: 5, $gt: 6}}. the // $ne needs to use the "all" logic and the $gt needs the "any" // logic + // XXX add test for this brokenness var combiner = (subSelector && (subSelector.$not || subSelector.$ne || subSelector.$nin)) ? _.all : _.any; - return combiner(branchValues, function (val) { + var result = combiner(branchValues, function (val) { return valueSelectorFunc(val, wholeDoc); }); + // XXX rewrite value selectors to use structured return + return {result: result}; }); } }); @@ -803,10 +810,9 @@ var andCompiledDocumentSelectors = function (selectors) { // XXX simplify to not involve 'arguments' once _distance is refactored return function (/*doc, sometimes wholeDoc*/) { var args = _.toArray(arguments); - // XXX take arrayIndex, etc into account + // XXX arrayIndex! var result = _.all(selectors, function (f) { - // XXX once sub-selectors return structed thing, add '.result' here - return f.apply(null, args); + return f.apply(null, args).result; }); return {result: result}; }; From 92b2c80be5e033ed489981a0f824484072283d4a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 14:48:10 -0800 Subject: [PATCH 042/124] Drop unused arg from logical operator compilation --- packages/minimongo/selector.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e29936bd3c..b76acefb23 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -121,7 +121,7 @@ var compileValueSelector = function (valueSelector, selector, cursor) { // XXX can factor out common logic below var LOGICAL_OPERATORS = { - "$and": function(subSelector, operators, cursor) { + "$and": function(subSelector, cursor) { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { @@ -130,7 +130,7 @@ var LOGICAL_OPERATORS = { return andCompiledDocumentSelectors(subSelectorFunctions); }, - "$or": function(subSelector, operators, cursor) { + "$or": function(subSelector, cursor) { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { @@ -145,7 +145,7 @@ var LOGICAL_OPERATORS = { }; }, - "$nor": function(subSelector, operators, cursor) { + "$nor": function(subSelector, cursor) { if (!isArray(subSelector) || _.isEmpty(subSelector)) throw Error("$and/$or/$nor must be nonempty array"); var subSelectorFunctions = _.map(subSelector, function (selector) { @@ -727,7 +727,7 @@ var compileDocumentSelector = function (docSelector, cursor) { throw new Error("Unrecognized logical operator: " + key); // XXX rename perKeySelectors perKeySelectors.push( - LOGICAL_OPERATORS[key](subSelector, docSelector, cursor)); + LOGICAL_OPERATORS[key](subSelector, cursor)); } else { var lookUpByIndex = LocalCollection._makeLookupFunction(key); var valueSelectorFunc = From 621fa164cdf722a274fe2015243365a427d89611 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 26 Dec 2013 14:55:27 -0800 Subject: [PATCH 043/124] Refactor logical ops; better error for non-object --- packages/minimongo/minimongo_tests.js | 9 +++++++ packages/minimongo/selector.js | 39 ++++++++++++++------------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index d939ac541c..6dc14ddfa9 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -670,6 +670,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { test.throws(function () { match({$or: []}, {}); }); + test.throws(function () { + match({$or: [5]}, {}); + }); test.throws(function () { match({$or: []}, {a: 1}); }); @@ -751,6 +754,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { test.throws(function () { match({$nor: []}, {}); }); + test.throws(function () { + match({$nor: [5]}, {}); + }); test.throws(function () { match({$nor: []}, {a: 1}); }); @@ -829,6 +835,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { test.throws(function () { match({$and: []}, {}); }); + test.throws(function () { + match({$and: [5]}, {}); + }); test.throws(function () { match({$and: []}, {a: 1}); }); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index b76acefb23..a5c975d54e 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -119,25 +119,31 @@ var compileValueSelector = function (valueSelector, selector, cursor) { }; }; + +// XXX drop "cursor" when _distance is improved +var compileArrayOfDocumentSelectors = function (selectors, cursor) { + if (!isArray(selectors) || _.isEmpty(selectors)) + throw Error("$and/$or/$nor must be nonempty array"); + return _.map(selectors, function (subSelector) { + if (!isPlainObject(subSelector)) + throw Error("$or/$and/$nor entries need to be full objects"); + return compileDocumentSelector(subSelector, cursor); + }); +}; + + // XXX can factor out common logic below var LOGICAL_OPERATORS = { "$and": function(subSelector, cursor) { - if (!isArray(subSelector) || _.isEmpty(subSelector)) - throw Error("$and/$or/$nor must be nonempty array"); - var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); - }); - return andCompiledDocumentSelectors(subSelectorFunctions); + var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); + return andCompiledDocumentSelectors(selectors); }, "$or": function(subSelector, cursor) { - if (!isArray(subSelector) || _.isEmpty(subSelector)) - throw Error("$and/$or/$nor must be nonempty array"); - var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); - }); + var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); + // XXX remove wholeDoc later return function (doc, wholeDoc) { - var result = _.any(subSelectorFunctions, function (f) { + var result = _.any(selectors, function (f) { return f(doc, wholeDoc).result; }); // XXX arrayIndex! @@ -146,13 +152,10 @@ var LOGICAL_OPERATORS = { }, "$nor": function(subSelector, cursor) { - if (!isArray(subSelector) || _.isEmpty(subSelector)) - throw Error("$and/$or/$nor must be nonempty array"); - var subSelectorFunctions = _.map(subSelector, function (selector) { - return compileDocumentSelector(selector, cursor); - }); + var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); + // XXX remove wholeDoc later return function (doc, wholeDoc) { - var result = _.all(subSelectorFunctions, function (f) { + var result = _.all(selectors, function (f) { return !f(doc, wholeDoc).result; }); // Never set arrayIndex, because we only match if nothing in particular From 9a8d3c42c01e9bec09a058a46ee3925fc45c4996 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 15:24:01 -0800 Subject: [PATCH 044/124] Add expect_fail test for an XXX comment --- packages/minimongo/minimongo_tests.js | 14 ++++++++++++++ packages/minimongo/selector.js | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 6dc14ddfa9..774f9bf935 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -449,6 +449,20 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: {$ne: {x: 1}}}, {a: {x: 2}}); match({a: {$ne: {x: 1}}}, {a: {x: 1, y: 2}}); + // This query means: All 'a.b' must be non-5, and some 'a.b' must be >6. + // Current bad code parses this as "All 'a.b' must be both non-5 and >6", so + // it doesn't allow for some 'a.b' to be <5. + test.expect_fail(); + match({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 10}]}); + nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 4}]}); + nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 5}]}); + nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 10}, {b: 5}]}); + // Should work the same if the branch is at the bottom. + match({a: {$ne: 5, $gt: 6}}, {a: [2, 10]}); + nomatch({a: {$ne: 5, $gt: 6}}, {a: [2, 4]}); + nomatch({a: {$ne: 5, $gt: 6}}, {a: [2, 5]}); + nomatch({a: {$ne: 5, $gt: 6}}, {a: [10, 5]}); + // $in match({a: {$in: [1, 2, 3]}}, {a: 2}); nomatch({a: {$in: [1, 2, 3]}}, {a: 4}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index a5c975d54e..d7bb8fd7aa 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -751,7 +751,7 @@ var compileDocumentSelector = function (docSelector, cursor) { // https://jira.mongodb.org/browse/SERVER-8585 // XXX this still isn't right. consider {a: {$ne: 5, $gt: 6}}. the // $ne needs to use the "all" logic and the $gt needs the "any" - // logic + // logic. (There is an expect_fail test for this now.) // XXX add test for this brokenness var combiner = (subSelector && (subSelector.$not || subSelector.$ne || From ff83533efd1992028133ccedb1f3abebc3977cb9 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 15:29:43 -0800 Subject: [PATCH 045/124] compileValueSelector: remove unused argument --- packages/minimongo/selector.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d7bb8fd7aa..0e37e8fad6 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -54,7 +54,7 @@ var hasOperators = function(valueSelector) { return !!theseAreOperators; // {} has no operators }; -var compileValueSelector = function (valueSelector, selector, cursor) { +var compileValueSelector = function (valueSelector, cursor) { if (valueSelector == null) { // undefined or null return function (value) { return _anyIfArray(value, function (x) { @@ -352,7 +352,7 @@ var VALUE_OPERATORS = { }, "$not": function (operand, operators, cursor) { - var matcher = compileValueSelector(operand, operators, cursor); + var matcher = compileValueSelector(operand, cursor); return function (value, doc) { return !matcher(value, doc); }; @@ -734,7 +734,7 @@ var compileDocumentSelector = function (docSelector, cursor) { } else { var lookUpByIndex = LocalCollection._makeLookupFunction(key); var valueSelectorFunc = - compileValueSelector(subSelector, docSelector, cursor); + compileValueSelector(subSelector, cursor); perKeySelectors.push(function (doc, wholeDoc) { var branchValues = lookUpByIndex(doc); // We apply the selector to each "branched" value and return true if any From f444571930b7f7429af0fbb4a7a6130a4c60f768 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 15:48:05 -0800 Subject: [PATCH 046/124] Refactor compileValueSelector Pull out equality and regexp. Nothing else changes. --- packages/minimongo/minimongo_tests.js | 7 +++ packages/minimongo/selector.js | 74 +++++++++++++++++---------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 774f9bf935..19313cda82 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -318,6 +318,13 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: 12, b: 13}, {a: [11, 12, 13], b: [13, 14, 15]}); nomatch({a: 12, b: 13}, {a: [11, 12, 13], b: [14, 15]}); + // dates + var date1 = new Date; + var date2 = new Date(date1.getTime() + 1000); + match({a: date1}, {a: date1}); + nomatch({a: date1}, {a: date2}); + + // arrays match({a: [1,2]}, {a: [1, 2]}); match({a: [1,2]}, {a: [[1, 2]]}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 0e37e8fad6..e5931a3ae6 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -41,20 +41,44 @@ var _anyIfArrayPlus = function (x, f) { return isArray(x) && _.any(x, f); }; -var hasOperators = function(valueSelector) { +var isOperatorObject = function (valueSelector) { + if (!isPlainObject(valueSelector)) + return false; + var theseAreOperators = undefined; - for (var selKey in valueSelector) { + _.each(valueSelector, function (value, selKey) { var thisIsOperator = selKey.substr(0, 1) === '$'; if (theseAreOperators === undefined) { theseAreOperators = thisIsOperator; } else if (theseAreOperators !== thisIsOperator) { throw new Error("Inconsistent selector: " + valueSelector); } - } + }); return !!theseAreOperators; // {} has no operators }; + var compileValueSelector = function (valueSelector, cursor) { + if (valueSelector instanceof RegExp) + return regexpValueSelector(valueSelector); + else if (isOperatorObject(valueSelector)) + return operatorValueSelector(valueSelector, cursor); + else + return equalityValueSelector(valueSelector); +}; + +var regexpValueSelector = function (regexp) { + return function (value) { + if (value === undefined) + return false; + return _anyIfArray(value, function (x) { + return regexp.test(x); + }); + }; +}; + +// XXX should be able to replace most of this with EJSON.equals +var equalityValueSelector = function (valueSelector) { if (valueSelector == null) { // undefined or null return function (value) { return _anyIfArray(value, function (x) { @@ -72,15 +96,7 @@ var compileValueSelector = function (valueSelector, cursor) { }; } - if (valueSelector instanceof RegExp) { - return function (value) { - if (value === undefined) - return false; - return _anyIfArray(value, function (x) { - return valueSelector.test(x); - }); - }; - } + // XXX what about dates? // Arrays match either identical arrays or arrays that contain it as a value. if (isArray(valueSelector)) { @@ -93,22 +109,8 @@ var compileValueSelector = function (valueSelector, cursor) { }; } - // It's an object, but not an array or regexp. - if (hasOperators(valueSelector)) { - var operatorFunctions = []; - _.each(valueSelector, function (operand, operator) { - if (!_.has(VALUE_OPERATORS, operator)) - throw new Error("Unrecognized operator: " + operator); - // Special case for location operators - operatorFunctions.push(VALUE_OPERATORS[operator]( - operand, valueSelector, cursor)); - }); - return function (value, doc) { - return _.all(operatorFunctions, function (f) { - return f(value, doc); - }); - }; - } + if (isOperatorObject(valueSelector)) + throw Error("Can't create equalityValueSelector for operator object"); // It's a literal; compare value (or element of value array) directly to the // selector. @@ -119,6 +121,22 @@ var compileValueSelector = function (valueSelector, cursor) { }; }; +// XXX get rid of cursor when possible +var operatorValueSelector = function (valueSelector, cursor) { + var operatorFunctions = []; + _.each(valueSelector, function (operand, operator) { + if (!_.has(VALUE_OPERATORS, operator)) + throw new Error("Unrecognized operator: " + operator); + // Special case for location operators + operatorFunctions.push(VALUE_OPERATORS[operator]( + operand, valueSelector, cursor)); + }); + return function (value, doc) { + return _.all(operatorFunctions, function (f) { + return f(value, doc); + }); + }; +}; // XXX drop "cursor" when _distance is improved var compileArrayOfDocumentSelectors = function (selectors, cursor) { From f617d6e6bc83a16b2d0b8de2e1105a0c87e51209 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 16:09:50 -0800 Subject: [PATCH 047/124] Refactor equality to use new lookup function --- packages/minimongo/selector.js | 79 ++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 22 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e5931a3ae6..e20bad6ced 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -63,8 +63,10 @@ var compileValueSelector = function (valueSelector, cursor) { return regexpValueSelector(valueSelector); else if (isOperatorObject(valueSelector)) return operatorValueSelector(valueSelector, cursor); - else - return equalityValueSelector(valueSelector); + else { + return convertElementSelectorToBranchedSelector( + equalityElementSelector(valueSelector)); + } }; var regexpValueSelector = function (regexp) { @@ -77,47 +79,54 @@ var regexpValueSelector = function (regexp) { }; }; + +var convertElementSelectorToBranchedSelector = function (elementSelector) { + var f = function (branches, wholeDoc) { + // XXX in some cases don't do this + var expanded = expandArraysInBranches(branches); + var result = _.any(expanded, function (element) { + // XXX arrayIndex! need to save the winner here + return elementSelector(element.value, wholeDoc); + }); + return {result: result}; + }; + // XXX take this tag away soon + f._groksExtendedLookup_ = true; + return f; +}; + // XXX should be able to replace most of this with EJSON.equals -var equalityValueSelector = function (valueSelector) { - if (valueSelector == null) { // undefined or null +var equalityElementSelector = function (elementSelector) { + if (elementSelector == null) { // undefined or null return function (value) { - return _anyIfArray(value, function (x) { - return x == null; // undefined or null - }); + return value == null; // undefined or null }; } // Selector is a non-null primitive (and not an array or RegExp either). - if (!_.isObject(valueSelector)) { + if (!_.isObject(elementSelector)) { return function (value) { - return _anyIfArray(value, function (x) { - return x === valueSelector; - }); + return value === elementSelector; }; } // XXX what about dates? + // XXX technically should allow equality between dates and timestamps // Arrays match either identical arrays or arrays that contain it as a value. - if (isArray(valueSelector)) { + if (isArray(elementSelector)) { return function (value) { - if (!isArray(value)) - return false; - return _anyIfArrayPlus(value, function (x) { - return LocalCollection._f._equal(valueSelector, x); - }); + return LocalCollection._f._equal(elementSelector, value); }; } - if (isOperatorObject(valueSelector)) + if (isOperatorObject(elementSelector)) throw Error("Can't create equalityValueSelector for operator object"); // It's a literal; compare value (or element of value array) directly to the // selector. return function (value) { - return _anyIfArray(value, function (x) { - return LocalCollection._f._equal(valueSelector, x); - }); + return LocalCollection._f._equal(elementSelector, value); }; }; @@ -737,6 +746,25 @@ LocalCollection._makeLookupFunction = function (key) { }; }; +var expandArraysInBranches = function (branches) { + var branchesOut = []; + _.each(branches, function (branch) { + branchesOut.push({ + value: branch.value, + arrayIndex: branch.arrayIndex + }); + if (isArray(branch.value) && !branch.dontIterate) { + _.each(branch.value, function (leaf, i) { + branchesOut.push({ + value: leaf, + arrayIndex: branch.arrayIndex === undefined ? i : branch.arrayIndex + }); + }); + } + }); + return branchesOut; +}; + // The main compilation function for a given selector. var compileDocumentSelector = function (docSelector, cursor) { var perKeySelectors = []; @@ -750,11 +778,18 @@ var compileDocumentSelector = function (docSelector, cursor) { perKeySelectors.push( LOGICAL_OPERATORS[key](subSelector, cursor)); } else { - var lookUpByIndex = LocalCollection._makeLookupFunction(key); + var lookUpByIndex = LocalCollection._makeLookupFunction2(key); var valueSelectorFunc = compileValueSelector(subSelector, cursor); perKeySelectors.push(function (doc, wholeDoc) { var branchValues = lookUpByIndex(doc); + + if (valueSelectorFunc._groksExtendedLookup_) { + return valueSelectorFunc(branchValues, wholeDoc); + } + + // XXX get rid of this all later + branchValues = _.pluck(branchValues, 'value'); // We apply the selector to each "branched" value and return true if any // match. However, for "negative" selectors like $ne or $not we actually // require *all* elements to match. From 61c51387a8cc01cab76ecca1b5029a3a944ec208 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 16:12:28 -0800 Subject: [PATCH 048/124] Major simplification to equality code --- packages/minimongo/selector.js | 29 ++++++----------------------- 1 file changed, 6 insertions(+), 23 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e20bad6ced..e86cc372fa 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -95,36 +95,19 @@ var convertElementSelectorToBranchedSelector = function (elementSelector) { return f; }; -// XXX should be able to replace most of this with EJSON.equals var equalityElementSelector = function (elementSelector) { + if (isOperatorObject(elementSelector)) + throw Error("Can't create equalityValueSelector for operator object"); + + // Special-case: null and undefined are equal (if you got undefined in there + // somewhere, or if you got it due to some branch being non-existent in the + // weird special case), even though they aren't with EJSON.equals. if (elementSelector == null) { // undefined or null return function (value) { return value == null; // undefined or null }; } - // Selector is a non-null primitive (and not an array or RegExp either). - if (!_.isObject(elementSelector)) { - return function (value) { - return value === elementSelector; - }; - } - - // XXX what about dates? - // XXX technically should allow equality between dates and timestamps - - // Arrays match either identical arrays or arrays that contain it as a value. - if (isArray(elementSelector)) { - return function (value) { - return LocalCollection._f._equal(elementSelector, value); - }; - } - - if (isOperatorObject(elementSelector)) - throw Error("Can't create equalityValueSelector for operator object"); - - // It's a literal; compare value (or element of value array) directly to the - // selector. return function (value) { return LocalCollection._f._equal(elementSelector, value); }; From ef92b6c5114d99a5b8d0ce86f1775eb36544f5a4 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 16:13:54 -0800 Subject: [PATCH 049/124] Two expect_fails now pass --- packages/minimongo/minimongo_tests.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 19313cda82..756dbef81b 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -642,9 +642,7 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({'a.1': [8, 9]}, {a: [7, [8, 9]]}); nomatch({'a.1': 6}, {a: [[6, 7], [8, 9]]}); nomatch({'a.1': 7}, {a: [[6, 7], [8, 9]]}); - test.expect_fail(); // XXX fix by observing dontIterate nomatch({'a.1': 8}, {a: [[6, 7], [8, 9]]}); - test.expect_fail(); // XXX fix by observing dontIterate nomatch({'a.1': 9}, {a: [[6, 7], [8, 9]]}); match({"a.1": 2}, {a: [0, {1: 2}, 3]}); match({"a.1": {1: 2}}, {a: [0, {1: 2}, 3]}); From ce54fa1d3b7000a3cac05963e0fe310dbec996a4 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 16:18:10 -0800 Subject: [PATCH 050/124] Fix incorrect expect_fail tests On further verification against MongoDB (and common sense), these young expect_fail tests were actually correctly failing. --- packages/minimongo/minimongo_tests.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 756dbef81b..5c511b24d6 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -631,10 +631,8 @@ Tinytest.add("minimongo - selector_compiler", function (test) { var big = {a: [{b: 1}, 2, {}, {b: [3, 4]}]}; match({"a.b": 1}, big); match({"a.b": [3, 4]}, big); - test.expect_fail(); // XXX fix by observing dontIterate - nomatch({"a.b": 3}, big); - test.expect_fail(); // XXX fix by observing dontIterate - nomatch({"a.b": 4}, big); + match({"a.b": 3}, big); + match({"a.b": 4}, big); match({"a.b": null}, big); // matches on slot 2 match({'a.1': 8}, {a: [7, 8, 9]}); nomatch({'a.1': 7}, {a: [7, 8, 9]}); From 75f1a6dc01bc1ab80f94232e7ed60d09b6e0a99c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 16:51:22 -0800 Subject: [PATCH 051/124] Implement $lt in the new model Fix an invalid test Don't run some $not/$lt tests, because $not is broken for now --- packages/minimongo/minimongo_tests.js | 11 ++- packages/minimongo/selector.js | 102 +++++++++++++++++++++----- 2 files changed, 89 insertions(+), 24 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 5c511b24d6..c7c399ff50 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -380,7 +380,7 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$lt: 10}}, {a: [11, 12]}); // (there's a full suite of ordering test elsewhere) - match({a: {$lt: "null"}}, {a: null}); // tested against mongodb + nomatch({a: {$lt: "null"}}, {a: null}); match({a: {$lt: {x: [2, 3, 4]}}}, {a: {x: [1, 3, 4]}}); match({a: {$gt: {x: [2, 3, 4]}}}, {a: {x: [3, 3, 4]}}); nomatch({a: {$gt: {x: [2, 3, 4]}}}, {a: {x: [1, 3, 4]}}); @@ -928,9 +928,12 @@ Tinytest.add("minimongo - selector_compiler", function (test) { // $and and $not match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); - nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); - match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); - nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); + // XXX fix immediately + if (false) { + nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); + match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); + nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); + } // $where match({$where: "this.a === 1"}, {a: 1}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index e86cc372fa..3ccdb4a4e7 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -81,7 +81,7 @@ var regexpValueSelector = function (regexp) { var convertElementSelectorToBranchedSelector = function (elementSelector) { - var f = function (branches, wholeDoc) { + return GROK(function (branches, wholeDoc) { // XXX in some cases don't do this var expanded = expandArraysInBranches(branches); var result = _.any(expanded, function (element) { @@ -89,10 +89,7 @@ var convertElementSelectorToBranchedSelector = function (elementSelector) { return elementSelector(element.value, wholeDoc); }); return {result: result}; - }; - // XXX take this tag away soon - f._groksExtendedLookup_ = true; - return f; + }); }; var equalityElementSelector = function (elementSelector) { @@ -113,21 +110,45 @@ var equalityElementSelector = function (elementSelector) { }; }; +// XXX this won't be necessary +var GROK = function (f) { + f._groksExtendedLookup_ = true; + return f; +}; + // XXX get rid of cursor when possible var operatorValueSelector = function (valueSelector, cursor) { + // XXX kill this soon + if (!_.all(valueSelector, function (operand, operator) { + return _.has(ELEMENT_OPERATORS, operator); + })) { + return operatorValueSelectorLegacy(valueSelector, cursor); + } + + // Each valueSelector works separately on the various branches. So one + // operator can match one branch and another can match another branch. This + // is OK. + var operatorFunctions = []; _.each(valueSelector, function (operand, operator) { - if (!_.has(VALUE_OPERATORS, operator)) + if (!_.has(ELEMENT_OPERATORS, operator)) throw new Error("Unrecognized operator: " + operator); - // Special case for location operators - operatorFunctions.push(VALUE_OPERATORS[operator]( - operand, valueSelector, cursor)); + // XXX justify three arguments + operatorFunctions.push( + convertElementSelectorToBranchedSelector( + ELEMENT_OPERATORS[operator]( + operand, valueSelector, cursor))); }); - return function (value, doc) { - return _.all(operatorFunctions, function (f) { - return f(value, doc); + // NB: this is very similar to andCompiledDocumentSelectors but that one + // "assumes" the first arg is a doc and here it "assumes" it's a branched + // value list. The code is identical for now, though. + return GROK(function (branches, doc) { + // XXX arrayIndex! + var result = _.all(operatorFunctions, function (f) { + return f(branches, doc).result; }); - }; + return {result: result}; + }); }; // XXX drop "cursor" when _distance is improved @@ -144,12 +165,12 @@ var compileArrayOfDocumentSelectors = function (selectors, cursor) { // XXX can factor out common logic below var LOGICAL_OPERATORS = { - "$and": function(subSelector, cursor) { + $and: function(subSelector, cursor) { var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); return andCompiledDocumentSelectors(selectors); }, - "$or": function(subSelector, cursor) { + $or: function(subSelector, cursor) { var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); // XXX remove wholeDoc later return function (doc, wholeDoc) { @@ -161,7 +182,7 @@ var LOGICAL_OPERATORS = { }; }, - "$nor": function(subSelector, cursor) { + $nor: function(subSelector, cursor) { var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); // XXX remove wholeDoc later return function (doc, wholeDoc) { @@ -174,7 +195,7 @@ var LOGICAL_OPERATORS = { }; }, - "$where": function(selectorValue) { + $where: function(selectorValue) { if (!(selectorValue instanceof Function)) { // XXX MongoDB seems to have more complex logic to decide where or or not // to add "return"; not sure exactly what it is. @@ -189,13 +210,14 @@ var LOGICAL_OPERATORS = { // This is just used as a comment in the query (in MongoDB, it also ends up in // query logs); it has no effect on the actual selection. - "$comment": function () { + $comment: function () { return function () { return {result: true}; }; } }; +// XXX redoc // Each value operator is a function with args: // - operand - Anything // - operators - Object - operators on the same level (neighbours) @@ -203,7 +225,31 @@ var LOGICAL_OPERATORS = { // returns a function with args: // - value - a value the operator is tested against // - doc - the whole document tested in this query -var VALUE_OPERATORS = { +var ELEMENT_OPERATORS = { + $lt: function (operand) { + // Arrays never compare with non-arrays (except for equality checks). + if (isArray(operand)) { + return function () { + return false; + }; + } + if (operand === undefined) + operand = null; + var operandType = LocalCollection._f._type(operand); + + return function (value) { + if (value === undefined) + value = null; + // Comparisons are never true among things of different type (except null + // vs undefined). + if (LocalCollection._f._type(value) !== operandType) + return false; + return LocalCollection._f._cmp(value, operand) < 0; + }; + } +}; + +var LEGACY_VALUE_OPERATORS = { "$in": function (operand) { if (!isArray(operand)) throw new Error("Argument to $in must be array"); @@ -273,7 +319,7 @@ var VALUE_OPERATORS = { "$nin": function (operand) { if (!isArray(operand)) throw new Error("Argument to $nin must be array"); - var inFunction = VALUE_OPERATORS.$in(operand); + var inFunction = LEGACY_VALUE_OPERATORS.$in(operand); return function (value, doc) { // Field doesn't exist, so it's not-in operand if (value === undefined) @@ -427,6 +473,22 @@ var VALUE_OPERATORS = { } }; +var operatorValueSelectorLegacy = function (valueSelector, cursor) { + var operatorFunctions = []; + _.each(valueSelector, function (operand, operator) { + if (!_.has(LEGACY_VALUE_OPERATORS, operator)) + throw new Error("Unrecognized legacy operator: " + operator); + // Special case for location operators + operatorFunctions.push(LEGACY_VALUE_OPERATORS[operator]( + operand, valueSelector, cursor)); + }); + return function (value, doc) { + return _.all(operatorFunctions, function (f) { + return f(value, doc); + }); + }; +}; + // helpers used by compiled selector code LocalCollection._f = { // XXX for _all and _in, consider building 'inquery' at compile time.. From bb85b79a5ed298004f2c575b4bf88ec958f668b7 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 17:08:15 -0800 Subject: [PATCH 052/124] Refactor regexp match to use new style Also implement regexp-vs-regexp matching, for completeness sake --- packages/minimongo/minimongo_tests.js | 26 +++++++++++++++++++++----- packages/minimongo/selector.js | 17 +++++++++++------ 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index c7c399ff50..6f3d19e5bc 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -284,6 +284,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { // XXX blog post about what I learned while writing these tests (weird // mongo edge cases) + // XXX fix soon + var NOT_WORKS_WELL = false; + // empty selectors match({}, {}); match({}, {a: 12}); @@ -589,6 +592,17 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: /a/}, {a: ['dog', 'cat']}); nomatch({a: /a/}, {a: ['dog', 'puppy']}); + // we don't support regexps in minimongo very well (eg, there's no EJSON + // encoding so it won't go over the wire), but run these tests anyway + match({a: /a/}, {a: /a/}); + match({a: /a/}, {a: ['x', /a/]}); + nomatch({a: /a/}, {a: /a/i}); + nomatch({a: /a/m}, {a: /a/}); + nomatch({a: /a/}, {a: /b/}); + nomatch({a: /5/}, {a: 5}); + nomatch({a: /t/}, {a: true}); + match({a: /m/i}, {a: ['x', 'xM']}); + test.throws(function () { match({a: {$regex: /a/, $options: 'x'}}, {a: 'cat'}); }); @@ -608,10 +622,12 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({x: {$not: {$gt: 7}}}, {x: [2, 3, 4, 10]}); nomatch({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}, {y:10}]}); - match({x: {$not: /a/}}, {x: "dog"}); - nomatch({x: {$not: /a/}}, {x: "cat"}); - match({x: {$not: /a/}}, {x: ["dog", "puppy"]}); - nomatch({x: {$not: /a/}}, {x: ["kitten", "cat"]}); + if (NOT_WORKS_WELL) { + match({x: {$not: /a/}}, {x: "dog"}); + nomatch({x: {$not: /a/}}, {x: "cat"}); + match({x: {$not: /a/}}, {x: ["dog", "puppy"]}); + nomatch({x: {$not: /a/}}, {x: ["kitten", "cat"]}); + } // dotted keypaths: bare values match({"a.b": 1}, {a: {b: 1}}); @@ -929,7 +945,7 @@ Tinytest.add("minimongo - selector_compiler", function (test) { // $and and $not match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); // XXX fix immediately - if (false) { + if (NOT_WORKS_WELL) { nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 3ccdb4a4e7..83e964471e 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -60,7 +60,8 @@ var isOperatorObject = function (valueSelector) { var compileValueSelector = function (valueSelector, cursor) { if (valueSelector instanceof RegExp) - return regexpValueSelector(valueSelector); + return convertElementSelectorToBranchedSelector( + regexpElementSelector(valueSelector)); else if (isOperatorObject(valueSelector)) return operatorValueSelector(valueSelector, cursor); else { @@ -69,13 +70,17 @@ var compileValueSelector = function (valueSelector, cursor) { } }; -var regexpValueSelector = function (regexp) { +var regexpElementSelector = function (regexp) { return function (value) { - if (value === undefined) + if (value instanceof RegExp) { + // Comparing two regexps means seeing if the regexps are identical + // (really!). Underscore knows how. + return _.isEqual(value, regexp); + } + // Regexps only work against strings. + if (typeof value !== 'string') return false; - return _anyIfArray(value, function (x) { - return regexp.test(x); - }); + return regexp.test(value); }; }; From 366e5a5f68c6f3aaeea6ccd3ce4a5c4dc2ad9c7d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 22:06:24 -0800 Subject: [PATCH 053/124] Move $gt/$gte/$lte to the new model --- packages/minimongo/minimongo_tests.js | 24 +++++++------- packages/minimongo/selector.js | 45 +++++++++++++++++++-------- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 6f3d19e5bc..a182dc1814 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -611,18 +611,18 @@ Tinytest.add("minimongo - selector_compiler", function (test) { }); // $not - match({x: {$not: {$gt: 7}}}, {x: 6}); - nomatch({x: {$not: {$gt: 7}}}, {x: 8}); - match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 11}); - nomatch({x: {$not: {$lt: 10, $gt: 7}}}, {x: 9}); - match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 6}); - - match({x: {$not: {$gt: 7}}}, {x: [2, 3, 4]}); - match({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}]}); - nomatch({x: {$not: {$gt: 7}}}, {x: [2, 3, 4, 10]}); - nomatch({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}, {y:10}]}); - if (NOT_WORKS_WELL) { + match({x: {$not: {$gt: 7}}}, {x: 6}); + nomatch({x: {$not: {$gt: 7}}}, {x: 8}); + match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 11}); + nomatch({x: {$not: {$lt: 10, $gt: 7}}}, {x: 9}); + match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 6}); + + match({x: {$not: {$gt: 7}}}, {x: [2, 3, 4]}); + match({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}]}); + nomatch({x: {$not: {$gt: 7}}}, {x: [2, 3, 4, 10]}); + nomatch({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}, {y:10}]}); + match({x: {$not: /a/}}, {x: "dog"}); nomatch({x: {$not: /a/}}, {x: "cat"}); match({x: {$not: /a/}}, {x: ["dog", "puppy"]}); @@ -943,9 +943,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({$and: [{a: {$ne: 1}}, {a: {$ne: 3}}]}, {a: 2}); // $and and $not - match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); // XXX fix immediately if (NOT_WORKS_WELL) { + match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 83e964471e..ba5e7558fd 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -222,24 +222,20 @@ var LOGICAL_OPERATORS = { } }; -// XXX redoc -// Each value operator is a function with args: -// - operand - Anything -// - operators - Object - operators on the same level (neighbours) -// - cursor - Object - original cursor -// returns a function with args: -// - value - a value the operator is tested against -// - doc - the whole document tested in this query -var ELEMENT_OPERATORS = { - $lt: function (operand) { - // Arrays never compare with non-arrays (except for equality checks). +var makeInequality = function (cmpValueComparator) { + return function (operand) { + // Arrays never compare false with non-arrays for any inequality. if (isArray(operand)) { return function () { return false; }; } + + // Special case: consider undefined and null the same (so true with + // $gte/$lte). if (operand === undefined) operand = null; + var operandType = LocalCollection._f._type(operand); return function (value) { @@ -249,9 +245,32 @@ var ELEMENT_OPERATORS = { // vs undefined). if (LocalCollection._f._type(value) !== operandType) return false; - return LocalCollection._f._cmp(value, operand) < 0; + return cmpValueComparator(LocalCollection._f._cmp(value, operand)) }; - } + }; +}; + +// XXX redoc +// Each value operator is a function with args: +// - operand - Anything +// - operators - Object - operators on the same level (neighbours) +// - cursor - Object - original cursor +// returns a function with args: +// - value - a value the operator is tested against +// - doc - the whole document tested in this query +var ELEMENT_OPERATORS = { + $lt: makeInequality(function (cmpValue) { + return cmpValue < 0; + }), + $gt: makeInequality(function (cmpValue) { + return cmpValue > 0; + }), + $lte: makeInequality(function (cmpValue) { + return cmpValue <= 0; + }), + $gte: makeInequality(function (cmpValue) { + return cmpValue >= 0; + }) }; var LEGACY_VALUE_OPERATORS = { From 20e1fba38c9f9fd96b7b9222f81ed2d2ea137f8a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 22:14:59 -0800 Subject: [PATCH 054/124] Move $mod to the new model --- packages/minimongo/minimongo_tests.js | 46 ++++++++++++++++++--------- packages/minimongo/selector.js | 15 ++++++++- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index a182dc1814..a87d98337b 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -441,6 +441,18 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$mod: [10, 1]}}, {a: 12}); match({a: {$mod: [10, 1]}}, {a: [10, 11, 12]}); nomatch({a: {$mod: [10, 1]}}, {a: [10, 12]}); + _.each([ + 5, + [10], + [10, 1, 2], + "foo", + {bar: 1}, + [] + ], function (badMod) { + test.throws(function () { + match({a: {$mod: badMod}}, {a: 11}); + }); + }); // $ne match({a: {$ne: 1}}, {a: 2}); @@ -774,14 +786,16 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({$or: [{a: {$ne: 1}}, {b: {$ne: 2}}]}, {a: 1, b: 2}); // $or and $not - match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {}); - nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); - nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); - // this is possibly an open-ended task, so we stop here ... + if (NOT_WORKS_WELL) { + match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {}); + nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); + nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); + // this is possibly an open-ended task, so we stop here ... + } // $nor test.throws(function () { @@ -855,13 +869,15 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({$nor: [{a: {$ne: 1}}, {b: {$ne: 2}}]}, {a: 1, b: 2}); // $nor and $not - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {}); - match({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); - match({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); + if (NOT_WORKS_WELL) { + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {}); + match({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); + match({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); + } // $and diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index ba5e7558fd..0f774a1f66 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -270,7 +270,20 @@ var ELEMENT_OPERATORS = { }), $gte: makeInequality(function (cmpValue) { return cmpValue >= 0; - }) + }), + $mod: function (operand) { + if (!(isArray(operand) && operand.length === 2 + && typeof(operand[0]) === 'number' + && typeof(operand[1]) === 'number')) { + throw Error("argument to $mod must be an array of two numbers"); + } + // XXX could require to be ints or round or something + var divisor = operand[0]; + var remainder = operand[1]; + return function (value) { + return typeof value === 'number' && value % divisor === remainder; + }; + } }; var LEGACY_VALUE_OPERATORS = { From d243d30768bf18fafa7e93c951b3e2b73711d1ce Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 22:17:44 -0800 Subject: [PATCH 055/124] Move $not to the new model --- packages/minimongo/minimongo_tests.js | 76 +++++++++++---------------- packages/minimongo/selector.js | 36 ++++++++++--- 2 files changed, 61 insertions(+), 51 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index a87d98337b..4f2fa6a41b 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -284,9 +284,6 @@ Tinytest.add("minimongo - selector_compiler", function (test) { // XXX blog post about what I learned while writing these tests (weird // mongo edge cases) - // XXX fix soon - var NOT_WORKS_WELL = false; - // empty selectors match({}, {}); match({}, {a: 12}); @@ -623,23 +620,21 @@ Tinytest.add("minimongo - selector_compiler", function (test) { }); // $not - if (NOT_WORKS_WELL) { - match({x: {$not: {$gt: 7}}}, {x: 6}); - nomatch({x: {$not: {$gt: 7}}}, {x: 8}); - match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 11}); - nomatch({x: {$not: {$lt: 10, $gt: 7}}}, {x: 9}); - match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 6}); + match({x: {$not: {$gt: 7}}}, {x: 6}); + nomatch({x: {$not: {$gt: 7}}}, {x: 8}); + match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 11}); + nomatch({x: {$not: {$lt: 10, $gt: 7}}}, {x: 9}); + match({x: {$not: {$lt: 10, $gt: 7}}}, {x: 6}); - match({x: {$not: {$gt: 7}}}, {x: [2, 3, 4]}); - match({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}]}); - nomatch({x: {$not: {$gt: 7}}}, {x: [2, 3, 4, 10]}); - nomatch({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}, {y:10}]}); + match({x: {$not: {$gt: 7}}}, {x: [2, 3, 4]}); + match({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}]}); + nomatch({x: {$not: {$gt: 7}}}, {x: [2, 3, 4, 10]}); + nomatch({'x.y': {$not: {$gt: 7}}}, {x: [{y:2}, {y:3}, {y:4}, {y:10}]}); - match({x: {$not: /a/}}, {x: "dog"}); - nomatch({x: {$not: /a/}}, {x: "cat"}); - match({x: {$not: /a/}}, {x: ["dog", "puppy"]}); - nomatch({x: {$not: /a/}}, {x: ["kitten", "cat"]}); - } + match({x: {$not: /a/}}, {x: "dog"}); + nomatch({x: {$not: /a/}}, {x: "cat"}); + match({x: {$not: /a/}}, {x: ["dog", "puppy"]}); + nomatch({x: {$not: /a/}}, {x: ["kitten", "cat"]}); // dotted keypaths: bare values match({"a.b": 1}, {a: {b: 1}}); @@ -786,16 +781,14 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({$or: [{a: {$ne: 1}}, {b: {$ne: 2}}]}, {a: 1, b: 2}); // $or and $not - if (NOT_WORKS_WELL) { - match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {}); - nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); - nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); - match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); - // this is possibly an open-ended task, so we stop here ... - } + match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {}); + nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); + nomatch({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); + match({$or: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); + // this is possibly an open-ended task, so we stop here ... // $nor test.throws(function () { @@ -869,15 +862,13 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({$nor: [{a: {$ne: 1}}, {b: {$ne: 2}}]}, {a: 1, b: 2}); // $nor and $not - if (NOT_WORKS_WELL) { - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {}); - match({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); - match({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); - nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); - } + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {}); + match({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 1}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}]}, {a: 2}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$not: {$mod: [10, 2]}}}]}, {a: 1}); + match({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 1}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 2}); + nomatch({$nor: [{a: {$not: {$mod: [10, 1]}}}, {a: {$mod: [10, 2]}}]}, {a: 3}); // $and @@ -959,13 +950,10 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({$and: [{a: {$ne: 1}}, {a: {$ne: 3}}]}, {a: 2}); // $and and $not - // XXX fix immediately - if (NOT_WORKS_WELL) { - match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); - nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); - match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); - nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); - } + match({$and: [{a: {$not: {$gt: 2}}}]}, {a: 1}); + nomatch({$and: [{a: {$not: {$lt: 2}}}]}, {a: 1}); + match({$and: [{a: {$not: {$lt: 0}}}, {a: {$not: {$gt: 2}}}]}, {a: 1}); + nomatch({$and: [{a: {$not: {$lt: 2}}}, {a: {$not: {$gt: 0}}}]}, {a: 1}); // $where match({$where: "this.a === 1"}, {a: 1}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 0f774a1f66..50c7456247 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -125,7 +125,8 @@ var GROK = function (f) { var operatorValueSelector = function (valueSelector, cursor) { // XXX kill this soon if (!_.all(valueSelector, function (operand, operator) { - return _.has(ELEMENT_OPERATORS, operator); + return _.has(ELEMENT_OPERATORS, operator) || + _.has(VALUE_OPERATORS, operator); })) { return operatorValueSelectorLegacy(valueSelector, cursor); } @@ -136,13 +137,18 @@ var operatorValueSelector = function (valueSelector, cursor) { var operatorFunctions = []; _.each(valueSelector, function (operand, operator) { - if (!_.has(ELEMENT_OPERATORS, operator)) + if (_.has(VALUE_OPERATORS, operator)) { + operatorFunctions.push( + VALUE_OPERATORS[operator](operand, valueSelector, cursor)); + } else if (_.has(ELEMENT_OPERATORS, operator)) { + // XXX justify three arguments + operatorFunctions.push( + convertElementSelectorToBranchedSelector( + ELEMENT_OPERATORS[operator]( + operand, valueSelector, cursor))); + } else { throw new Error("Unrecognized operator: " + operator); - // XXX justify three arguments - operatorFunctions.push( - convertElementSelectorToBranchedSelector( - ELEMENT_OPERATORS[operator]( - operand, valueSelector, cursor))); + } }); // NB: this is very similar to andCompiledDocumentSelectors but that one // "assumes" the first arg is a doc and here it "assumes" it's a branched @@ -222,6 +228,22 @@ var LOGICAL_OPERATORS = { } }; +// XXX doc +var VALUE_OPERATORS = { + $not: function (operand, operator, cursor) { + var innerBranchedSelector = compileValueSelector(operand, cursor); + // Note that this implicitly "deMorganizes" the wrapped function. ie, it + // means that ALL branch values need to fail to match innerBranchedSelector. + return function (branchValues, doc) { + var invertMe = innerBranchedSelector(branchValues, doc); + // We explicitly choose to strip arrayIndex here: it doesn't make sense to + // say "update the array element that does not match something", at least + // in mongo-land. + return {result: !invertMe.result}; + }; + } +}; + var makeInequality = function (cmpValueComparator) { return function (operand) { // Arrays never compare false with non-arrays for any inequality. From fdd16c6530615eef0513788dbdf769663dd606de Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 22:25:25 -0800 Subject: [PATCH 056/124] Move $ne to the new model. Fixes an expect_fail --- packages/minimongo/minimongo_tests.js | 3 --- packages/minimongo/selector.js | 27 +++++++++++++++++---------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 4f2fa6a41b..2b0cd57b90 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -469,9 +469,6 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: {$ne: {x: 1}}}, {a: {x: 1, y: 2}}); // This query means: All 'a.b' must be non-5, and some 'a.b' must be >6. - // Current bad code parses this as "All 'a.b' must be both non-5 and >6", so - // it doesn't allow for some 'a.b' to be <5. - test.expect_fail(); match({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 10}]}); nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 4}]}); nomatch({'a.b': {$ne: 5, $gt: 6}}, {a: [{b: 2}, {b: 5}]}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 50c7456247..141d0b53c7 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -228,19 +228,26 @@ var LOGICAL_OPERATORS = { } }; +var invertBranchedSelector = function (branchedSelector) { + // Note that this implicitly "deMorganizes" the wrapped function. ie, it + // means that ALL branch values need to fail to match innerBranchedSelector. + return function (branchValues, doc) { + var invertMe = branchedSelector(branchValues, doc); + // We explicitly choose to strip arrayIndex here: it doesn't make sense to + // say "update the array element that does not match something", at least + // in mongo-land. + return {result: !invertMe.result}; + }; +}; + // XXX doc var VALUE_OPERATORS = { $not: function (operand, operator, cursor) { - var innerBranchedSelector = compileValueSelector(operand, cursor); - // Note that this implicitly "deMorganizes" the wrapped function. ie, it - // means that ALL branch values need to fail to match innerBranchedSelector. - return function (branchValues, doc) { - var invertMe = innerBranchedSelector(branchValues, doc); - // We explicitly choose to strip arrayIndex here: it doesn't make sense to - // say "update the array element that does not match something", at least - // in mongo-land. - return {result: !invertMe.result}; - }; + return invertBranchedSelector(compileValueSelector(operand, cursor)); + }, + $ne: function (operand) { + return invertBranchedSelector(convertElementSelectorToBranchedSelector( + equalityElementSelector(operand))); } }; From 90e94c8af11efe38ce96cd6ba8ac0e8178107813 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 27 Dec 2013 22:51:20 -0800 Subject: [PATCH 057/124] Move $in to the new model Implement regexp options for $in. Fixes #1707. Remaining: - $nin - $exists - $size - $type - $regex/$option - $all - $elemMatch - $near/$maxDistance --- packages/minimongo/minimongo_tests.js | 17 +++++++++++++++++ packages/minimongo/selector.js | 23 +++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 2b0cd57b90..86488bb6b8 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -492,6 +492,23 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: {$in: [1, 2, 3]}}, {a: [4, 2]}); nomatch({a: {$in: [1, 2, 3]}}, {a: [4]}); + match({a: {$in: ['x', /foo/i]}}, {a: 'x'}); + match({a: {$in: ['x', /foo/i]}}, {a: 'fOo'}); + match({a: {$in: ['x', /foo/i]}}, {a: ['f', 'fOo']}); + nomatch({a: {$in: ['x', /foo/i]}}, {a: ['f', 'fOx']}); + + match({a: {$in: [1, null]}}, {}); + match({'a.b': {$in: [1, null]}}, {}); + match({'a.b': {$in: [1, null]}}, {a: {}}); + match({'a.b': {$in: [1, null]}}, {a: {b: null}}); + nomatch({'a.b': {$in: [1, null]}}, {a: {b: 5}}); + nomatch({'a.b': {$in: [1]}}, {a: {b: null}}); + nomatch({'a.b': {$in: [1]}}, {a: {}}); + nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}]}); + match({'a.b': {$in: [1, null]}}, {a: [{b: 5}, {}]}); + nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}, []]}); + nomatch({'a.b': {$in: [1, null]}}, {a: [{b: 5}, 5]}); + // $nin nomatch({a: {$nin: [1, 2, 3]}}, {a: 2}); match({a: {$nin: [1, 2, 3]}}, {a: 4}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 141d0b53c7..fdf1fec1d8 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -312,6 +312,29 @@ var ELEMENT_OPERATORS = { return function (value) { return typeof value === 'number' && value % divisor === remainder; }; + }, + $in: function (operand) { + if (!isArray(operand)) + throw Error("$in needs an array"); + + var elementSelectors = []; + _.each(operand, function (option) { + if (option instanceof RegExp) + elementSelectors.push(regexpElementSelector(option)); + else if (isOperatorObject(option)) + throw Error("cannot nest $ under $in"); + else + elementSelectors.push(equalityElementSelector(option)); + }); + + return function (value) { + // Allow {a: {$in: [null]}} to match when 'a' does not exist. + if (value === undefined) + value = null; + return _.any(elementSelectors, function (e) { + return e(value); + }); + }; } }; From d863883e9fcc33eaf686c606c6d3d2b6e1c222d1 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sat, 28 Dec 2013 22:22:38 -0800 Subject: [PATCH 058/124] Move $nin to the new model This implement regexp options for $in Remaining: - $exists - $size - $type - $regex/$option - $all - $elemMatch - $near/$maxDistance --- packages/minimongo/minimongo_tests.js | 17 +++++++++++++++++ packages/minimongo/selector.js | 6 +++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 86488bb6b8..6ae30c5437 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -524,6 +524,23 @@ Tinytest.add("minimongo - selector_compiler", function (test) { match({a: {$nin: [1, 2, 3]}}, {a: [4]}); match({'a.b': {$nin: [1, 2, 3]}}, {a: [{b:4}]}); + nomatch({a: {$nin: ['x', /foo/i]}}, {a: 'x'}); + nomatch({a: {$nin: ['x', /foo/i]}}, {a: 'fOo'}); + nomatch({a: {$nin: ['x', /foo/i]}}, {a: ['f', 'fOo']}); + match({a: {$nin: ['x', /foo/i]}}, {a: ['f', 'fOx']}); + + nomatch({a: {$nin: [1, null]}}, {}); + nomatch({'a.b': {$nin: [1, null]}}, {}); + nomatch({'a.b': {$nin: [1, null]}}, {a: {}}); + nomatch({'a.b': {$nin: [1, null]}}, {a: {b: null}}); + match({'a.b': {$nin: [1, null]}}, {a: {b: 5}}); + match({'a.b': {$nin: [1]}}, {a: {b: null}}); + match({'a.b': {$nin: [1]}}, {a: {}}); + match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}]}); + nomatch({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, {}]}); + match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, []]}); + match({'a.b': {$nin: [1, null]}}, {a: [{b: 5}, 5]}); + // $size match({a: {$size: 0}}, {a: []}); match({a: {$size: 1}}, {a: [2]}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index fdf1fec1d8..6436243efa 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -248,6 +248,10 @@ var VALUE_OPERATORS = { $ne: function (operand) { return invertBranchedSelector(convertElementSelectorToBranchedSelector( equalityElementSelector(operand))); + }, + $nin: function (operand) { + return invertBranchedSelector(convertElementSelectorToBranchedSelector( + ELEMENT_OPERATORS.$in(operand))); } }; @@ -274,7 +278,7 @@ var makeInequality = function (cmpValueComparator) { // vs undefined). if (LocalCollection._f._type(value) !== operandType) return false; - return cmpValueComparator(LocalCollection._f._cmp(value, operand)) + return cmpValueComparator(LocalCollection._f._cmp(value, operand)); }; }; }; From 38a9b08ff62779cf19b65066570dfac960c9c260 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sat, 28 Dec 2013 23:39:14 -0800 Subject: [PATCH 059/124] Move $size to the new model --- packages/minimongo/selector.js | 43 ++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 6436243efa..d2cde73201 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -85,10 +85,13 @@ var regexpElementSelector = function (regexp) { }; -var convertElementSelectorToBranchedSelector = function (elementSelector) { +var convertElementSelectorToBranchedSelector = function ( + elementSelector, options) { + options = options || {}; return GROK(function (branches, wholeDoc) { - // XXX in some cases don't do this - var expanded = expandArraysInBranches(branches); + var expanded = branches; + if (!options.dontExpandLeafArrays) + expanded = expandArraysInBranches(branches); var result = _.any(expanded, function (element) { // XXX arrayIndex! need to save the winner here return elementSelector(element.value, wholeDoc); @@ -142,10 +145,13 @@ var operatorValueSelector = function (valueSelector, cursor) { VALUE_OPERATORS[operator](operand, valueSelector, cursor)); } else if (_.has(ELEMENT_OPERATORS, operator)) { // XXX justify three arguments + var options = ELEMENT_OPERATORS[operator]; + if (typeof options === 'function') + options = {elementSelector: options}; operatorFunctions.push( convertElementSelectorToBranchedSelector( - ELEMENT_OPERATORS[operator]( - operand, valueSelector, cursor))); + options.elementSelector(operand, valueSelector, cursor), + options)); } else { throw new Error("Unrecognized operator: " + operator); } @@ -339,6 +345,24 @@ var ELEMENT_OPERATORS = { return e(value); }); }; + }, + $size: { + // {a: [[5, 5]]} must match {a: {$size: 1}} but not {a: {$size: 2}}, so we + // don't want to consider the element [5,5] in the leaf array [[5,5]] as a + // possible value. + dontExpandLeafArrays: true, + elementSelector: function (operand) { + if (typeof operand === 'string') { + // Don't ask me why, but by experimentation, this seems to be what Mongo + // does. + operand = 0; + } else if (typeof operand !== 'number') { + throw Error("$size needs a number"); + } + return function (value) { + return isArray(value) && value.length === operand; + }; + } } }; @@ -1011,3 +1035,12 @@ var andCompiledDocumentSelectors = function (selectors) { return {result: result}; }; }; + + +// Remaining to update: +// - $exists +// - $type +// - $regex/$option +// - $all +// - $elemMatch +// - $near/$maxDistance From a72c2b89a87dece2937c523d03ebbb6b70ea7dfc Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sat, 28 Dec 2013 23:47:15 -0800 Subject: [PATCH 060/124] Move $exists to the new model Fixes two incompatibilities: - {$exists: false} did the wrong logic when there were multiple branches (we (poorly) special-cased $not and $nin but not this negative case) - No longer require the argument to $exists to be a boolean; {$exists: 0} and {$exists: 1} should work, eg --- packages/minimongo/minimongo_tests.js | 9 +++++++++ packages/minimongo/selector.js | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 6ae30c5437..363d476beb 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -433,6 +433,15 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$exists: false}}, {a: [1]}); match({a: {$exists: false}}, {b: [1]}); + match({a: {$exists: 1}}, {a: 5}); + match({a: {$exists: 0}}, {b: 5}); + + nomatch({'a.x':{$exists: false}}, {a: [{}, {x: 5}]}); + match({'a.x':{$exists: true}}, {a: [{}, {x: 5}]}); + match({'a.x':{$exists: true}}, {a: [{}, {x: 5}]}); + match({'a.x':{$exists: true}}, {a: {x: []}}); + match({'a.x':{$exists: true}}, {a: {x: null}}); + // $mod match({a: {$mod: [10, 1]}}, {a: 11}); nomatch({a: {$mod: [10, 1]}}, {a: 12}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d2cde73201..a793c737c5 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -258,6 +258,12 @@ var VALUE_OPERATORS = { $nin: function (operand) { return invertBranchedSelector(convertElementSelectorToBranchedSelector( ELEMENT_OPERATORS.$in(operand))); + }, + $exists: function (operand) { + var exists = convertElementSelectorToBranchedSelector(function (value) { + return value !== undefined; + }); + return operand ? exists : invertBranchedSelector(exists); } }; @@ -1038,7 +1044,6 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: -// - $exists // - $type // - $regex/$option // - $all From dafcd0d52c8f85034bae97e2c76c459172010765 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sat, 28 Dec 2013 23:55:07 -0800 Subject: [PATCH 061/124] Move $type to the new model Requires another special option, yay. --- packages/minimongo/selector.js | 37 +++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index a793c737c5..44f0375abc 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -90,8 +90,10 @@ var convertElementSelectorToBranchedSelector = function ( options = options || {}; return GROK(function (branches, wholeDoc) { var expanded = branches; - if (!options.dontExpandLeafArrays) - expanded = expandArraysInBranches(branches); + if (!options.dontExpandLeafArrays) { + expanded = expandArraysInBranches( + branches, options.dontIncludeLeafArrays); + } var result = _.any(expanded, function (element) { // XXX arrayIndex! need to save the winner here return elementSelector(element.value, wholeDoc); @@ -369,6 +371,21 @@ var ELEMENT_OPERATORS = { return isArray(value) && value.length === operand; }; } + }, + $type: { + // {a: [5]} must not match {a: {$type: 4}} (4 means array), but it should + // match {a: {$type: 1}} (1 means number), and {a: [[5]]} must match {$a: + // {$type: 4}}. Thus, when we see a leaf array, we *should* expand it but + // should *not* include it itself. + dontIncludeLeafArrays: true, + elementSelector: function (operand) { + if (typeof operand !== 'number') + throw Error("$type needs a number"); + return function (value) { + return value !== undefined + && LocalCollection._f._type(value) === operand; + }; + } } }; @@ -914,14 +931,17 @@ LocalCollection._makeLookupFunction = function (key) { }; }; -var expandArraysInBranches = function (branches) { +var expandArraysInBranches = function (branches, skipTheArrays) { var branchesOut = []; _.each(branches, function (branch) { - branchesOut.push({ - value: branch.value, - arrayIndex: branch.arrayIndex - }); - if (isArray(branch.value) && !branch.dontIterate) { + var thisIsArray = isArray(branch.value); + if (!skipTheArrays || !thisIsArray) { + branchesOut.push({ + value: branch.value, + arrayIndex: branch.arrayIndex + }); + } + if (thisIsArray && !branch.dontIterate) { _.each(branch.value, function (leaf, i) { branchesOut.push({ value: leaf, @@ -1044,7 +1064,6 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: -// - $type // - $regex/$option // - $all // - $elemMatch From 1a8a45e6b98b855f43258856fcda703e22ce7f16 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 29 Dec 2013 17:18:00 -0800 Subject: [PATCH 062/124] Move $regex/$options to the new model Better type-checking for $regex, and it's an error to provide $options without $regex. --- packages/minimongo/minimongo_tests.js | 5 ++-- packages/minimongo/selector.js | 38 +++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 363d476beb..a640cb7fc7 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -635,8 +635,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: /xxx/}, {}); nomatch({a: {$regex: 'xxx'}}, {}); - match({a: {$options: 'i'}}, {a: 12}); - match({b: {$options: 'i'}}, {a: 12}); + test.throws(function () { + match({a: {$options: 'i'}}, {a: 12}); + }); match({a: /a/}, {a: ['dog', 'cat']}); nomatch({a: /a/}, {a: ['dog', 'puppy']}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 44f0375abc..04473fd529 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -266,6 +266,12 @@ var VALUE_OPERATORS = { return value !== undefined; }); return operand ? exists : invertBranchedSelector(exists); + }, + // $options just provides options for $regex; its logic is inside $regex + $options: function (operand, valueSelector) { + if (!valueSelector.$regex) + throw Error("$options needs a $regex"); + return matchesEverythingSelector; } }; @@ -386,6 +392,31 @@ var ELEMENT_OPERATORS = { && LocalCollection._f._type(value) === operand; }; } + }, + $regex: function (operand, valueSelector) { + if (!(typeof operand === 'string' || operand instanceof RegExp)) + throw Error("$regex has to be a string or RegExp"); + + var regexp; + if (valueSelector.$options !== undefined) { + // Options passed in $options (even the empty string) always overrides + // options in the RegExp object itself. (See also + // Meteor.Collection._rewriteSelector.) + + // Be clear that we only support the JS-supported options, not extended + // ones (eg, Mongo supports x and s). Ideally we would implement x and s + // by transforming the regexp, but not today... + if (/[^gim]/.test(valueSelector.$options)) + throw new Error("Only the i, m, and g regexp options are supported"); + + var regexSource = operand instanceof RegExp ? operand.source : operand; + regexp = new RegExp(regexSource, valueSelector.$options); + } else if (operand instanceof RegExp) { + regexp = operand; + } else { + regexp = new RegExp(operand); + } + return regexpElementSelector(regexp); } }; @@ -1046,10 +1077,14 @@ LocalCollection._compileSelector = function (selector, cursor) { }; }; -var matchesNothingSelector = function (doc) { +var matchesNothingSelector = function (docOrBranchedValues) { return {result: false}; }; +var matchesEverythingSelector = function (docOrBranchedValues) { + return {result: true}; +}; + var andCompiledDocumentSelectors = function (selectors) { // XXX simplify to not involve 'arguments' once _distance is refactored return function (/*doc, sometimes wholeDoc*/) { @@ -1064,7 +1099,6 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: -// - $regex/$option // - $all // - $elemMatch // - $near/$maxDistance From c71c5598dbf733d162a0d8be74a7cec7d30cb365 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 29 Dec 2013 22:35:59 -0800 Subject: [PATCH 063/124] Move $elemMatch into the new model Support {x: {$elemMatch: {$gt: 5}}} syntax where the $elemMatch argument is a "value selector" and not a "full doc selector". --- packages/minimongo/minimongo_tests.js | 9 ++++++ packages/minimongo/selector.js | 44 ++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index a640cb7fc7..1d3dc00317 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1043,6 +1043,15 @@ Tinytest.add("minimongo - selector_compiler", function (test) { {dogs: [{name: "Fido", age: 5}, {name: "Rex", age: 3}]}); nomatch({dogs: {$elemMatch: {name: /e/, age: 5}}}, {dogs: [{name: "Fido", age: 5}, {name: "Rex", age: 3}]}); + match({x: {$elemMatch: {y: 9}}}, {x: [{y: 9}]}); + nomatch({x: {$elemMatch: {y: 9}}}, {x: [[{y: 9}]]}); + match({x: {$elemMatch: {$gt: 5, $lt: 9}}}, {x: [8]}); + nomatch({x: {$elemMatch: {$gt: 5, $lt: 9}}}, {x: [[8]]}); + match({'a.x': {$elemMatch: {y: 9}}}, + {a: [{x: []}, {x: [{y: 9}]}]}); + nomatch({a: {$elemMatch: {x: 5}}}, {a: {x: 5}}); + match({a: {$elemMatch: {0: {$gt: 5, $lt: 9}}}}, {a: [[6]]}); + match({a: {$elemMatch: {'0.b': {$gt: 5, $lt: 9}}}}, {a: [[{b:6}]]}); // $comment match({a: 5, $comment: "asdf"}, {a: 5}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 04473fd529..a6c30e5cba 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -417,6 +417,49 @@ var ELEMENT_OPERATORS = { regexp = new RegExp(operand); } return regexpElementSelector(regexp); + }, + $elemMatch: { + dontExpandLeafArrays: true, + elementSelector: function (operand, valueSelector, cursor) { + if (!isPlainObject(operand)) + throw Error("$elemMatch need an object"); + + var matcher, isDocMatcher; + if (isOperatorObject(operand)) { + matcher = compileValueSelector(operand); + isDocMatcher = false; + } else { + // This is NOT the same as compileValueSelector(operand), and not just + // because of the slightly different calling convention. + // {$elemMatch: {x: 3}} means "an element has a field x:3", not + // "consists only of a field x:3". Also, regexps and sub-$ are allowed. + matcher = compileDocumentSelector(operand); + isDocMatcher = true; + } + + return function (value) { + if (!isArray(value)) + return false; + return _.any(value, function (arrayElement) { + // XXX arrayIndex! + // XXX nesting geo stuff in here! + var arg; + if (isDocMatcher) { + // We can only match {$elemMatch: {b: 3}} against objects. + // (We can also match against arrays, if there's numeric indices, + // eg {$elemMatch: {'0.b': 3}} or {$elemMatch: {0: 3}}.) + if (!isPlainObject(arrayElement) && !isArray(arrayElement)) + return false; + arg = arrayElement; + } else { + // dontIterate ensures that {a: {$elemMatch: {$gt: 5}}} matches + // {a: [8]} but not {a: [[8]]} + arg = [{value: arrayElement, dontIterate: true}]; + } + return matcher(arg).result; + }); + }; + } } }; @@ -1100,5 +1143,4 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: // - $all -// - $elemMatch // - $near/$maxDistance From 60cd9a67ba008f1f73aec3653ae0a0e74c62f5c4 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 29 Dec 2013 23:12:28 -0800 Subject: [PATCH 064/124] Move $all to the new model Implement $all with regexps. Note that $all with $elemMatch needs to be separately implemented. It's an entirely different operation. --- packages/minimongo/minimongo_tests.js | 11 +++++++++ packages/minimongo/selector.js | 35 +++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 1d3dc00317..182d3a28e6 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -416,6 +416,17 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$all: [1, 2]}}, {a: [[1, 2]]}); // tested against mongodb nomatch({a: {$all: [1, 2]}}, {}); // tested against mongodb, field doesn't exist nomatch({a: {$all: [1, 2]}}, {a: {foo: 'bar'}}); // tested against mongodb, field is not an object + nomatch({a: {$all: []}}, {a: []}); + nomatch({a: {$all: []}}, {a: [5]}); + match({a: {$all: [/i/, /e/i]}}, {a: ["foo", "bEr", "biz"]}); + nomatch({a: {$all: [/i/, /e/i]}}, {a: ["foo", "bar", "biz"]}); + match({a: {$all: [{b: 3}]}}, {a: [{b: 3}]}); + // Members of $all other than regexps are *equality matches*, not document + // matches. + nomatch({a: {$all: [{b: 3}]}}, {a: [{b: 3, k: 4}]}); + test.throws(function () { + match({a: {$all: [{$gt: 4}]}}, {}); + }); // $exists match({a: {$exists: true}}, {a: 12}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index a6c30e5cba..7022187194 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -158,12 +158,17 @@ var operatorValueSelector = function (valueSelector, cursor) { throw new Error("Unrecognized operator: " + operator); } }); - // NB: this is very similar to andCompiledDocumentSelectors but that one - // "assumes" the first arg is a doc and here it "assumes" it's a branched - // value list. The code is identical for now, though. + + return andBranchedSelectors(operatorFunctions); +}; + +// NB: this is very similar to andCompiledDocumentSelectors but that one +// "assumes" the first arg is a doc and here it "assumes" it's a branched +// value list. The code is identical for now, though. +var andBranchedSelectors = function (branchedSelectors) { return GROK(function (branches, doc) { // XXX arrayIndex! - var result = _.all(operatorFunctions, function (f) { + var result = _.all(branchedSelectors, function (f) { return f(branches, doc).result; }); return {result: result}; @@ -272,6 +277,25 @@ var VALUE_OPERATORS = { if (!valueSelector.$regex) throw Error("$options needs a $regex"); return matchesEverythingSelector; + }, + $all: function (operand) { + if (!isArray(operand)) + throw Error("$all requires array"); + // Not sure why, but this seems to be what MongoDB does. + if (_.isEmpty(operand)) + return matchesNothingSelector; + + var branchedSelectors = []; + _.each(operand, function (criterion) { + // XXX handle $all/$elemMatch combination + if (isOperatorObject(criterion)) + throw Error("no $ expressions in $all"); + // This is always a regexp or equality selector. + branchedSelectors.push(compileValueSelector(criterion)); + }); + // andBranchedSelectors does NOT require all selectors to return true on the + // SAME branch. + return andBranchedSelectors(branchedSelectors); } }; @@ -1142,5 +1166,6 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: -// - $all // - $near/$maxDistance +// Remaining to implement: +// - $all with $elemMatch From a8d10b9a02446b7527330cadb62ac0f2d7f9338a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Sun, 29 Dec 2013 23:41:55 -0800 Subject: [PATCH 065/124] Refactor sort/$near distance stuff Instead of having a semi-implicit dependency on the cursor, specify distances explicitly when "instantiating" the comparator. The next refactoring will remove _distance from cursor entirely. --- packages/minimongo/minimongo.js | 35 ++++++++++++++--------- packages/minimongo/minimongo_tests.js | 9 +++--- packages/minimongo/modify.js | 3 +- packages/minimongo/selector.js | 3 +- packages/minimongo/sort.js | 41 +++++++++++++++++++++------ 5 files changed, 62 insertions(+), 29 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index b23effb8e0..547ba52e48 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -90,7 +90,7 @@ LocalCollection.Cursor = function (collection, selector, options) { // stash for fast path self.selector_id = LocalCollection._idStringify(selector); self.selector_f = LocalCollection._compileSelector(selector, self); - self.sort_f = undefined; + self.sorter = undefined; } else { // MongoDB throws different errors on different branching operators // containing $near @@ -99,8 +99,8 @@ LocalCollection.Cursor = function (collection, selector, options) { self.selector_id = undefined; self.selector_f = LocalCollection._compileSelector(selector, self); - self.sort_f = (isGeoQuery(selector) || options.sort) ? - LocalCollection._compileSort(options.sort || [], self) : null; + self.sorter = (isGeoQuery(selector) || options.sort) ? + new Sorter(options.sort || []) : null; } self.skip = options.skip; self.limit = options.limit; @@ -276,7 +276,7 @@ _.extend(LocalCollection.Cursor.prototype, { // XXX merge this object w/ "this" Cursor. they're the same. var query = { selector_f: self.selector_f, // not fast pathed - sort_f: ordered && self.sort_f, + sorter: ordered && self.sorter, results_snapshot: null, ordered: ordered, cursor: self, @@ -371,11 +371,11 @@ _.extend(LocalCollection.Cursor.prototype, { // Returns a collection of matching objects, but doesn't deep copy them. // -// If ordered is set, returns a sorted array, respecting sort_f, skip, and limit -// properties of the query. if sort_f is falsey, no sort -- you get the natural +// If ordered is set, returns a sorted array, respecting sorter, skip, and limit +// properties of the query. if sorter is falsey, no sort -- you get the natural // order. // -// If ordered is not set, returns an object mapping from ID to doc (sort_f, skip +// If ordered is not set, returns an object mapping from ID to doc (sorter, skip // and limit should not be set). LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { var self = this; @@ -410,7 +410,7 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { results[id] = doc; } // Fast path for limited unsorted queries. - if (self.limit && !self.skip && !self.sort_f && + if (self.limit && !self.skip && !self.sorter && results.length === self.limit) return results; } @@ -418,14 +418,21 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { if (!ordered) return results; - if (self.sort_f) - results.sort(self.sort_f); + if (self.sorter) { + var comparator = self._getComparator(); + results.sort(comparator); + } var idx_start = self.skip || 0; var idx_end = self.limit ? (self.limit + idx_start) : results.length; return results.slice(idx_start, idx_end); }; +LocalCollection.Cursor.prototype._getComparator = function () { + var self = this; + return self.sorter.getComparator({distances: self._distance}); +}; + // XXX Maybe we need a version of observe that just calls a callback if // anything changed. LocalCollection.Cursor.prototype._depend = function (changers, _allow_unordered) { @@ -715,12 +722,12 @@ LocalCollection._insertInResults = function (query, doc) { var fields = EJSON.clone(doc); delete fields._id; if (query.ordered) { - if (!query.sort_f) { + if (!query.sorter) { query.addedBefore(doc._id, fields, null); query.results.push(doc); } else { var i = LocalCollection._insertInSortedList( - query.sort_f, query.results, doc); + query.cursor._getComparator(), query.results, doc); var next = query.results[i+1]; if (next) next = next._id; @@ -763,14 +770,14 @@ LocalCollection._updateInResults = function (query, doc, old_doc) { if (!_.isEmpty(changedFields)) query.changed(doc._id, changedFields); - if (!query.sort_f) + if (!query.sorter) return; // just take it out and put it back in again, and see if the index // changes query.results.splice(orig_idx, 1); var new_idx = LocalCollection._insertInSortedList( - query.sort_f, query.results, doc); + query.cursor._getComparator(), query.results, doc); if (orig_idx !== new_idx) { var next = query.results[new_idx+1]; if (next) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 182d3a28e6..8d70d4c180 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1504,7 +1504,8 @@ Tinytest.add("minimongo - ordering", function (test) { // document ordering under a sort specification var verify = function (sorts, docs) { _.each(sorts, function (sort) { - assert_ordering(test, LocalCollection._compileSort(sort), docs); + var sorter = new MinimongoTest.Sorter(sort); + assert_ordering(test, sorter.getComparator(), docs); }); }; @@ -1531,14 +1532,14 @@ Tinytest.add("minimongo - ordering", function (test) { [{c: 1}, {a: 1, b: 2}, {a: 1, b: 3}, {a: 2, b: 0}]); test.throws(function () { - LocalCollection._compileSort("a"); + new MinimongoTest.Sorter("a"); }); test.throws(function () { - LocalCollection._compileSort(123); + new MinimongoTest.Sorter(123); }); - test.equal(LocalCollection._compileSort({})({a:1}, {a:2}), 0); + test.equal(new MinimongoTest.Sorter({}).getComparator()({a:1}, {a:2}), 0); }); Tinytest.add("minimongo - sort", function (test) { diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 4ea173d79b..9a2c4ca0cd 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -218,7 +218,8 @@ LocalCollection._modifiers = { // XXX this allows us to use a $sort whose value is an array, but that's // actually an extension of the Node driver, so it won't work // server-side. Could be confusing! - sortFunction = LocalCollection._compileSort(arg.$sort); + // XXX is it correct that we don't do geo-stuff here? + sortFunction = new Sorter(arg.$sort).getComparator(); for (var i = 0; i < toPush.length; i++) { if (LocalCollection._f._type(toPush[i]) !== 3) { throw MinimongoError("$push like modifiers using $sort " + diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 7022187194..69faae7669 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -905,7 +905,7 @@ numericKey = function (s) { }; // XXX redoc -// XXX be aware that _compileSort currently assumes that lookup functions +// XXX be aware that Sorter currently assumes that lookup functions // return non-empty arrays but that is no longer the case // _makeLookupFunction(key) returns a lookup function. // @@ -1167,5 +1167,6 @@ var andCompiledDocumentSelectors = function (selectors) { // Remaining to update: // - $near/$maxDistance +// - make sure to switch _distance to be an idmap // Remaining to implement: // - $all with $elemMatch diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index fb3b80d881..1aca9bf8ce 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -11,7 +11,10 @@ // first object comes first in order, 1 if the second object comes // first, or 0 if neither object comes before the other. -LocalCollection._compileSort = function (spec, cursor) { +Sorter = function (spec) { + var self = this; + self._sortFunction = null; + var sortSpecParts = []; if (spec instanceof Array) { @@ -39,14 +42,11 @@ LocalCollection._compileSort = function (spec, cursor) { throw Error("Bad sort specification: ", JSON.stringify(spec)); } - // If there are no sorting rules specified, try to sort on _distance hidden - // fields on cursor we may acquire if query involved $near operator. + // If there are no sorting rules specified, leave _sortFunction as null. This + // will allow us to have a special case where we sort on distances if query + // involved the $near operator. if (sortSpecParts.length === 0) - return function (a, b) { - if (!cursor || !cursor._distance) - return 0; - return cursor._distance[a._id] - cursor._distance[b._id]; - }; + return; // reduceValue takes in all the possible values for the sort key along various // branches, and returns the min or max value (according to the bool @@ -86,7 +86,7 @@ LocalCollection._compileSort = function (spec, cursor) { return reduced; }; - return function (a, b) { + self._sortFunction = function (a, b) { for (var i = 0; i < sortSpecParts.length; ++i) { var specPart = sortSpecParts[i]; var aValue = reduceValue(specPart.lookup(a), specPart.ascending); @@ -98,3 +98,26 @@ LocalCollection._compileSort = function (spec, cursor) { return 0; }; }; + +Sorter.prototype.getComparator = function (options) { + var self = this; + // If there was a sort specification, use it. + // XXX do we not use distance as a secondary sort key? + if (self._sortFunction) + return self._sortFunction; + + // If there was no sort specification and we have no distances, everything is + // equal. + if (!options || !options.distances) { + return function (a, b) { + return 0; + }; + } + + var distances = _.clone(options.distances); + return function (a, b) { + return distances[a._id] - distances[b._id]; + }; +}; + +MinimongoTest.Sorter = Sorter; From 2cc3bf8635299b4f19d880c47e1712cb8e792434 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 30 Dec 2013 14:16:12 -0800 Subject: [PATCH 066/124] move "no $near under $" check into compiler --- packages/minimongo/minimongo.js | 17 +---------------- packages/minimongo/selector.js | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 547ba52e48..2f8c324c04 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -92,11 +92,6 @@ LocalCollection.Cursor = function (collection, selector, options) { self.selector_f = LocalCollection._compileSelector(selector, self); self.sorter = undefined; } else { - // MongoDB throws different errors on different branching operators - // containing $near - if (isGeoQuerySpecial(selector)) - throw new Error("$near can't be inside $or/$and/$nor/$not"); - self.selector_id = undefined; self.selector_f = LocalCollection._compileSelector(selector, self); self.sorter = (isGeoQuery(selector) || options.sort) ? @@ -986,20 +981,10 @@ LocalCollection._makeChangedFields = function (newDoc, oldDoc) { // Searches $near operator in the selector recursively // (including all $or/$and/$nor/$not branches) +// XXX this should be something determined by the selector compiler, not later var isGeoQuery = function (selector) { return _.any(selector, function (val, key) { // Note: _.isObject matches objects and arrays return key === "$near" || (_.isObject(val) && isGeoQuery(val)); }); }; - -// Checks if $near appears under some $or/$and/$nor/$not branch -var isGeoQuerySpecial = function (selector) { - return _.any(selector, function (val, key) { - if (_.contains(['$or', '$and', '$nor', '$not'], key)) - return isGeoQuery(val); - // Note: _.isObject matches objects and arrays - return _.isObject(val) && isGeoQuerySpecial(val); - }); -}; - diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 69faae7669..d1c14bb0e3 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -58,12 +58,12 @@ var isOperatorObject = function (valueSelector) { }; -var compileValueSelector = function (valueSelector, cursor) { +var compileValueSelector = function (valueSelector, cursor, inRoot) { if (valueSelector instanceof RegExp) return convertElementSelectorToBranchedSelector( regexpElementSelector(valueSelector)); else if (isOperatorObject(valueSelector)) - return operatorValueSelector(valueSelector, cursor); + return operatorValueSelector(valueSelector, cursor, inRoot); else { return convertElementSelectorToBranchedSelector( equalityElementSelector(valueSelector)); @@ -127,13 +127,13 @@ var GROK = function (f) { }; // XXX get rid of cursor when possible -var operatorValueSelector = function (valueSelector, cursor) { +var operatorValueSelector = function (valueSelector, cursor, inRoot) { // XXX kill this soon if (!_.all(valueSelector, function (operand, operator) { return _.has(ELEMENT_OPERATORS, operator) || _.has(VALUE_OPERATORS, operator); })) { - return operatorValueSelectorLegacy(valueSelector, cursor); + return operatorValueSelectorLegacy(valueSelector, cursor, inRoot); } // Each valueSelector works separately on the various branches. So one @@ -152,7 +152,7 @@ var operatorValueSelector = function (valueSelector, cursor) { options = {elementSelector: options}; operatorFunctions.push( convertElementSelectorToBranchedSelector( - options.elementSelector(operand, valueSelector, cursor), + options.elementSelector(operand, valueSelector, cursor, inRoot), options)); } else { throw new Error("Unrecognized operator: " + operator); @@ -652,7 +652,10 @@ var LEGACY_VALUE_OPERATORS = { }; }, - "$near": function (operand, operators, cursor) { + "$near": function (operand, operators, cursor, inRoot) { + if (!inRoot) + throw Error("$near can't be inside another $ operator"); + function distanceCoordinatePairs (a, b) { a = pointToArray(a); b = pointToArray(b); @@ -711,14 +714,14 @@ var LEGACY_VALUE_OPERATORS = { } }; -var operatorValueSelectorLegacy = function (valueSelector, cursor) { +var operatorValueSelectorLegacy = function (valueSelector, cursor, inRoot) { var operatorFunctions = []; _.each(valueSelector, function (operand, operator) { if (!_.has(LEGACY_VALUE_OPERATORS, operator)) throw new Error("Unrecognized legacy operator: " + operator); // Special case for location operators operatorFunctions.push(LEGACY_VALUE_OPERATORS[operator]( - operand, valueSelector, cursor)); + operand, valueSelector, cursor, inRoot)); }); return function (value, doc) { return _.all(operatorFunctions, function (f) { @@ -1052,7 +1055,7 @@ var expandArraysInBranches = function (branches, skipTheArrays) { }; // The main compilation function for a given selector. -var compileDocumentSelector = function (docSelector, cursor) { +var compileDocumentSelector = function (docSelector, cursor, isRoot) { var perKeySelectors = []; _.each(docSelector, function (subSelector, key) { if (key.substr(0, 1) === '$') { @@ -1066,7 +1069,7 @@ var compileDocumentSelector = function (docSelector, cursor) { } else { var lookUpByIndex = LocalCollection._makeLookupFunction2(key); var valueSelectorFunc = - compileValueSelector(subSelector, cursor); + compileValueSelector(subSelector, cursor, isRoot); perKeySelectors.push(function (doc, wholeDoc) { var branchValues = lookUpByIndex(doc); @@ -1138,7 +1141,7 @@ LocalCollection._compileSelector = function (selector, cursor) { throw new Error("Invalid selector: " + selector); // XXX get rid of second argument once _distance refactored - var s = compileDocumentSelector(selector, cursor); + var s = compileDocumentSelector(selector, cursor, true); return function (doc) { return s(doc, doc); }; From a2dba04d782b73fb374b1cbbb56016eb51cefc16 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 30 Dec 2013 15:03:25 -0800 Subject: [PATCH 067/124] Move $near to the new model Implement branching for $near (not quite the way Mongo does it, but better than nothing) That was the last $operator to be moved to the new model, so I could delete a lot of obsolete stuff. --- packages/minimongo/minimongo_tests.js | 25 ++ packages/minimongo/selector.js | 421 ++++++-------------------- 2 files changed, 126 insertions(+), 320 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 8d70d4c180..4811f6dcd2 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -2610,5 +2610,30 @@ Tinytest.add("minimongo - $near operator tests", function (test) { }] }); }); + + // array tests + coll = new LocalCollection(); + coll.insert({ + _id: "x", + a: [ + {b: [ + [100, 100], + [1, 1]]}, + {b: [150, 150]}]}); + coll.insert({ + _id: "y", + a: {b: [5, 5]}}); + var testNear = function (near, md, expected) { + test.equal( + _.pluck( + coll.find({'a.b': {$near: near, $maxDistance: md}}).fetch(), '_id'), + expected); + }; + testNear([149, 149], 4, ['x']); + testNear([149, 149], 1000, ['x', 'y']); + // It's important that we figure out that 'x' is closer than 'y' to [2,2] even + // though the first within-1000 point in 'x' (ie, [100,100]) is farther than + // 'y'. + testNear([2, 2], 1000, ['x', 'y']); }); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d1c14bb0e3..641ad22139 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -16,31 +16,6 @@ var isIndexable = function (x) { return isArray(x) || isPlainObject(x); }; -// If x is an array, true if f(e) is true for some e in x -// (but never try f(x) directly) -// Otherwise, true if f(x) is true. -// -// Use this in cases where f(Array) should never be true... -// for example, equality comparisons to non-arrays, -// ordering comparisons (which should always be false if either side -// is an array), regexps (need string), mod (needs number)... -// XXX ensure comparisons are always false if LHS is an array -// XXX ensure comparisons among different types are false -var _anyIfArray = function (x, f) { - if (isArray(x)) - return _.any(x, f); - return f(x); -}; - -// True if f(x) is true, or x is an array and f(e) is true for some e in x. -// -// Use this for most operators where an array could satisfy the predicate. -var _anyIfArrayPlus = function (x, f) { - if (f(x)) - return true; - return isArray(x) && _.any(x, f); -}; - var isOperatorObject = function (valueSelector) { if (!isPlainObject(valueSelector)) return false; @@ -88,7 +63,7 @@ var regexpElementSelector = function (regexp) { var convertElementSelectorToBranchedSelector = function ( elementSelector, options) { options = options || {}; - return GROK(function (branches, wholeDoc) { + return function (branches, wholeDoc) { var expanded = branches; if (!options.dontExpandLeafArrays) { expanded = expandArraysInBranches( @@ -99,7 +74,7 @@ var convertElementSelectorToBranchedSelector = function ( return elementSelector(element.value, wholeDoc); }); return {result: result}; - }); + }; }; var equalityElementSelector = function (elementSelector) { @@ -120,22 +95,8 @@ var equalityElementSelector = function (elementSelector) { }; }; -// XXX this won't be necessary -var GROK = function (f) { - f._groksExtendedLookup_ = true; - return f; -}; - // XXX get rid of cursor when possible var operatorValueSelector = function (valueSelector, cursor, inRoot) { - // XXX kill this soon - if (!_.all(valueSelector, function (operand, operator) { - return _.has(ELEMENT_OPERATORS, operator) || - _.has(VALUE_OPERATORS, operator); - })) { - return operatorValueSelectorLegacy(valueSelector, cursor, inRoot); - } - // Each valueSelector works separately on the various branches. So one // operator can match one branch and another can match another branch. This // is OK. @@ -144,7 +105,7 @@ var operatorValueSelector = function (valueSelector, cursor, inRoot) { _.each(valueSelector, function (operand, operator) { if (_.has(VALUE_OPERATORS, operator)) { operatorFunctions.push( - VALUE_OPERATORS[operator](operand, valueSelector, cursor)); + VALUE_OPERATORS[operator](operand, valueSelector, cursor, inRoot)); } else if (_.has(ELEMENT_OPERATORS, operator)) { // XXX justify three arguments var options = ELEMENT_OPERATORS[operator]; @@ -152,7 +113,7 @@ var operatorValueSelector = function (valueSelector, cursor, inRoot) { options = {elementSelector: options}; operatorFunctions.push( convertElementSelectorToBranchedSelector( - options.elementSelector(operand, valueSelector, cursor, inRoot), + options.elementSelector(operand, valueSelector, cursor), options)); } else { throw new Error("Unrecognized operator: " + operator); @@ -166,13 +127,13 @@ var operatorValueSelector = function (valueSelector, cursor, inRoot) { // "assumes" the first arg is a doc and here it "assumes" it's a branched // value list. The code is identical for now, though. var andBranchedSelectors = function (branchedSelectors) { - return GROK(function (branches, doc) { + return function (branches, doc) { // XXX arrayIndex! var result = _.all(branchedSelectors, function (f) { return f(branches, doc).result; }); return {result: result}; - }); + }; }; // XXX drop "cursor" when _distance is improved @@ -278,6 +239,12 @@ var VALUE_OPERATORS = { throw Error("$options needs a $regex"); return matchesEverythingSelector; }, + // $maxDistance is basically an argument to $near + $maxDistance: function (operand, valueSelector) { + if (!valueSelector.$near) + throw Error("$maxDistance needs a $near"); + return matchesEverythingSelector; + }, $all: function (operand) { if (!isArray(operand)) throw Error("$all requires array"); @@ -296,9 +263,97 @@ var VALUE_OPERATORS = { // andBranchedSelectors does NOT require all selectors to return true on the // SAME branch. return andBranchedSelectors(branchedSelectors); + }, + + $near: function (operand, valueSelector, cursor, inRoot) { + if (!inRoot) + throw Error("$near can't be inside another $ operator"); + + // There are two kinds of geodata in MongoDB: coordinate pairs and + // GeoJSON. They use different distance metrics, too. GeoJSON queries are + // marked with a $geometry property. + + var maxDistance, point, distance; + if (isPlainObject(operand) && _.has(operand, '$geometry')) { + // GeoJSON "2dsphere" mode. + maxDistance = operand.$maxDistance; + point = operand.$geometry; + distance = function (value) { + // XXX: for now, we don't calculate the actual distance between, say, + // polygon and circle. If people care about this use-case it will get + // a priority. + if (!value || !value.type) + return null; + if (value.type === "Point") { + return GeoJSON.pointDistance(point, value); + } else { + return GeoJSON.geometryWithinRadius(value, point, maxDistance) + ? 0 : maxDistance + 1; + } + }; + } else { + maxDistance = valueSelector.$maxDistance; + if (!isArray(operand) && !isPlainObject(operand)) + throw Error("$near argument must be coordinate pair or GeoJSON"); + point = pointToArray(operand); + distance = function (value) { + if (!isArray(value) && !isPlainObject(value)) + return null; + return distanceCoordinatePairs(point, value); + }; + } + + return function (branchedValues, doc) { + // There might be multiple points in the document that match the given + // field. Only one of them needs to be within $maxDistance, but we need to + // evaluate all of them and use the nearest one for the implicit sort + // specifier. (That's why we can't just use ELEMENT_OPERATORS here.) + // + // Note: This differs from MongoDB's implementation, where a document will + // actually show up *multiple times* in the result set, with one entry for + // each within-$maxDistance branching point. + branchedValues = expandArraysInBranches(branchedValues); + var minDistance = null; + _.each(branchedValues, function (branch) { + var curDistance = distance(branch.value); + // Skip branches that aren't real points or are too far away. + if (curDistance === null || curDistance > maxDistance) + return; + // Skip anything that's a tie. + if (minDistance !== null && minDistance <= curDistance) + return; + minDistance = curDistance; + }); + if (minDistance !== null) { + if (cursor) { + if (!cursor._distance) + cursor._distance = {}; + cursor._distance[doc._id] = minDistance; + } + // XXX arrayIndex! + return {result: true}; + } + return {result: false}; + }; } }; +var distanceCoordinatePairs = function (a, b) { + a = pointToArray(a); + b = pointToArray(b); + var x = a[0] - b[0]; + var y = a[1] - b[1]; + if (_.isNaN(x) || _.isNaN(y)) + return null; + return Math.sqrt(x * x + y * y); +}; +// Makes sure we get 2 elements array and assume the first one to be x and +// the second one to y no matter what user passes. +// In case user passes { lon: x, lat: y } returns [x, y] +var pointToArray = function (point) { + return _.map(point, _.identity); +}; + var makeInequality = function (cmpValueComparator) { return function (operand) { // Arrays never compare false with non-arrays for any inequality. @@ -487,249 +542,6 @@ var ELEMENT_OPERATORS = { } }; -var LEGACY_VALUE_OPERATORS = { - "$in": function (operand) { - if (!isArray(operand)) - throw new Error("Argument to $in must be array"); - return function (value) { - return _anyIfArrayPlus(value, function (x) { - return _.any(operand, function (operandElt) { - return LocalCollection._f._equal(operandElt, x); - }); - }); - }; - }, - - "$all": function (operand) { - if (!isArray(operand)) - throw new Error("Argument to $all must be array"); - return function (value) { - if (!isArray(value)) - return false; - return _.all(operand, function (operandElt) { - return _.any(value, function (valueElt) { - return LocalCollection._f._equal(operandElt, valueElt); - }); - }); - }; - }, - - "$lt": function (operand) { - return function (value) { - return _anyIfArray(value, function (x) { - return LocalCollection._f._cmp(x, operand) < 0; - }); - }; - }, - - "$lte": function (operand) { - return function (value) { - return _anyIfArray(value, function (x) { - return LocalCollection._f._cmp(x, operand) <= 0; - }); - }; - }, - - "$gt": function (operand) { - return function (value) { - return _anyIfArray(value, function (x) { - return LocalCollection._f._cmp(x, operand) > 0; - }); - }; - }, - - "$gte": function (operand) { - return function (value) { - return _anyIfArray(value, function (x) { - return LocalCollection._f._cmp(x, operand) >= 0; - }); - }; - }, - - "$ne": function (operand) { - return function (value) { - return ! _anyIfArrayPlus(value, function (x) { - return LocalCollection._f._equal(x, operand); - }); - }; - }, - - "$nin": function (operand) { - if (!isArray(operand)) - throw new Error("Argument to $nin must be array"); - var inFunction = LEGACY_VALUE_OPERATORS.$in(operand); - return function (value, doc) { - // Field doesn't exist, so it's not-in operand - if (value === undefined) - return true; - return !inFunction(value, doc); - }; - }, - - "$exists": function (operand) { - return function (value) { - return operand === (value !== undefined); - }; - }, - - "$mod": function (operand) { - var divisor = operand[0], - remainder = operand[1]; - return function (value) { - return _anyIfArray(value, function (x) { - return x % divisor === remainder; - }); - }; - }, - - "$size": function (operand) { - return function (value) { - return isArray(value) && operand === value.length; - }; - }, - - "$type": function (operand) { - return function (value) { - // A nonexistent field is of no type. - if (value === undefined) - return false; - // Definitely not _anyIfArrayPlus: $type: 4 only matches arrays that have - // arrays as elements according to the Mongo docs. - return _anyIfArray(value, function (x) { - return LocalCollection._f._type(x) === operand; - }); - }; - }, - - "$regex": function (operand, operators) { - var options = operators.$options; - if (options !== undefined) { - // Options passed in $options (even the empty string) always overrides - // options in the RegExp object itself. (See also - // Meteor.Collection._rewriteSelector.) - - // Be clear that we only support the JS-supported options, not extended - // ones (eg, Mongo supports x and s). Ideally we would implement x and s - // by transforming the regexp, but not today... - if (/[^gim]/.test(options)) - throw new Error("Only the i, m, and g regexp options are supported"); - - var regexSource = operand instanceof RegExp ? operand.source : operand; - operand = new RegExp(regexSource, options); - } else if (!(operand instanceof RegExp)) { - operand = new RegExp(operand); - } - - return function (value) { - if (value === undefined) - return false; - return _anyIfArray(value, function (x) { - return operand.test(x); - }); - }; - }, - - "$options": function (operand) { - // evaluation happens at the $regex function above - return function (value) { return true; }; - }, - - "$elemMatch": function (operand, selector, cursor) { - var matcher = compileDocumentSelector(operand, cursor); - return function (value, doc) { - if (!isArray(value)) - return false; - return _.any(value, function (x) { - return matcher(x, doc).result; - }); - }; - }, - - "$not": function (operand, operators, cursor) { - var matcher = compileValueSelector(operand, cursor); - return function (value, doc) { - return !matcher(value, doc); - }; - }, - - "$near": function (operand, operators, cursor, inRoot) { - if (!inRoot) - throw Error("$near can't be inside another $ operator"); - - function distanceCoordinatePairs (a, b) { - a = pointToArray(a); - b = pointToArray(b); - var x = a[0] - b[0]; - var y = a[1] - b[1]; - if (_.isNaN(x) || _.isNaN(y)) - return null; - return Math.sqrt(x * x + y * y); - } - // Makes sure we get 2 elements array and assume the first one to be x and - // the second one to y no matter what user passes. - // In case user passes { lon: x, lat: y } returns [x, y] - function pointToArray (point) { - return _.map(point, _.identity); - } - // GeoJSON query is marked as $geometry property - var mode = _.isObject(operand) && _.has(operand, '$geometry') ? "2dsphere" : "2d"; - var maxDistance = mode === "2d" ? operators.$maxDistance : operand.$maxDistance; - var point = mode === "2d" ? operand : operand.$geometry; - return function (value, doc) { - var dist = null; - switch (mode) { - case "2d": - dist = distanceCoordinatePairs(point, value); - break; - case "2dsphere": - // XXX: for now, we don't calculate the actual distance between, say, - // polygon and circle. If people care about this use-case it will get - // a priority. - if (value.type === "Point") - dist = GeoJSON.pointDistance(point, value); - else - dist = GeoJSON.geometryWithinRadius(value, point, maxDistance) ? - 0 : maxDistance + 1; - break; - } - // Used later in sorting by distance, since $near queries are sorted by - // distance from closest to farthest. - if (cursor) { - if (!cursor._distance) - cursor._distance = {}; - cursor._distance[doc._id] = dist; - } - - // Distance couldn't parse a geometry object - if (dist === null) - return false; - - return maxDistance === undefined ? true : dist <= maxDistance; - }; - }, - - "$maxDistance": function () { - // evaluation happens in the $near operator - return function () { return true; } - } -}; - -var operatorValueSelectorLegacy = function (valueSelector, cursor, inRoot) { - var operatorFunctions = []; - _.each(valueSelector, function (operand, operator) { - if (!_.has(LEGACY_VALUE_OPERATORS, operator)) - throw new Error("Unrecognized legacy operator: " + operator); - // Special case for location operators - operatorFunctions.push(LEGACY_VALUE_OPERATORS[operator]( - operand, valueSelector, cursor, inRoot)); - }); - return function (value, doc) { - return _.all(operatorFunctions, function (f) { - return f(value, doc); - }); - }; -}; - // helpers used by compiled selector code LocalCollection._f = { // XXX for _all and _in, consider building 'inquery' at compile time.. @@ -1072,38 +884,7 @@ var compileDocumentSelector = function (docSelector, cursor, isRoot) { compileValueSelector(subSelector, cursor, isRoot); perKeySelectors.push(function (doc, wholeDoc) { var branchValues = lookUpByIndex(doc); - - if (valueSelectorFunc._groksExtendedLookup_) { - return valueSelectorFunc(branchValues, wholeDoc); - } - - // XXX get rid of this all later - branchValues = _.pluck(branchValues, 'value'); - // We apply the selector to each "branched" value and return true if any - // match. However, for "negative" selectors like $ne or $not we actually - // require *all* elements to match. - // - // This is because {'x.tag': {$ne: "foo"}} applied to {x: [{tag: 'foo'}, - // {tag: 'bar'}]} should NOT match even though there is a branch that - // matches. (This matches the fact that $ne uses a negated - // _anyIfArrayPlus, for when the last level of the key is the array, - // which deMorgans into an 'all'.) - // - // XXX This isn't 100% consistent with MongoDB in 'null' cases: - // https://jira.mongodb.org/browse/SERVER-8585 - // XXX this still isn't right. consider {a: {$ne: 5, $gt: 6}}. the - // $ne needs to use the "all" logic and the $gt needs the "any" - // logic. (There is an expect_fail test for this now.) - // XXX add test for this brokenness - var combiner = (subSelector && - (subSelector.$not || subSelector.$ne || - subSelector.$nin)) - ? _.all : _.any; - var result = combiner(branchValues, function (val) { - return valueSelectorFunc(val, wholeDoc); - }); - // XXX rewrite value selectors to use structured return - return {result: result}; + return valueSelectorFunc(branchValues, wholeDoc); }); } }); From 9b9f34bb04b538090aab9bf53363839a120e8ec5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 30 Dec 2013 16:03:16 -0800 Subject: [PATCH 068/124] Refactor: compiled selector is a class For now this just lets us do isGeoQuery as something other than "reparse", but this should have some better effects later. --- packages/minimongo/minimongo.js | 45 ++++++++----------- packages/minimongo/modify.js | 6 +-- packages/minimongo/package.js | 1 + packages/minimongo/selector.js | 40 ++++++++++++----- packages/minimongo/selector_modifier.js | 6 ++- .../mongo-livedata/oplog_observe_driver.js | 6 +-- 6 files changed, 59 insertions(+), 45 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 2f8c324c04..5e297b0dc1 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -20,7 +20,7 @@ LocalCollection = function (name) { // results: array (ordered) or object (unordered) of current results // results_snapshot: snapshot of results. null if not paused. // cursor: Cursor object for the query. - // selector_f, sort_f, (callbacks): functions + // selector, sorter, (callbacks): functions this.queries = {}; // null if not saving originals; a map from id to original document value if @@ -31,6 +31,8 @@ LocalCollection = function (name) { this.paused = false; }; +Minimongo = {}; + // Object exported only for unit testing. // Use it to export private functions to test in Tinytest. MinimongoTest = {}; @@ -89,12 +91,12 @@ LocalCollection.Cursor = function (collection, selector, options) { if (LocalCollection._selectorIsId(selector)) { // stash for fast path self.selector_id = LocalCollection._idStringify(selector); - self.selector_f = LocalCollection._compileSelector(selector, self); + self.selector = new Minimongo.Selector(selector, self); self.sorter = undefined; } else { self.selector_id = undefined; - self.selector_f = LocalCollection._compileSelector(selector, self); - self.sorter = (isGeoQuery(selector) || options.sort) ? + self.selector = new Minimongo.Selector(selector, self); + self.sorter = (self.selector.isGeoQuery() || options.sort) ? new Sorter(options.sort || []) : null; } self.skip = options.skip; @@ -270,7 +272,7 @@ _.extend(LocalCollection.Cursor.prototype, { // XXX merge this object w/ "this" Cursor. they're the same. var query = { - selector_f: self.selector_f, // not fast pathed + selector: self.selector, // not fast pathed sorter: ordered && self.sorter, results_snapshot: null, ordered: ordered, @@ -398,7 +400,7 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { // slow path for arbitrary selector, sort, skip, limit for (var id in self.collection.docs) { var doc = self.collection.docs[id]; - if (self.selector_f(doc).result) { + if (self.selector.documentMatches(doc).result) { if (ordered) results.push(doc); else @@ -479,7 +481,7 @@ LocalCollection.prototype.insert = function (doc, callback) { // trigger live queries that match for (var qid in self.queries) { var query = self.queries[qid]; - if (query.selector_f(doc).result) { + if (query.selector.documentMatches(doc).result) { if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -507,22 +509,23 @@ LocalCollection.prototype.remove = function (selector, callback) { var remove = []; var queriesToRecompute = []; - var selector_f = LocalCollection._compileSelector(selector, self); + var compiledSelector = new Minimongo.Selector(selector, self); // Avoid O(n) for "remove a single doc by ID". var specificIds = LocalCollection._idsMatchedBySelector(selector); if (specificIds) { _.each(specificIds, function (id) { var strId = LocalCollection._idStringify(id); - // We still have to run selector_f, in case it's something like + // We still have to run compiledSelector, in case it's something like // {_id: "X", a: 42} - if (_.has(self.docs, strId) && selector_f(self.docs[strId]).result) + if (_.has(self.docs, strId) + && compiledSelector.documentMatches(self.docs[strId]).result) remove.push(strId); }); } else { for (var id in self.docs) { var doc = self.docs[id]; - if (selector_f(doc).result) { + if (compiledSelector.documentMatches(doc).result) { remove.push(id); } } @@ -533,7 +536,7 @@ LocalCollection.prototype.remove = function (selector, callback) { var removeId = remove[i]; var removeDoc = self.docs[removeId]; _.each(self.queries, function (query, qid) { - if (query.selector_f(removeDoc).result) { + if (query.selector.documentMatches(removeDoc).result) { if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -574,7 +577,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { } if (!options) options = {}; - var selector_f = LocalCollection._compileSelector(selector, self); + var compiledSelector = new Minimongo.Selector(selector, self); // Save the original results of any query that we might need to // _recomputeResults on, because _modifyAndNotify will mutate the objects in @@ -592,7 +595,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { for (var id in self.docs) { var doc = self.docs[id]; - var queryResult = selector_f(doc); + var queryResult = compiledSelector.documentMatches(doc); if (queryResult.result) { // XXX Should we save the original even if mod ends up being a no-op? // XXX queryResult should have arrayIndex on it, useful for '$' @@ -669,7 +672,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (var qid in self.queries) { var query = self.queries[qid]; if (query.ordered) { - matched_before[qid] = query.selector_f(doc).result; + matched_before[qid] = query.selector.documentMatches(doc).result; } else { // Because we don't support skip or limit (yet) in unordered queries, we // can just do a direct lookup. @@ -685,7 +688,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (qid in self.queries) { query = self.queries[qid]; var before = matched_before[qid]; - var after = query.selector_f(doc).result; + var after = query.selector.documentMatches(doc).result; if (query.cursor.skip || query.cursor.limit) { // We need to recompute any query where the doc may have been in the @@ -978,13 +981,3 @@ LocalCollection._makeChangedFields = function (newDoc, oldDoc) { }); return fields; }; - -// Searches $near operator in the selector recursively -// (including all $or/$and/$nor/$not branches) -// XXX this should be something determined by the selector compiler, not later -var isGeoQuery = function (selector) { - return _.any(selector, function (val, key) { - // Note: _.isObject matches objects and arrays - return key === "$near" || (_.isObject(val) && isGeoQuery(val)); - }); -}; diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 9a2c4ca0cd..d6bc96dc07 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -312,13 +312,13 @@ LocalCollection._modifiers = { // modifying that many documents, so we'll let it slide for // now - // XXX _compileSelector isn't up for the job, because we need + // XXX Minimongo.Selector isn't up for the job, because we need // to permit stuff like {$pull: {a: {$gt: 4}}}.. something // like {$gt: 4} is not normally a complete selector. // same issue as $elemMatch possibly? - var match = LocalCollection._compileSelector(arg); + var match = new Minimongo.Selector(arg); for (var i = 0; i < x.length; i++) - if (!match(x[i]).result) + if (!match.documentMatches(x[i]).result) out.push(x[i]); } else { for (var i = 0; i < x.length; i++) diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js index 69dc2045a4..5d16c82f3f 100644 --- a/packages/minimongo/package.js +++ b/packages/minimongo/package.js @@ -5,6 +5,7 @@ Package.describe({ Package.on_use(function (api) { api.export('LocalCollection'); + api.export('Minimongo'); api.export('MinimongoTest', { testOnly: true }); api.use(['underscore', 'json', 'ejson', 'ordered-dict', 'deps', 'random', 'ordered-dict']); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 641ad22139..57ae3b0033 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -33,12 +33,12 @@ var isOperatorObject = function (valueSelector) { }; -var compileValueSelector = function (valueSelector, cursor, inRoot) { +var compileValueSelector = function (valueSelector, cursor, selectorObjIfRoot) { if (valueSelector instanceof RegExp) return convertElementSelectorToBranchedSelector( regexpElementSelector(valueSelector)); else if (isOperatorObject(valueSelector)) - return operatorValueSelector(valueSelector, cursor, inRoot); + return operatorValueSelector(valueSelector, cursor, selectorObjIfRoot); else { return convertElementSelectorToBranchedSelector( equalityElementSelector(valueSelector)); @@ -96,7 +96,7 @@ var equalityElementSelector = function (elementSelector) { }; // XXX get rid of cursor when possible -var operatorValueSelector = function (valueSelector, cursor, inRoot) { +var operatorValueSelector = function (valueSelector, cursor, selectorObjIfRoot) { // Each valueSelector works separately on the various branches. So one // operator can match one branch and another can match another branch. This // is OK. @@ -105,7 +105,7 @@ var operatorValueSelector = function (valueSelector, cursor, inRoot) { _.each(valueSelector, function (operand, operator) { if (_.has(VALUE_OPERATORS, operator)) { operatorFunctions.push( - VALUE_OPERATORS[operator](operand, valueSelector, cursor, inRoot)); + VALUE_OPERATORS[operator](operand, valueSelector, cursor, selectorObjIfRoot)); } else if (_.has(ELEMENT_OPERATORS, operator)) { // XXX justify three arguments var options = ELEMENT_OPERATORS[operator]; @@ -265,9 +265,10 @@ var VALUE_OPERATORS = { return andBranchedSelectors(branchedSelectors); }, - $near: function (operand, valueSelector, cursor, inRoot) { - if (!inRoot) + $near: function (operand, valueSelector, cursor, selectorObjIfRoot) { + if (!selectorObjIfRoot) throw Error("$near can't be inside another $ operator"); + selectorObjIfRoot._isGeoQuery = true; // There are two kinds of geodata in MongoDB: coordinate pairs and // GeoJSON. They use different distance metrics, too. GeoJSON queries are @@ -710,7 +711,7 @@ LocalCollection._f = { // For unit tests. True if the given document matches the given // selector. MinimongoTest.matches = function (selector, doc) { - return (LocalCollection._compileSelector(selector))(doc).result; + return new Minimongo.Selector(selector).documentMatches(doc).result; }; @@ -867,7 +868,7 @@ var expandArraysInBranches = function (branches, skipTheArrays) { }; // The main compilation function for a given selector. -var compileDocumentSelector = function (docSelector, cursor, isRoot) { +var compileDocumentSelector = function (docSelector, cursor, selectorObjIfRoot) { var perKeySelectors = []; _.each(docSelector, function (subSelector, key) { if (key.substr(0, 1) === '$') { @@ -881,7 +882,7 @@ var compileDocumentSelector = function (docSelector, cursor, isRoot) { } else { var lookUpByIndex = LocalCollection._makeLookupFunction2(key); var valueSelectorFunc = - compileValueSelector(subSelector, cursor, isRoot); + compileValueSelector(subSelector, cursor, selectorObjIfRoot); perKeySelectors.push(function (doc, wholeDoc) { var branchValues = lookUpByIndex(doc); return valueSelectorFunc(branchValues, wholeDoc); @@ -892,11 +893,28 @@ var compileDocumentSelector = function (docSelector, cursor, isRoot) { return andCompiledDocumentSelectors(perKeySelectors); }; +// XXX doc and move around +// XXX remove 'cursor' +Minimongo.Selector = function (selector, cursor) { + var self = this; + self._isGeoQuery = false; // can get overwritten by compilation + self._docSelector = compileSelector(selector, self, cursor); +}; + +_.extend(Minimongo.Selector.prototype, { + documentMatches: function (doc) { + return this._docSelector(doc); + }, + isGeoQuery: function () { + return this._isGeoQuery; + } +}); + // Given a selector, return a function that takes one argument, a // document. It returns an object with fields // - result: bool, true if the document matches the selector // XXX add "arrayIndex" for use by update with '$' -LocalCollection._compileSelector = function (selector, cursor) { +var compileSelector = function (selector, selectorObject, cursor) { // you can pass a literal function instead of a selector if (selector instanceof Function) return function (doc) { @@ -922,7 +940,7 @@ LocalCollection._compileSelector = function (selector, cursor) { throw new Error("Invalid selector: " + selector); // XXX get rid of second argument once _distance refactored - var s = compileDocumentSelector(selector, cursor, true); + var s = compileDocumentSelector(selector, cursor, selectorObject); return function (doc) { return s(doc, doc); }; diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index 0901647b53..7dda04d6ce 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -77,7 +77,9 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) function (path) { return selector[path]; }, _.identity /*conflict resolution is no resolution*/); - var selectorFn = LocalCollection._compileSelector(selector); + // XXX we should move this function to being a method on Selector so we aren't + // recompiling over and over + var selectorCompiled = new Minimongo.Selector(selector); try { LocalCollection._modify(doc, modifier); @@ -97,7 +99,7 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) throw e; } - return selectorFn(doc).result; + return selectorCompiled.documentMatches(doc).result; }; // Returns a list of key paths the given selector is looking for diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 3319d74951..7d50fcd3e2 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -32,7 +32,7 @@ OplogObserveDriver = function (options) { self._published = new LocalCollection._IdMap; var selector = self._cursorDescription.selector; - self._selectorFn = LocalCollection._compileSelector( + self._selector = new Minimongo.Selector( self._cursorDescription.selector); var projection = self._cursorDescription.options.fields || {}; self._projectionFn = LocalCollection._compileProjection(projection); @@ -131,7 +131,7 @@ _.extend(OplogObserveDriver.prototype, { var self = this; newDoc = _.clone(newDoc); - var matchesNow = newDoc && self._selectorFn(newDoc).result; + var matchesNow = newDoc && self._selector.documentMatches(newDoc).result; if (mustMatchNow && !matchesNow) { throw Error("expected " + EJSON.stringify(newDoc) + " to match " + EJSON.stringify(self._cursorDescription)); @@ -238,7 +238,7 @@ _.extend(OplogObserveDriver.prototype, { // XXX what if selector yields? for now it can't but later it could have // $where - if (self._selectorFn(op.o).result) + if (self._selector.documentMatches(op.o).result) self._add(op.o); } else if (op.op === 'u') { // Is this a modifier ($set/$unset, which may require us to poll the From df3223cc05e862320f5ef98d1936d4f5956cf4eb Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 30 Dec 2013 16:28:31 -0800 Subject: [PATCH 069/124] Get rid of cursor._distance Now selectors don't have a link to the internals of the cursor; instead, Selector.documentMatches() return values can include a 'distance' field. (This is the dress rehearsal for also adding an 'arrayIndex' field.) Contexts that run selectors and may need to sort later now explicitly keep around a distances IdMap. Specifically, the handles associated with observeChanges calls each have a distances IdMap, and the _getRawObjects call has a temporary one (which might alias one of the observeChanges-related ones if it's being called from observeChanges, either for the initial query or to re-compute). --- packages/minimongo/minimongo.js | 69 ++++++++----- packages/minimongo/minimongo_tests.js | 17 +++- packages/minimongo/selector.js | 137 ++++++++++++-------------- packages/minimongo/sort.js | 3 +- 4 files changed, 123 insertions(+), 103 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 5e297b0dc1..13a0096bbe 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -270,10 +270,11 @@ _.extend(LocalCollection.Cursor.prototype, { if (!options._allow_unordered && !ordered && (self.skip || self.limit)) throw new Error("must use ordered observe with skip or limit"); - // XXX merge this object w/ "this" Cursor. they're the same. var query = { selector: self.selector, // not fast pathed sorter: ordered && self.sorter, + distances: ( + self.selector.isGeoQuery() && ordered && new LocalCollection._IdMap), results_snapshot: null, ordered: ordered, cursor: self, @@ -289,7 +290,7 @@ _.extend(LocalCollection.Cursor.prototype, { qid = self.collection.next_qid++; self.collection.queries[qid] = query; } - query.results = self._getRawObjects(ordered); + query.results = self._getRawObjects(ordered, query.distances); if (self.collection.paused) query.results_snapshot = (ordered ? [] : {}); @@ -374,7 +375,8 @@ _.extend(LocalCollection.Cursor.prototype, { // // If ordered is not set, returns an object mapping from ID to doc (sorter, skip // and limit should not be set). -LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { +LocalCollection.Cursor.prototype._getRawObjects = function (ordered, + distances) { var self = this; var results = ordered ? [] : {}; @@ -398,13 +400,25 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { } // slow path for arbitrary selector, sort, skip, limit - for (var id in self.collection.docs) { - var doc = self.collection.docs[id]; - if (self.selector.documentMatches(doc).result) { - if (ordered) + + // in the observeChanges case, distances is actually part of the "query" (ie, + // live results set) object. in other cases, distances is only used inside + // this function. + if (self.selector.isGeoQuery() && ordered && !distances) + distances = new LocalCollection._IdMap(); + + for (var idStringified in self.collection.docs) { + var doc = self.collection.docs[idStringified]; + var id = LocalCollection._idParse(idStringified); // XXX use more idmaps + var matchResult = self.selector.documentMatches(doc); + if (matchResult.result) { + if (ordered) { results.push(doc); - else - results[id] = doc; + if (distances && matchResult.distance !== undefined) + distances.set(id, matchResult.distance); + } else { + results[idStringified] = doc; + } } // Fast path for limited unsorted queries. if (self.limit && !self.skip && !self.sorter && @@ -416,7 +430,7 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { return results; if (self.sorter) { - var comparator = self._getComparator(); + var comparator = self.sorter.getComparator({distances: distances}); results.sort(comparator); } @@ -425,11 +439,6 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered) { return results.slice(idx_start, idx_end); }; -LocalCollection.Cursor.prototype._getComparator = function () { - var self = this; - return self.sorter.getComparator({distances: self._distance}); -}; - // XXX Maybe we need a version of observe that just calls a callback if // anything changed. LocalCollection.Cursor.prototype._depend = function (changers, _allow_unordered) { @@ -481,7 +490,10 @@ LocalCollection.prototype.insert = function (doc, callback) { // trigger live queries that match for (var qid in self.queries) { var query = self.queries[qid]; - if (query.selector.documentMatches(doc).result) { + var matchResult = query.selector.documentMatches(doc); + if (matchResult.result) { + if (query.distances && matchResult.distance !== undefined) + query.distances.set(doc._id, matchResult.distance); if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -523,10 +535,10 @@ LocalCollection.prototype.remove = function (selector, callback) { remove.push(strId); }); } else { - for (var id in self.docs) { - var doc = self.docs[id]; + for (var strId in self.docs) { + var doc = self.docs[strId]; if (compiledSelector.documentMatches(doc).result) { - remove.push(id); + remove.push(strId); } } } @@ -550,8 +562,10 @@ LocalCollection.prototype.remove = function (selector, callback) { // run live query callbacks _after_ we've removed the documents. _.each(queryRemove, function (remove) { var query = self.queries[remove.qid]; - if (query) + if (query) { + query.distances && query.distances.remove(remove.doc._id); LocalCollection._removeFromResults(query, remove.doc); + } }); _.each(queriesToRecompute, function (qid) { var query = self.queries[qid]; @@ -688,7 +702,10 @@ LocalCollection.prototype._modifyAndNotify = function ( for (qid in self.queries) { query = self.queries[qid]; var before = matched_before[qid]; - var after = query.selector.documentMatches(doc).result; + var afterMatch = query.selector.documentMatches(doc); + var after = afterMatch.result; + if (after && query.distances && afterMatch.distance !== undefined) + query.distances.set(doc._id, afterMatch.distance); if (query.cursor.skip || query.cursor.limit) { // We need to recompute any query where the doc may have been in the @@ -725,7 +742,8 @@ LocalCollection._insertInResults = function (query, doc) { query.results.push(doc); } else { var i = LocalCollection._insertInSortedList( - query.cursor._getComparator(), query.results, doc); + query.sorter.getComparator({distances: query.distances}), + query.results, doc); var next = query.results[i+1]; if (next) next = next._id; @@ -775,7 +793,8 @@ LocalCollection._updateInResults = function (query, doc, old_doc) { // changes query.results.splice(orig_idx, 1); var new_idx = LocalCollection._insertInSortedList( - query.cursor._getComparator(), query.results, doc); + query.sorter.getComparator({distances: query.distances}), + query.results, doc); if (orig_idx !== new_idx) { var next = query.results[new_idx+1]; if (next) @@ -797,7 +816,9 @@ LocalCollection._updateInResults = function (query, doc, old_doc) { LocalCollection._recomputeResults = function (query, oldResults) { if (!oldResults) oldResults = query.results; - query.results = query.cursor._getRawObjects(query.ordered); + if (query.distances) + query.distances.clear(); + query.results = query.cursor._getRawObjects(query.ordered, query.distances); if (!query.paused) { LocalCollection._diffQueryChanges( diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 4811f6dcd2..7e558932fb 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -2617,7 +2617,7 @@ Tinytest.add("minimongo - $near operator tests", function (test) { _id: "x", a: [ {b: [ - [100, 100], + [100, 100], [1, 1]]}, {b: [150, 150]}]}); coll.insert({ @@ -2635,5 +2635,20 @@ Tinytest.add("minimongo - $near operator tests", function (test) { // though the first within-1000 point in 'x' (ie, [100,100]) is farther than // 'y'. testNear([2, 2], 1000, ['x', 'y']); + + var operations = []; + var cbs = log_callbacks(operations); + var handle = coll.find({'a.b': {$near: [7,7]}}).observe(cbs); + + test.length(operations, 2); + test.equal(operations.shift(), ['added', {a:{b:[5,5]}}, 0, null]); + test.equal(operations.shift(), + ['added', {a:[{b:[[100,100],[1,1]]},{b:[150,150]}]}, 1, null]); + // This needs to be inserted in the MIDDLE of the two existing ones. + coll.insert({a: {b: [3,3]}}); + test.length(operations, 1); + test.equal(operations.shift(), ['added', {a: {b: [3, 3]}}, 1, 'x']); + + handle.stop(); }); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 57ae3b0033..fa58416e0a 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -33,12 +33,12 @@ var isOperatorObject = function (valueSelector) { }; -var compileValueSelector = function (valueSelector, cursor, selectorObjIfRoot) { +var compileValueSelector = function (valueSelector, selectorObjIfRoot) { if (valueSelector instanceof RegExp) return convertElementSelectorToBranchedSelector( regexpElementSelector(valueSelector)); else if (isOperatorObject(valueSelector)) - return operatorValueSelector(valueSelector, cursor, selectorObjIfRoot); + return operatorValueSelector(valueSelector, selectorObjIfRoot); else { return convertElementSelectorToBranchedSelector( equalityElementSelector(valueSelector)); @@ -63,7 +63,7 @@ var regexpElementSelector = function (regexp) { var convertElementSelectorToBranchedSelector = function ( elementSelector, options) { options = options || {}; - return function (branches, wholeDoc) { + return function (branches) { var expanded = branches; if (!options.dontExpandLeafArrays) { expanded = expandArraysInBranches( @@ -71,7 +71,7 @@ var convertElementSelectorToBranchedSelector = function ( } var result = _.any(expanded, function (element) { // XXX arrayIndex! need to save the winner here - return elementSelector(element.value, wholeDoc); + return elementSelector(element.value); }); return {result: result}; }; @@ -95,8 +95,7 @@ var equalityElementSelector = function (elementSelector) { }; }; -// XXX get rid of cursor when possible -var operatorValueSelector = function (valueSelector, cursor, selectorObjIfRoot) { +var operatorValueSelector = function (valueSelector, selectorObjIfRoot) { // Each valueSelector works separately on the various branches. So one // operator can match one branch and another can match another branch. This // is OK. @@ -105,7 +104,7 @@ var operatorValueSelector = function (valueSelector, cursor, selectorObjIfRoot) _.each(valueSelector, function (operand, operator) { if (_.has(VALUE_OPERATORS, operator)) { operatorFunctions.push( - VALUE_OPERATORS[operator](operand, valueSelector, cursor, selectorObjIfRoot)); + VALUE_OPERATORS[operator](operand, valueSelector, selectorObjIfRoot)); } else if (_.has(ELEMENT_OPERATORS, operator)) { // XXX justify three arguments var options = ELEMENT_OPERATORS[operator]; @@ -113,7 +112,7 @@ var operatorValueSelector = function (valueSelector, cursor, selectorObjIfRoot) options = {elementSelector: options}; operatorFunctions.push( convertElementSelectorToBranchedSelector( - options.elementSelector(operand, valueSelector, cursor), + options.elementSelector(operand, valueSelector), options)); } else { throw new Error("Unrecognized operator: " + operator); @@ -123,56 +122,40 @@ var operatorValueSelector = function (valueSelector, cursor, selectorObjIfRoot) return andBranchedSelectors(operatorFunctions); }; -// NB: this is very similar to andCompiledDocumentSelectors but that one -// "assumes" the first arg is a doc and here it "assumes" it's a branched -// value list. The code is identical for now, though. -var andBranchedSelectors = function (branchedSelectors) { - return function (branches, doc) { - // XXX arrayIndex! - var result = _.all(branchedSelectors, function (f) { - return f(branches, doc).result; - }); - return {result: result}; - }; -}; - -// XXX drop "cursor" when _distance is improved -var compileArrayOfDocumentSelectors = function (selectors, cursor) { +var compileArrayOfDocumentSelectors = function (selectors) { if (!isArray(selectors) || _.isEmpty(selectors)) throw Error("$and/$or/$nor must be nonempty array"); return _.map(selectors, function (subSelector) { if (!isPlainObject(subSelector)) throw Error("$or/$and/$nor entries need to be full objects"); - return compileDocumentSelector(subSelector, cursor); + return compileDocumentSelector(subSelector); }); }; // XXX can factor out common logic below var LOGICAL_OPERATORS = { - $and: function(subSelector, cursor) { - var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); + $and: function(subSelector) { + var selectors = compileArrayOfDocumentSelectors(subSelector); return andCompiledDocumentSelectors(selectors); }, - $or: function(subSelector, cursor) { - var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); - // XXX remove wholeDoc later - return function (doc, wholeDoc) { + $or: function(subSelector) { + var selectors = compileArrayOfDocumentSelectors(subSelector); + return function (doc) { var result = _.any(selectors, function (f) { - return f(doc, wholeDoc).result; + return f(doc).result; }); // XXX arrayIndex! return {result: result}; }; }, - $nor: function(subSelector, cursor) { - var selectors = compileArrayOfDocumentSelectors(subSelector, cursor); - // XXX remove wholeDoc later - return function (doc, wholeDoc) { + $nor: function(subSelector) { + var selectors = compileArrayOfDocumentSelectors(subSelector); + return function (doc) { var result = _.all(selectors, function (f) { - return !f(doc, wholeDoc).result; + return !f(doc).result; }); // Never set arrayIndex, because we only match if nothing in particular // "matched". @@ -205,8 +188,8 @@ var LOGICAL_OPERATORS = { var invertBranchedSelector = function (branchedSelector) { // Note that this implicitly "deMorganizes" the wrapped function. ie, it // means that ALL branch values need to fail to match innerBranchedSelector. - return function (branchValues, doc) { - var invertMe = branchedSelector(branchValues, doc); + return function (branchValues) { + var invertMe = branchedSelector(branchValues); // We explicitly choose to strip arrayIndex here: it doesn't make sense to // say "update the array element that does not match something", at least // in mongo-land. @@ -216,8 +199,8 @@ var invertBranchedSelector = function (branchedSelector) { // XXX doc var VALUE_OPERATORS = { - $not: function (operand, operator, cursor) { - return invertBranchedSelector(compileValueSelector(operand, cursor)); + $not: function (operand, operator) { + return invertBranchedSelector(compileValueSelector(operand)); }, $ne: function (operand) { return invertBranchedSelector(convertElementSelectorToBranchedSelector( @@ -265,7 +248,7 @@ var VALUE_OPERATORS = { return andBranchedSelectors(branchedSelectors); }, - $near: function (operand, valueSelector, cursor, selectorObjIfRoot) { + $near: function (operand, valueSelector, selectorObjIfRoot) { if (!selectorObjIfRoot) throw Error("$near can't be inside another $ operator"); selectorObjIfRoot._isGeoQuery = true; @@ -304,7 +287,7 @@ var VALUE_OPERATORS = { }; } - return function (branchedValues, doc) { + return function (branchedValues) { // There might be multiple points in the document that match the given // field. Only one of them needs to be within $maxDistance, but we need to // evaluate all of them and use the nearest one for the implicit sort @@ -326,13 +309,8 @@ var VALUE_OPERATORS = { minDistance = curDistance; }); if (minDistance !== null) { - if (cursor) { - if (!cursor._distance) - cursor._distance = {}; - cursor._distance[doc._id] = minDistance; - } // XXX arrayIndex! - return {result: true}; + return {result: true, distance: minDistance}; } return {result: false}; }; @@ -387,10 +365,8 @@ var makeInequality = function (cmpValueComparator) { // Each value operator is a function with args: // - operand - Anything // - operators - Object - operators on the same level (neighbours) -// - cursor - Object - original cursor // returns a function with args: // - value - a value the operator is tested against -// - doc - the whole document tested in this query var ELEMENT_OPERATORS = { $lt: makeInequality(function (cmpValue) { return cmpValue < 0; @@ -500,7 +476,7 @@ var ELEMENT_OPERATORS = { }, $elemMatch: { dontExpandLeafArrays: true, - elementSelector: function (operand, valueSelector, cursor) { + elementSelector: function (operand, valueSelector) { if (!isPlainObject(operand)) throw Error("$elemMatch need an object"); @@ -868,7 +844,7 @@ var expandArraysInBranches = function (branches, skipTheArrays) { }; // The main compilation function for a given selector. -var compileDocumentSelector = function (docSelector, cursor, selectorObjIfRoot) { +var compileDocumentSelector = function (docSelector, selectorObjIfRoot) { var perKeySelectors = []; _.each(docSelector, function (subSelector, key) { if (key.substr(0, 1) === '$') { @@ -877,15 +853,14 @@ var compileDocumentSelector = function (docSelector, cursor, selectorObjIfRoot) if (!_.has(LOGICAL_OPERATORS, key)) throw new Error("Unrecognized logical operator: " + key); // XXX rename perKeySelectors - perKeySelectors.push( - LOGICAL_OPERATORS[key](subSelector, cursor)); + perKeySelectors.push(LOGICAL_OPERATORS[key](subSelector)); } else { var lookUpByIndex = LocalCollection._makeLookupFunction2(key); var valueSelectorFunc = - compileValueSelector(subSelector, cursor, selectorObjIfRoot); - perKeySelectors.push(function (doc, wholeDoc) { + compileValueSelector(subSelector, selectorObjIfRoot); + perKeySelectors.push(function (doc) { var branchValues = lookUpByIndex(doc); - return valueSelectorFunc(branchValues, wholeDoc); + return valueSelectorFunc(branchValues); }); } }); @@ -894,11 +869,10 @@ var compileDocumentSelector = function (docSelector, cursor, selectorObjIfRoot) }; // XXX doc and move around -// XXX remove 'cursor' -Minimongo.Selector = function (selector, cursor) { +Minimongo.Selector = function (selector) { var self = this; self._isGeoQuery = false; // can get overwritten by compilation - self._docSelector = compileSelector(selector, self, cursor); + self._docSelector = compileSelector(selector, self); }; _.extend(Minimongo.Selector.prototype, { @@ -914,7 +888,7 @@ _.extend(Minimongo.Selector.prototype, { // document. It returns an object with fields // - result: bool, true if the document matches the selector // XXX add "arrayIndex" for use by update with '$' -var compileSelector = function (selector, selectorObject, cursor) { +var compileSelector = function (selector, selectorObject) { // you can pass a literal function instead of a selector if (selector instanceof Function) return function (doc) { @@ -939,11 +913,7 @@ var compileSelector = function (selector, selectorObject, cursor) { EJSON.isBinary(selector)) throw new Error("Invalid selector: " + selector); - // XXX get rid of second argument once _distance refactored - var s = compileDocumentSelector(selector, cursor, selectorObject); - return function (doc) { - return s(doc, doc); - }; + return compileDocumentSelector(selector, selectorObject); }; var matchesNothingSelector = function (docOrBranchedValues) { @@ -954,21 +924,36 @@ var matchesEverythingSelector = function (docOrBranchedValues) { return {result: true}; }; -var andCompiledDocumentSelectors = function (selectors) { - // XXX simplify to not involve 'arguments' once _distance is refactored - return function (/*doc, sometimes wholeDoc*/) { - var args = _.toArray(arguments); + +// NB: We are cheating and using this function to implement "AND" for both +// "document selectors" and "branched selectors". They have the same return type +// but the argument is different: for the former it's a whole doc, whereas for +// the latter it's an array of "branches" that match a given key path. +var andSomeSelectors = function (branchedSelectors) { + return function (branches, doc) { // XXX arrayIndex! - var result = _.all(selectors, function (f) { - return f.apply(null, args).result; + var ret = {}; + var distance; + ret.result = _.all(branchedSelectors, function (f) { + var subResult = f(branches, doc); + // Copy a 'distance' number out of the first sub-selector that has + // one. Yes, this means that if there are multiple $near fields in a + // query, something arbitrary happens; this appears to be consistent with + // Mongo. + if (subResult.result && subResult.distance !== undefined + && distance === undefined) { + distance = subResult.distance; + } + return subResult.result; }); - return {result: result}; + if (ret.result && distance !== undefined) + ret.distance = distance; + return ret; }; }; +var andCompiledDocumentSelectors = andSomeSelectors; +var andBranchedSelectors = andSomeSelectors; -// Remaining to update: -// - $near/$maxDistance -// - make sure to switch _distance to be an idmap // Remaining to implement: // - $all with $elemMatch diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index 1aca9bf8ce..03767a6d8c 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -114,9 +114,8 @@ Sorter.prototype.getComparator = function (options) { }; } - var distances = _.clone(options.distances); return function (a, b) { - return distances[a._id] - distances[b._id]; + return options.distances.get(a._id) - options.distances.get(b._id); }; }; From bea993e1cc9d1171ecab10507b0008f0dfaf99a2 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 00:13:04 -0800 Subject: [PATCH 070/124] style --- packages/minimongo/selector.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index fa58416e0a..1232dcb360 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -135,12 +135,12 @@ var compileArrayOfDocumentSelectors = function (selectors) { // XXX can factor out common logic below var LOGICAL_OPERATORS = { - $and: function(subSelector) { + $and: function (subSelector) { var selectors = compileArrayOfDocumentSelectors(subSelector); return andCompiledDocumentSelectors(selectors); }, - $or: function(subSelector) { + $or: function (subSelector) { var selectors = compileArrayOfDocumentSelectors(subSelector); return function (doc) { var result = _.any(selectors, function (f) { @@ -151,7 +151,7 @@ var LOGICAL_OPERATORS = { }; }, - $nor: function(subSelector) { + $nor: function (subSelector) { var selectors = compileArrayOfDocumentSelectors(subSelector); return function (doc) { var result = _.all(selectors, function (f) { @@ -163,7 +163,7 @@ var LOGICAL_OPERATORS = { }; }, - $where: function(selectorValue) { + $where: function (selectorValue) { if (!(selectorValue instanceof Function)) { // XXX MongoDB seems to have more complex logic to decide where or or not // to add "return"; not sure exactly what it is. From 3e6ef8580eeb99714aa164a1da8e8823409e661c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 12:00:16 -0800 Subject: [PATCH 071/124] Can always implement $all-with-$elemMatch later (It's an error now) --- packages/minimongo/selector.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 1232dcb360..45a72c2b56 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -954,6 +954,3 @@ var andSomeSelectors = function (branchedSelectors) { var andCompiledDocumentSelectors = andSomeSelectors; var andBranchedSelectors = andSomeSelectors; - -// Remaining to implement: -// - $all with $elemMatch From 87a71a9298bb3788813cf8e9f44082e3f756fba5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 12:13:15 -0800 Subject: [PATCH 072/124] Use new lookup function directly in Sorter --- packages/minimongo/selector.js | 2 +- packages/minimongo/sort.js | 44 +++++++++++++--------------------- 2 files changed, 18 insertions(+), 28 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 45a72c2b56..f2e3c96e40 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -821,7 +821,7 @@ LocalCollection._makeLookupFunction = function (key) { }; }; -var expandArraysInBranches = function (branches, skipTheArrays) { +expandArraysInBranches = function (branches, skipTheArrays) { var branchesOut = []; _.each(branches, function (branch) { var thisIsArray = isArray(branch.value); diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index 03767a6d8c..9faa7d1f44 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -21,12 +21,12 @@ Sorter = function (spec) { for (var i = 0; i < spec.length; i++) { if (typeof spec[i] === "string") { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(spec[i]), + lookup: LocalCollection._makeLookupFunction2(spec[i]), ascending: true }); } else { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(spec[i][0]), + lookup: LocalCollection._makeLookupFunction2(spec[i][0]), ascending: spec[i][1] !== "desc" }); } @@ -34,7 +34,7 @@ Sorter = function (spec) { } else if (typeof spec === "object") { for (var key in spec) { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction(key), + lookup: LocalCollection._makeLookupFunction2(key), ascending: spec[key] >= 0 }); } @@ -54,34 +54,24 @@ Sorter = function (spec) { // too. (ie, we do a single level of flattening on branchValues, then find the // min/max.) var reduceValue = function (branchValues, findMin) { - var reduced; + // Expand any leaf arrays that we find, and ignore those arrays themselves. + branchValues = expandArraysInBranches(branchValues, true); + var reduced = undefined; var first = true; // Iterate over all the values found in all the branches, and if a value is // an array itself, iterate over the values in the array separately. _.each(branchValues, function (branchValue) { - // Value not an array? Pretend it is. - if (!isArray(branchValue)) - branchValue = [branchValue]; - // Value is an empty array? Pretend it was missing, since that's where it - // should be sorted. - if (isArray(branchValue) && branchValue.length === 0) - branchValue = [undefined]; - _.each(branchValue, function (value) { - // We should get here at least once: lookup functions return non-empty - // arrays, so the outer loop runs at least once, and we prevented - // branchValue from being an empty array. - if (first) { - reduced = value; - first = false; - } else { - // Compare the value we found to the value we found so far, saving it - // if it's less (for an ascending sort) or more (for a descending - // sort). - var cmp = LocalCollection._f._cmp(reduced, value); - if ((findMin && cmp > 0) || (!findMin && cmp < 0)) - reduced = value; - } - }); + if (first) { + reduced = branchValue.value; + first = false; + } else { + // Compare the value we found to the value we found so far, saving it + // if it's less (for an ascending sort) or more (for a descending + // sort). + var cmp = LocalCollection._f._cmp(reduced, branchValue.value); + if ((findMin && cmp > 0) || (!findMin && cmp < 0)) + reduced = branchValue.value; + } }); return reduced; }; From 5aa5a5c05e26f883555e2ae7be2d014d6744be3c Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 12:46:28 -0800 Subject: [PATCH 073/124] Rename: makeLookupFunction, get rid of legacy one --- packages/minimongo/minimongo_tests.js | 11 ++++++++--- packages/minimongo/selector.js | 14 ++++---------- packages/minimongo/sort.js | 6 +++--- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 7e558932fb..089d477561 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -243,19 +243,24 @@ Tinytest.add("minimongo - misc", function (test) { }); Tinytest.add("minimongo - lookup", function (test) { - var lookupA = LocalCollection._makeLookupFunction('a'); + var justValues = function (lookupFunction) { + return function (doc) { + return _.pluck(lookupFunction(doc), 'value'); + }; + }; + var lookupA = justValues(MinimongoTest.makeLookupFunction('a')); test.equal(lookupA({}), [undefined]); test.equal(lookupA({a: 1}), [1]); test.equal(lookupA({a: [1]}), [[1]]); - var lookupAX = LocalCollection._makeLookupFunction('a.x'); + var lookupAX = justValues(MinimongoTest.makeLookupFunction('a.x')); test.equal(lookupAX({a: {x: 1}}), [1]); test.equal(lookupAX({a: {x: [1]}}), [[1]]); test.equal(lookupAX({a: 5}), [undefined]); test.equal(lookupAX({a: [{x: 1}, {x: [2]}, {y: 3}]}), [1, [2], undefined]); - var lookupA0X = LocalCollection._makeLookupFunction('a.0.x'); + var lookupA0X = justValues(MinimongoTest.makeLookupFunction('a.0.x')); test.equal(lookupA0X({a: [{x: 1}]}), [1, undefined]); test.equal(lookupA0X({a: [{x: [1]}]}), [[1], undefined]); test.equal(lookupA0X({a: 5}), [undefined]); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index f2e3c96e40..d27ee2d653 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -723,13 +723,13 @@ numericKey = function (s) { // {x: [2]}, // {y: 3}]}) // returns [1, [2], undefined] -LocalCollection._makeLookupFunction2 = function (key) { +makeLookupFunction = function (key) { var parts = key.split('.'); var firstPart = parts.length ? parts[0] : ''; var firstPartIsNumeric = numericKey(firstPart); var lookupRest; if (parts.length > 1) { - lookupRest = LocalCollection._makeLookupFunction2(parts.slice(1).join('.')); + lookupRest = makeLookupFunction(parts.slice(1).join('.')); } // Doc will always be a plain object or an array. @@ -813,13 +813,7 @@ LocalCollection._makeLookupFunction2 = function (key) { return result; }; }; - -LocalCollection._makeLookupFunction = function (key) { - var real = LocalCollection._makeLookupFunction2(key); - return function (doc) { - return _.pluck(real(doc), 'value'); - }; -}; +MinimongoTest.makeLookupFunction = makeLookupFunction; expandArraysInBranches = function (branches, skipTheArrays) { var branchesOut = []; @@ -855,7 +849,7 @@ var compileDocumentSelector = function (docSelector, selectorObjIfRoot) { // XXX rename perKeySelectors perKeySelectors.push(LOGICAL_OPERATORS[key](subSelector)); } else { - var lookUpByIndex = LocalCollection._makeLookupFunction2(key); + var lookUpByIndex = makeLookupFunction(key); var valueSelectorFunc = compileValueSelector(subSelector, selectorObjIfRoot); perKeySelectors.push(function (doc) { diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index 9faa7d1f44..adc6708cee 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -21,12 +21,12 @@ Sorter = function (spec) { for (var i = 0; i < spec.length; i++) { if (typeof spec[i] === "string") { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction2(spec[i]), + lookup: makeLookupFunction(spec[i]), ascending: true }); } else { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction2(spec[i][0]), + lookup: makeLookupFunction(spec[i][0]), ascending: spec[i][1] !== "desc" }); } @@ -34,7 +34,7 @@ Sorter = function (spec) { } else if (typeof spec === "object") { for (var key in spec) { sortSpecParts.push({ - lookup: LocalCollection._makeLookupFunction2(key), + lookup: makeLookupFunction(key), ascending: spec[key] >= 0 }); } From 2f9aecc77d12bfb464751f2fb5cbe869f678d303 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 13:08:17 -0800 Subject: [PATCH 074/124] Improve lookup function docs and tests --- packages/minimongo/minimongo_tests.js | 51 ++++++++++++-------- packages/minimongo/selector.js | 67 +++++++++++++++++---------- 2 files changed, 74 insertions(+), 44 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 089d477561..2afe9b8dec 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -243,29 +243,40 @@ Tinytest.add("minimongo - misc", function (test) { }); Tinytest.add("minimongo - lookup", function (test) { - var justValues = function (lookupFunction) { - return function (doc) { - return _.pluck(lookupFunction(doc), 'value'); - }; - }; - var lookupA = justValues(MinimongoTest.makeLookupFunction('a')); - test.equal(lookupA({}), [undefined]); - test.equal(lookupA({a: 1}), [1]); - test.equal(lookupA({a: [1]}), [[1]]); + var lookupA = MinimongoTest.makeLookupFunction('a'); + test.equal(lookupA({}), [{value: undefined}]); + test.equal(lookupA({a: 1}), [{value: 1}]); + test.equal(lookupA({a: [1]}), [{value: [1]}]); - var lookupAX = justValues(MinimongoTest.makeLookupFunction('a.x')); - test.equal(lookupAX({a: {x: 1}}), [1]); - test.equal(lookupAX({a: {x: [1]}}), [[1]]); - test.equal(lookupAX({a: 5}), [undefined]); + var lookupAX = MinimongoTest.makeLookupFunction('a.x'); + test.equal(lookupAX({a: {x: 1}}), [{value: 1}]); + test.equal(lookupAX({a: {x: [1]}}), [{value: [1]}]); + test.equal(lookupAX({a: 5}), [{value: undefined}]); test.equal(lookupAX({a: [{x: 1}, {x: [2]}, {y: 3}]}), - [1, [2], undefined]); + [{value: 1, arrayIndex: 0}, + {value: [2], arrayIndex: 1}, + {value: undefined, arrayIndex: 2}]); - var lookupA0X = justValues(MinimongoTest.makeLookupFunction('a.0.x')); - test.equal(lookupA0X({a: [{x: 1}]}), [1, undefined]); - test.equal(lookupA0X({a: [{x: [1]}]}), [[1], undefined]); - test.equal(lookupA0X({a: 5}), [undefined]); - test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), - [1, undefined, undefined, undefined]); + var lookupA0X = MinimongoTest.makeLookupFunction('a.0.x'); + test.equal(lookupA0X({a: [{x: 1}]}), [ + // From interpreting '0' as "0th array element". + {value: 1, arrayIndex: 0}, + // From interpreting '0' as "after branching in the array, look in the + // object {x:1} for a field named 0". + {value: undefined, arrayIndex: 0}]); + test.equal(lookupA0X({a: [{x: [1]}]}), [ + {value: [1], arrayIndex: 0}, + {value: undefined, arrayIndex: 0}]); + test.equal(lookupA0X({a: 5}), [{value: undefined}]); + test.equal(lookupA0X({a: [{x: 1}, {x: [2]}, {y: 3}]}), [ + // From interpreting '0' as "0th array element". + {value: 1, arrayIndex: 0}, + // From interpreting '0' as "after branching in the array, look in the + // object {x:1} for a field named 0". + {value: undefined, arrayIndex: 0}, + {value: undefined, arrayIndex: 1}, + {value: undefined, arrayIndex: 2} + ]); }); Tinytest.add("minimongo - selector_compiler", function (test) { diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d27ee2d653..49237fd0d8 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -696,33 +696,41 @@ numericKey = function (s) { return /^[0-9]+$/.test(s); }; -// XXX redoc -// XXX be aware that Sorter currently assumes that lookup functions -// return non-empty arrays but that is no longer the case -// _makeLookupFunction(key) returns a lookup function. +// makeLookupFunction(key) returns a lookup function. // // A lookup function takes in a document and returns an array of matching -// values. If no arrays are found while looking up the key, this array will -// have exactly one value (possibly 'undefined', if some segment of the key was -// not found). +// branches. If no arrays are found while looking up the key, this array will +// have exactly one branches (possibly 'undefined', if some segment of the key +// was not found). // // If arrays are found in the middle, this can have more than one element, since // we "branch". When we "branch", if there are more key segments to look up, // then we only pursue branches that are plain objects (not arrays or scalars). -// This means we can actually end up with no entries! +// This means we can actually end up with no branches! // -// At the top level, you may only pass in a plain object. +// We do *NOT* branch on arrays that are found at the end (ie, at the last +// dotted member of the key). We just return that array; if you want to +// effectively "branch" over the array's values, post-process the lookup +// function with expandArraysInBranches. // -// _makeLookupFunction('a.x')({a: {x: 1}}) returns [1] -// _makeLookupFunction('a.x')({a: {x: [1]}}) returns [[1]] -// _makeLookupFunction('a.x')({a: 5}) returns [undefined] -// _makeLookupFunction('a.x')({a: [5]}) returns [] -// _makeLookupFunction('a.x')({a: [{x: 1}, -// [], -// 4, -// {x: [2]}, -// {y: 3}]}) -// returns [1, [2], undefined] +// Each branch is an object with keys: +// - value: the value at the branch +// - dontIterate: an optional bool; if true, it means that 'value' is an array +// that expandArraysInBranches should NOT expand. This specifically happens +// when there is a numeric index in the key, and ensures the +// perhaps-surprising MongoDB behavior where {'a.0': 5} does NOT +// match {a: [[5]]}. +// - arrayIndex: if any array indexing was done during lookup (either +// due to explicit numeric indices or implicit branching), this will +// be the FIRST (outermost) array index used; it is undefined or absent +// if no array index is used. (Make sure to check its value vs undefined, +// not just for truth, since '0' is a legit array index!) This is used +// to implement the '$' modifier feature. +// +// At the top level, you may only pass in a plain object or arraym. +// +// See the text 'minimongo - lookup' for some examples of what lookup functions +// return. makeLookupFunction = function (key) { var parts = key.split('.'); var firstPart = parts.length ? parts[0] : ''; @@ -732,6 +740,14 @@ makeLookupFunction = function (key) { lookupRest = makeLookupFunction(parts.slice(1).join('.')); } + var elideUnnecessaryFields = function (retVal) { + if (!retVal.dontIterate) + delete retVal.dontIterate; + if (retVal.arrayIndex === undefined) + delete retVal.arrayIndex; + return retVal; + }; + // Doc will always be a plain object or an array. // apply an explicit numeric index, an array. return function (doc, firstArrayIndex) { @@ -764,9 +780,10 @@ makeLookupFunction = function (key) { // selectors to iterate over it. eg, {'a.0': 5} does not match {a: [[5]]}. // So in that case, we mark the return value as "don't iterate". if (!lookupRest) { - return [{value: firstLevel, - dontIterate: isArray(doc) && isArray(firstLevel), - arrayIndex: firstArrayIndex}]; + return [elideUnnecessaryFields({ + value: firstLevel, + dontIterate: isArray(doc) && isArray(firstLevel), + arrayIndex: firstArrayIndex})]; } // We need to dig deeper. But if we can't, because what we've found is not @@ -776,8 +793,10 @@ makeLookupFunction = function (key) { // return a single `undefined` (which can, for example, match via equality // with `null`). if (!isIndexable(firstLevel)) { - return isArray(doc) ? [] : [{value: undefined, - arrayIndex: firstArrayIndex}]; + if (isArray(doc)) + return []; + return [elideUnnecessaryFields({value: undefined, + arrayIndex: firstArrayIndex})]; } var result = []; From e3c8bf65ab00cd379c8dd5a029d309a68a450e09 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 13:20:06 -0800 Subject: [PATCH 075/124] Start to reorder/rename selector.js --- packages/minimongo/selector.js | 553 +++++++++++++++++---------------- 1 file changed, 286 insertions(+), 267 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 49237fd0d8..9b7f3ae812 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -1,35 +1,85 @@ -// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as -// arrays. -// XXX maybe this should be EJSON.isArray -isArray = function (x) { - return _.isArray(x) && !EJSON.isBinary(x); +// The minimongo selector compiler! + + +// Main entry point. +// var selector = new Minimongo.Selector({a: {$gt: 5}}); +// if (selector.documentMatches({a: 7})) ... +Minimongo.Selector = function (selector) { + var self = this; + self._isGeoQuery = false; // can get overwritten by compilation + self._docSelector = self._compileSelector(selector); }; -// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about -// RegExp -// XXX note that _type(undefined) === 3!!!! -var isPlainObject = function (x) { - return x && LocalCollection._f._type(x) === 3; -}; +_.extend(Minimongo.Selector.prototype, { + documentMatches: function (doc) { + return this._docSelector(doc); + }, + isGeoQuery: function () { + return this._isGeoQuery; + }, -var isIndexable = function (x) { - return isArray(x) || isPlainObject(x); -}; + // Given a selector, return a function that takes one argument, a + // document. It returns an object with fields + // - result: bool, true if the document matches the selector + // - distance: if a $near was evaluated, the distance to the point + // XXX add "arrayIndex" for use by update with '$' + _compileSelector: function (selector) { + var self = this; + // you can pass a literal function instead of a selector + if (selector instanceof Function) + return function (doc) { + return {result: !!selector.call(doc)}; + }; -var isOperatorObject = function (valueSelector) { - if (!isPlainObject(valueSelector)) - return false; + // shorthand -- scalars match _id + if (LocalCollection._selectorIsId(selector)) { + return function (doc) { + return {result: EJSON.equals(doc._id, selector)}; + }; + } - var theseAreOperators = undefined; - _.each(valueSelector, function (value, selKey) { - var thisIsOperator = selKey.substr(0, 1) === '$'; - if (theseAreOperators === undefined) { - theseAreOperators = thisIsOperator; - } else if (theseAreOperators !== thisIsOperator) { - throw new Error("Inconsistent selector: " + valueSelector); + // protect against dangerous selectors. falsey and {_id: falsey} are both + // likely programmer error, and not what you want, particularly for + // destructive operations. + if (!selector || (('_id' in selector) && !selector._id)) + return matchesNothingSelector; + + // Top level can't be an array or true or binary. + if (typeof(selector) === 'boolean' || isArray(selector) || + EJSON.isBinary(selector)) + throw new Error("Invalid selector: " + selector); + + return compileDocumentSelector(selector, self); + } +}); + + +// Takes in a selector that could match a full document (eg, the original +// selector). Returns a function mapping doc->{result, distance, arrayIndex}. +// +// If this is the root document selector (ie, not wrapped in $and or the like), +// then selectorObjIfRoot is the Selector object. (This is used by $near.) +var compileDocumentSelector = function (docSelector, selectorObjIfRoot) { + var individualMatchers = []; + _.each(docSelector, function (subSelector, key) { + if (key.substr(0, 1) === '$') { + // Outer operators are either logical operators (they recurse back into + // this function), or $where. + if (!_.has(LOGICAL_OPERATORS, key)) + throw new Error("Unrecognized logical operator: " + key); + individualMatchers.push(LOGICAL_OPERATORS[key](subSelector)); + } else { + var lookUpByIndex = makeLookupFunction(key); + var valueSelectorFunc = + compileValueSelector(subSelector, selectorObjIfRoot); + individualMatchers.push(function (doc) { + var branchValues = lookUpByIndex(doc); + return valueSelectorFunc(branchValues); + }); } }); - return !!theseAreOperators; // {} has no operators + + return andDocumentMatchers(individualMatchers); }; @@ -137,7 +187,7 @@ var compileArrayOfDocumentSelectors = function (selectors) { var LOGICAL_OPERATORS = { $and: function (subSelector) { var selectors = compileArrayOfDocumentSelectors(subSelector); - return andCompiledDocumentSelectors(selectors); + return andDocumentMatchers(selectors); }, $or: function (subSelector) { @@ -519,171 +569,6 @@ var ELEMENT_OPERATORS = { } }; -// helpers used by compiled selector code -LocalCollection._f = { - // XXX for _all and _in, consider building 'inquery' at compile time.. - - _type: function (v) { - if (typeof v === "number") - return 1; - if (typeof v === "string") - return 2; - if (typeof v === "boolean") - return 8; - if (isArray(v)) - return 4; - if (v === null) - return 10; - if (v instanceof RegExp) - // note that typeof(/x/) === "object" - return 11; - if (typeof v === "function") - return 13; - if (v instanceof Date) - return 9; - if (EJSON.isBinary(v)) - return 5; - if (v instanceof LocalCollection._ObjectID) - return 7; - return 3; // object - - // XXX support some/all of these: - // 14, symbol - // 15, javascript code with scope - // 16, 18: 32-bit/64-bit integer - // 17, timestamp - // 255, minkey - // 127, maxkey - }, - - // deep equality test: use for literal document and array matches - _equal: function (a, b) { - return EJSON.equals(a, b, {keyOrderSensitive: true}); - }, - - // maps a type code to a value that can be used to sort values of - // different types - _typeorder: function (t) { - // http://www.mongodb.org/display/DOCS/What+is+the+Compare+Order+for+BSON+Types - // XXX what is the correct sort position for Javascript code? - // ('100' in the matrix below) - // XXX minkey/maxkey - return [-1, // (not a type) - 1, // number - 2, // string - 3, // object - 4, // array - 5, // binary - -1, // deprecated - 6, // ObjectID - 7, // bool - 8, // Date - 0, // null - 9, // RegExp - -1, // deprecated - 100, // JS code - 2, // deprecated (symbol) - 100, // JS code - 1, // 32-bit int - 8, // Mongo timestamp - 1 // 64-bit int - ][t]; - }, - - // compare two values of unknown type according to BSON ordering - // semantics. (as an extension, consider 'undefined' to be less than - // any other value.) return negative if a is less, positive if b is - // less, or 0 if equal - _cmp: function (a, b) { - if (a === undefined) - return b === undefined ? 0 : -1; - if (b === undefined) - return 1; - var ta = LocalCollection._f._type(a); - var tb = LocalCollection._f._type(b); - var oa = LocalCollection._f._typeorder(ta); - var ob = LocalCollection._f._typeorder(tb); - if (oa !== ob) - return oa < ob ? -1 : 1; - if (ta !== tb) - // XXX need to implement this if we implement Symbol or integers, or - // Timestamp - throw Error("Missing type coercion logic in _cmp"); - if (ta === 7) { // ObjectID - // Convert to string. - ta = tb = 2; - a = a.toHexString(); - b = b.toHexString(); - } - if (ta === 9) { // Date - // Convert to millis. - ta = tb = 1; - a = a.getTime(); - b = b.getTime(); - } - - if (ta === 1) // double - return a - b; - if (tb === 2) // string - return a < b ? -1 : (a === b ? 0 : 1); - if (ta === 3) { // Object - // this could be much more efficient in the expected case ... - var to_array = function (obj) { - var ret = []; - for (var key in obj) { - ret.push(key); - ret.push(obj[key]); - } - return ret; - }; - return LocalCollection._f._cmp(to_array(a), to_array(b)); - } - if (ta === 4) { // Array - for (var i = 0; ; i++) { - if (i === a.length) - return (i === b.length) ? 0 : -1; - if (i === b.length) - return 1; - var s = LocalCollection._f._cmp(a[i], b[i]); - if (s !== 0) - return s; - } - } - if (ta === 5) { // binary - // Surprisingly, a small binary blob is always less than a large one in - // Mongo. - if (a.length !== b.length) - return a.length - b.length; - for (i = 0; i < a.length; i++) { - if (a[i] < b[i]) - return -1; - if (a[i] > b[i]) - return 1; - } - return 0; - } - if (ta === 8) { // boolean - if (a) return b ? 0 : 1; - return b ? -1 : 0; - } - if (ta === 10) // null - return 0; - if (ta === 11) // regexp - throw Error("Sorting not supported on regular expression"); // XXX - // 13: javascript code - // 14: symbol - // 15: javascript code with scope - // 16: 32-bit integer - // 17: timestamp - // 18: 64-bit integer - // 255: minkey - // 127: maxkey - if (ta === 13) // javascript code - throw Error("Sorting not supported on Javascript code"); // XXX - throw Error("Unknown type to sort"); - } -}; - // For unit tests. True if the given document matches the given // selector. MinimongoTest.matches = function (selector, doc) { @@ -856,79 +741,6 @@ expandArraysInBranches = function (branches, skipTheArrays) { return branchesOut; }; -// The main compilation function for a given selector. -var compileDocumentSelector = function (docSelector, selectorObjIfRoot) { - var perKeySelectors = []; - _.each(docSelector, function (subSelector, key) { - if (key.substr(0, 1) === '$') { - // Outer operators are either logical operators (they recurse back into - // this function), or $where. - if (!_.has(LOGICAL_OPERATORS, key)) - throw new Error("Unrecognized logical operator: " + key); - // XXX rename perKeySelectors - perKeySelectors.push(LOGICAL_OPERATORS[key](subSelector)); - } else { - var lookUpByIndex = makeLookupFunction(key); - var valueSelectorFunc = - compileValueSelector(subSelector, selectorObjIfRoot); - perKeySelectors.push(function (doc) { - var branchValues = lookUpByIndex(doc); - return valueSelectorFunc(branchValues); - }); - } - }); - - return andCompiledDocumentSelectors(perKeySelectors); -}; - -// XXX doc and move around -Minimongo.Selector = function (selector) { - var self = this; - self._isGeoQuery = false; // can get overwritten by compilation - self._docSelector = compileSelector(selector, self); -}; - -_.extend(Minimongo.Selector.prototype, { - documentMatches: function (doc) { - return this._docSelector(doc); - }, - isGeoQuery: function () { - return this._isGeoQuery; - } -}); - -// Given a selector, return a function that takes one argument, a -// document. It returns an object with fields -// - result: bool, true if the document matches the selector -// XXX add "arrayIndex" for use by update with '$' -var compileSelector = function (selector, selectorObject) { - // you can pass a literal function instead of a selector - if (selector instanceof Function) - return function (doc) { - return {result: !!selector.call(doc)}; - }; - - // shorthand -- scalars match _id - if (LocalCollection._selectorIsId(selector)) { - return function (doc) { - return {result: EJSON.equals(doc._id, selector)}; - }; - } - - // protect against dangerous selectors. falsey and {_id: falsey} are both - // likely programmer error, and not what you want, particularly for - // destructive operations. - if (!selector || (('_id' in selector) && !selector._id)) - return matchesNothingSelector; - - // Top level can't be an array or true or binary. - if (typeof(selector) === 'boolean' || isArray(selector) || - EJSON.isBinary(selector)) - throw new Error("Invalid selector: " + selector); - - return compileDocumentSelector(selector, selectorObject); -}; - var matchesNothingSelector = function (docOrBranchedValues) { return {result: false}; }; @@ -943,12 +755,17 @@ var matchesEverythingSelector = function (docOrBranchedValues) { // but the argument is different: for the former it's a whole doc, whereas for // the latter it's an array of "branches" that match a given key path. var andSomeSelectors = function (branchedSelectors) { - return function (branches, doc) { + if (branchedSelectors.length === 0) + return matchesEverythingSelector; + if (branchedSelectors.length === 1) + return branchedSelectors[0]; + + return function (branches) { // XXX arrayIndex! var ret = {}; var distance; ret.result = _.all(branchedSelectors, function (f) { - var subResult = f(branches, doc); + var subResult = f(branches); // Copy a 'distance' number out of the first sub-selector that has // one. Yes, this means that if there are multiple $near fields in a // query, something arbitrary happens; this appears to be consistent with @@ -965,5 +782,207 @@ var andSomeSelectors = function (branchedSelectors) { }; }; -var andCompiledDocumentSelectors = andSomeSelectors; +var andDocumentMatchers = andSomeSelectors; var andBranchedSelectors = andSomeSelectors; + + +// HELPER FUNCTIONS + +// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as +// arrays. +// XXX maybe this should be EJSON.isArray +isArray = function (x) { + return _.isArray(x) && !EJSON.isBinary(x); +}; + +// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about +// RegExp +// XXX note that _type(undefined) === 3!!!! +var isPlainObject = function (x) { + return x && LocalCollection._f._type(x) === 3; +}; + +var isIndexable = function (x) { + return isArray(x) || isPlainObject(x); +}; + +var isOperatorObject = function (valueSelector) { + if (!isPlainObject(valueSelector)) + return false; + + var theseAreOperators = undefined; + _.each(valueSelector, function (value, selKey) { + var thisIsOperator = selKey.substr(0, 1) === '$'; + if (theseAreOperators === undefined) { + theseAreOperators = thisIsOperator; + } else if (theseAreOperators !== thisIsOperator) { + throw new Error("Inconsistent selector: " + valueSelector); + } + }); + return !!theseAreOperators; // {} has no operators +}; + +// helpers used by compiled selector code +LocalCollection._f = { + // XXX for _all and _in, consider building 'inquery' at compile time.. + + _type: function (v) { + if (typeof v === "number") + return 1; + if (typeof v === "string") + return 2; + if (typeof v === "boolean") + return 8; + if (isArray(v)) + return 4; + if (v === null) + return 10; + if (v instanceof RegExp) + // note that typeof(/x/) === "object" + return 11; + if (typeof v === "function") + return 13; + if (v instanceof Date) + return 9; + if (EJSON.isBinary(v)) + return 5; + if (v instanceof LocalCollection._ObjectID) + return 7; + return 3; // object + + // XXX support some/all of these: + // 14, symbol + // 15, javascript code with scope + // 16, 18: 32-bit/64-bit integer + // 17, timestamp + // 255, minkey + // 127, maxkey + }, + + // deep equality test: use for literal document and array matches + _equal: function (a, b) { + return EJSON.equals(a, b, {keyOrderSensitive: true}); + }, + + // maps a type code to a value that can be used to sort values of + // different types + _typeorder: function (t) { + // http://www.mongodb.org/display/DOCS/What+is+the+Compare+Order+for+BSON+Types + // XXX what is the correct sort position for Javascript code? + // ('100' in the matrix below) + // XXX minkey/maxkey + return [-1, // (not a type) + 1, // number + 2, // string + 3, // object + 4, // array + 5, // binary + -1, // deprecated + 6, // ObjectID + 7, // bool + 8, // Date + 0, // null + 9, // RegExp + -1, // deprecated + 100, // JS code + 2, // deprecated (symbol) + 100, // JS code + 1, // 32-bit int + 8, // Mongo timestamp + 1 // 64-bit int + ][t]; + }, + + // compare two values of unknown type according to BSON ordering + // semantics. (as an extension, consider 'undefined' to be less than + // any other value.) return negative if a is less, positive if b is + // less, or 0 if equal + _cmp: function (a, b) { + if (a === undefined) + return b === undefined ? 0 : -1; + if (b === undefined) + return 1; + var ta = LocalCollection._f._type(a); + var tb = LocalCollection._f._type(b); + var oa = LocalCollection._f._typeorder(ta); + var ob = LocalCollection._f._typeorder(tb); + if (oa !== ob) + return oa < ob ? -1 : 1; + if (ta !== tb) + // XXX need to implement this if we implement Symbol or integers, or + // Timestamp + throw Error("Missing type coercion logic in _cmp"); + if (ta === 7) { // ObjectID + // Convert to string. + ta = tb = 2; + a = a.toHexString(); + b = b.toHexString(); + } + if (ta === 9) { // Date + // Convert to millis. + ta = tb = 1; + a = a.getTime(); + b = b.getTime(); + } + + if (ta === 1) // double + return a - b; + if (tb === 2) // string + return a < b ? -1 : (a === b ? 0 : 1); + if (ta === 3) { // Object + // this could be much more efficient in the expected case ... + var to_array = function (obj) { + var ret = []; + for (var key in obj) { + ret.push(key); + ret.push(obj[key]); + } + return ret; + }; + return LocalCollection._f._cmp(to_array(a), to_array(b)); + } + if (ta === 4) { // Array + for (var i = 0; ; i++) { + if (i === a.length) + return (i === b.length) ? 0 : -1; + if (i === b.length) + return 1; + var s = LocalCollection._f._cmp(a[i], b[i]); + if (s !== 0) + return s; + } + } + if (ta === 5) { // binary + // Surprisingly, a small binary blob is always less than a large one in + // Mongo. + if (a.length !== b.length) + return a.length - b.length; + for (i = 0; i < a.length; i++) { + if (a[i] < b[i]) + return -1; + if (a[i] > b[i]) + return 1; + } + return 0; + } + if (ta === 8) { // boolean + if (a) return b ? 0 : 1; + return b ? -1 : 0; + } + if (ta === 10) // null + return 0; + if (ta === 11) // regexp + throw Error("Sorting not supported on regular expression"); // XXX + // 13: javascript code + // 14: symbol + // 15: javascript code with scope + // 16: 32-bit integer + // 17: timestamp + // 18: 64-bit integer + // 255: minkey + // 127: maxkey + if (ta === 13) // javascript code + throw Error("Sorting not supported on Javascript code"); // XXX + throw Error("Unknown type to sort"); + } +}; From b80d0df73631b865cb5592c2b47a60f92278c5af Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 13:26:38 -0800 Subject: [PATCH 076/124] Rename Minimongo.Selector -> Minimongo.Matcher The idea will be that "selector" will always mean the EJSON representation of a selector, and "matcher" will be some compiled form (whether Minimongo.Matcher or the various lambdas that make it up) --- packages/minimongo/minimongo.js | 34 +++++++++---------- packages/minimongo/modify.js | 6 ++-- packages/minimongo/selector.js | 12 ++++--- packages/minimongo/selector_modifier.js | 6 ++-- .../mongo-livedata/oplog_observe_driver.js | 6 ++-- 5 files changed, 33 insertions(+), 31 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 13a0096bbe..0ab739c57e 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -91,12 +91,12 @@ LocalCollection.Cursor = function (collection, selector, options) { if (LocalCollection._selectorIsId(selector)) { // stash for fast path self.selector_id = LocalCollection._idStringify(selector); - self.selector = new Minimongo.Selector(selector, self); + self.matcher = new Minimongo.Matcher(selector, self); self.sorter = undefined; } else { self.selector_id = undefined; - self.selector = new Minimongo.Selector(selector, self); - self.sorter = (self.selector.isGeoQuery() || options.sort) ? + self.matcher = new Minimongo.Matcher(selector, self); + self.sorter = (self.matcher.isGeoQuery() || options.sort) ? new Sorter(options.sort || []) : null; } self.skip = options.skip; @@ -271,10 +271,10 @@ _.extend(LocalCollection.Cursor.prototype, { throw new Error("must use ordered observe with skip or limit"); var query = { - selector: self.selector, // not fast pathed + matcher: self.matcher, // not fast pathed sorter: ordered && self.sorter, distances: ( - self.selector.isGeoQuery() && ordered && new LocalCollection._IdMap), + self.matcher.isGeoQuery() && ordered && new LocalCollection._IdMap), results_snapshot: null, ordered: ordered, cursor: self, @@ -404,13 +404,13 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered, // in the observeChanges case, distances is actually part of the "query" (ie, // live results set) object. in other cases, distances is only used inside // this function. - if (self.selector.isGeoQuery() && ordered && !distances) + if (self.matcher.isGeoQuery() && ordered && !distances) distances = new LocalCollection._IdMap(); for (var idStringified in self.collection.docs) { var doc = self.collection.docs[idStringified]; var id = LocalCollection._idParse(idStringified); // XXX use more idmaps - var matchResult = self.selector.documentMatches(doc); + var matchResult = self.matcher.documentMatches(doc); if (matchResult.result) { if (ordered) { results.push(doc); @@ -490,7 +490,7 @@ LocalCollection.prototype.insert = function (doc, callback) { // trigger live queries that match for (var qid in self.queries) { var query = self.queries[qid]; - var matchResult = query.selector.documentMatches(doc); + var matchResult = query.matcher.documentMatches(doc); if (matchResult.result) { if (query.distances && matchResult.distance !== undefined) query.distances.set(doc._id, matchResult.distance); @@ -521,23 +521,23 @@ LocalCollection.prototype.remove = function (selector, callback) { var remove = []; var queriesToRecompute = []; - var compiledSelector = new Minimongo.Selector(selector, self); + var matcher = new Minimongo.Matcher(selector, self); // Avoid O(n) for "remove a single doc by ID". var specificIds = LocalCollection._idsMatchedBySelector(selector); if (specificIds) { _.each(specificIds, function (id) { var strId = LocalCollection._idStringify(id); - // We still have to run compiledSelector, in case it's something like + // We still have to run matcher, in case it's something like // {_id: "X", a: 42} if (_.has(self.docs, strId) - && compiledSelector.documentMatches(self.docs[strId]).result) + && matcher.documentMatches(self.docs[strId]).result) remove.push(strId); }); } else { for (var strId in self.docs) { var doc = self.docs[strId]; - if (compiledSelector.documentMatches(doc).result) { + if (matcher.documentMatches(doc).result) { remove.push(strId); } } @@ -548,7 +548,7 @@ LocalCollection.prototype.remove = function (selector, callback) { var removeId = remove[i]; var removeDoc = self.docs[removeId]; _.each(self.queries, function (query, qid) { - if (query.selector.documentMatches(removeDoc).result) { + if (query.matcher.documentMatches(removeDoc).result) { if (query.cursor.skip || query.cursor.limit) queriesToRecompute.push(qid); else @@ -591,7 +591,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { } if (!options) options = {}; - var compiledSelector = new Minimongo.Selector(selector, self); + var matcher = new Minimongo.Matcher(selector, self); // Save the original results of any query that we might need to // _recomputeResults on, because _modifyAndNotify will mutate the objects in @@ -609,7 +609,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { for (var id in self.docs) { var doc = self.docs[id]; - var queryResult = compiledSelector.documentMatches(doc); + var queryResult = matcher.documentMatches(doc); if (queryResult.result) { // XXX Should we save the original even if mod ends up being a no-op? // XXX queryResult should have arrayIndex on it, useful for '$' @@ -686,7 +686,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (var qid in self.queries) { var query = self.queries[qid]; if (query.ordered) { - matched_before[qid] = query.selector.documentMatches(doc).result; + matched_before[qid] = query.matcher.documentMatches(doc).result; } else { // Because we don't support skip or limit (yet) in unordered queries, we // can just do a direct lookup. @@ -702,7 +702,7 @@ LocalCollection.prototype._modifyAndNotify = function ( for (qid in self.queries) { query = self.queries[qid]; var before = matched_before[qid]; - var afterMatch = query.selector.documentMatches(doc); + var afterMatch = query.matcher.documentMatches(doc); var after = afterMatch.result; if (after && query.distances && afterMatch.distance !== undefined) query.distances.set(doc._id, afterMatch.distance); diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index d6bc96dc07..3be2c9447d 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -312,13 +312,13 @@ LocalCollection._modifiers = { // modifying that many documents, so we'll let it slide for // now - // XXX Minimongo.Selector isn't up for the job, because we need + // XXX Minimongo.Matcher isn't up for the job, because we need // to permit stuff like {$pull: {a: {$gt: 4}}}.. something // like {$gt: 4} is not normally a complete selector. // same issue as $elemMatch possibly? - var match = new Minimongo.Selector(arg); + var matcher = new Minimongo.Matcher(arg); for (var i = 0; i < x.length; i++) - if (!match.documentMatches(x[i]).result) + if (!matcher.documentMatches(x[i]).result) out.push(x[i]); } else { for (var i = 0; i < x.length; i++) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 9b7f3ae812..d77e284162 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -2,15 +2,17 @@ // Main entry point. -// var selector = new Minimongo.Selector({a: {$gt: 5}}); -// if (selector.documentMatches({a: 7})) ... -Minimongo.Selector = function (selector) { +// var matcher = new Minimongo.Matcher({a: {$gt: 5}}); +// if (matcher.documentMatches({a: 7})) ... +// (Terminology: a "selector" is the EJSON object representing a selctor; a +// "matcher" is its compiled form.) +Minimongo.Matcher = function (selector) { var self = this; self._isGeoQuery = false; // can get overwritten by compilation self._docSelector = self._compileSelector(selector); }; -_.extend(Minimongo.Selector.prototype, { +_.extend(Minimongo.Matcher.prototype, { documentMatches: function (doc) { return this._docSelector(doc); }, @@ -572,7 +574,7 @@ var ELEMENT_OPERATORS = { // For unit tests. True if the given document matches the given // selector. MinimongoTest.matches = function (selector, doc) { - return new Minimongo.Selector(selector).documentMatches(doc).result; + return new Minimongo.Matcher(selector).documentMatches(doc).result; }; diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index 7dda04d6ce..877b8b338d 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -77,9 +77,9 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) function (path) { return selector[path]; }, _.identity /*conflict resolution is no resolution*/); - // XXX we should move this function to being a method on Selector so we aren't + // XXX we should move this function to being a method on Matcher so we aren't // recompiling over and over - var selectorCompiled = new Minimongo.Selector(selector); + var matcher = new Minimongo.Matcher(selector); try { LocalCollection._modify(doc, modifier); @@ -99,7 +99,7 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) throw e; } - return selectorCompiled.documentMatches(doc).result; + return matcher.documentMatches(doc).result; }; // Returns a list of key paths the given selector is looking for diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 7d50fcd3e2..60be922a16 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -32,7 +32,7 @@ OplogObserveDriver = function (options) { self._published = new LocalCollection._IdMap; var selector = self._cursorDescription.selector; - self._selector = new Minimongo.Selector( + self._matcher = new Minimongo.Matcher( self._cursorDescription.selector); var projection = self._cursorDescription.options.fields || {}; self._projectionFn = LocalCollection._compileProjection(projection); @@ -131,7 +131,7 @@ _.extend(OplogObserveDriver.prototype, { var self = this; newDoc = _.clone(newDoc); - var matchesNow = newDoc && self._selector.documentMatches(newDoc).result; + var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result; if (mustMatchNow && !matchesNow) { throw Error("expected " + EJSON.stringify(newDoc) + " to match " + EJSON.stringify(self._cursorDescription)); @@ -238,7 +238,7 @@ _.extend(OplogObserveDriver.prototype, { // XXX what if selector yields? for now it can't but later it could have // $where - if (self._selector.documentMatches(op.o).result) + if (self._matcher.documentMatches(op.o).result) self._add(op.o); } else if (op.op === 'u') { // Is this a modifier ($set/$unset, which may require us to poll the From e762fbc04564eeb1abb9220b1a674dba416a92c5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 13:51:36 -0800 Subject: [PATCH 077/124] Finish reorganizing/renaming in selector.js --- packages/minimongo/minimongo_tests.js | 8 +- packages/minimongo/selector.js | 265 +++++++++++++----------- packages/minimongo/selector_modifier.js | 10 +- 3 files changed, 152 insertions(+), 131 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 2afe9b8dec..e7cc36a5b0 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -280,13 +280,13 @@ Tinytest.add("minimongo - lookup", function (test) { }); Tinytest.add("minimongo - selector_compiler", function (test) { - var matches = function (should_match, selector, doc) { - var does_match = MinimongoTest.matches(selector, doc); - if (does_match != should_match) { + var matches = function (shouldMatch, selector, doc) { + var doesMatch = new Minimongo.Matcher(selector).documentMatches(doc).result; + if (doesMatch != shouldMatch) { // XXX super janky test.fail({type: "minimongo-ordering", message: "minimongo match failure: document " + - (should_match ? "should match, but doesn't" : + (shouldMatch ? "should match, but doesn't" : "shouldn't match, but does"), selector: JSON.stringify(selector), document: JSON.stringify(doc) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d77e284162..9004e83c7b 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -1,30 +1,39 @@ // The minimongo selector compiler! +// Terminology: +// - a "selector" is the EJSON object representing a selector +// - a "matcher" is its compiled form (whether a full Minimongo.Matcher +// object or one of the component lambdas that matches parts of it) +// - a "result object" is an object with a "result" field and maybe +// distance and arrayIndex. +// - a "branched value" is an object with a "value" field and maybe +// "dontIterate" and "arrayIndex". +// - a "document" is a top-level object that can be stored in a collection. +// - a "lookup function" is a function that takes in a document and returns +// an array of "branched values". +// - a "branched matcher" maps from an array of branched values to a result +// object. +// - an "element matcher" maps from a single value to a bool. // Main entry point. // var matcher = new Minimongo.Matcher({a: {$gt: 5}}); // if (matcher.documentMatches({a: 7})) ... -// (Terminology: a "selector" is the EJSON object representing a selctor; a -// "matcher" is its compiled form.) Minimongo.Matcher = function (selector) { var self = this; self._isGeoQuery = false; // can get overwritten by compilation - self._docSelector = self._compileSelector(selector); + self._docMatcher = self._compileSelector(selector); }; _.extend(Minimongo.Matcher.prototype, { documentMatches: function (doc) { - return this._docSelector(doc); + return this._docMatcher(doc); }, isGeoQuery: function () { return this._isGeoQuery; }, // Given a selector, return a function that takes one argument, a - // document. It returns an object with fields - // - result: bool, true if the document matches the selector - // - distance: if a $near was evaluated, the distance to the point - // XXX add "arrayIndex" for use by update with '$' + // document. It returns a result object. _compileSelector: function (selector) { var self = this; // you can pass a literal function instead of a selector @@ -44,7 +53,7 @@ _.extend(Minimongo.Matcher.prototype, { // likely programmer error, and not what you want, particularly for // destructive operations. if (!selector || (('_id' in selector) && !selector._id)) - return matchesNothingSelector; + return nothingMatcher; // Top level can't be an array or true or binary. if (typeof(selector) === 'boolean' || isArray(selector) || @@ -57,47 +66,71 @@ _.extend(Minimongo.Matcher.prototype, { // Takes in a selector that could match a full document (eg, the original -// selector). Returns a function mapping doc->{result, distance, arrayIndex}. +// selector). Returns a function mapping document->result object. // // If this is the root document selector (ie, not wrapped in $and or the like), -// then selectorObjIfRoot is the Selector object. (This is used by $near.) -var compileDocumentSelector = function (docSelector, selectorObjIfRoot) { - var individualMatchers = []; +// then matcherIfRoot is the Matcher object. (This is used by $near.) +var compileDocumentSelector = function (docSelector, matcherIfRoot) { + var docMatchers = []; _.each(docSelector, function (subSelector, key) { if (key.substr(0, 1) === '$') { // Outer operators are either logical operators (they recurse back into // this function), or $where. if (!_.has(LOGICAL_OPERATORS, key)) throw new Error("Unrecognized logical operator: " + key); - individualMatchers.push(LOGICAL_OPERATORS[key](subSelector)); + docMatchers.push(LOGICAL_OPERATORS[key](subSelector)); } else { var lookUpByIndex = makeLookupFunction(key); - var valueSelectorFunc = - compileValueSelector(subSelector, selectorObjIfRoot); - individualMatchers.push(function (doc) { + var valueMatcher = + compileValueSelector(subSelector, matcherIfRoot); + docMatchers.push(function (doc) { var branchValues = lookUpByIndex(doc); - return valueSelectorFunc(branchValues); + return valueMatcher(branchValues); }); } }); - return andDocumentMatchers(individualMatchers); + return andDocumentMatchers(docMatchers); }; - -var compileValueSelector = function (valueSelector, selectorObjIfRoot) { +// Takes in a selector that could match a key-indexed value in a document; eg, +// {$gt: 5, $lt: 9}, or a regular expression, or any non-expression object (to +// indicate equality). Returns a branched matcher: a function mapping +// [branched value]->result object. +var compileValueSelector = function (valueSelector, matcherIfRoot) { if (valueSelector instanceof RegExp) - return convertElementSelectorToBranchedSelector( - regexpElementSelector(valueSelector)); + return convertElementMatcherToBranchedMatcher( + regexpElementMatcher(valueSelector)); else if (isOperatorObject(valueSelector)) - return operatorValueSelector(valueSelector, selectorObjIfRoot); + return operatorBranchedMatcher(valueSelector, matcherIfRoot); else { - return convertElementSelectorToBranchedSelector( - equalityElementSelector(valueSelector)); + return convertElementMatcherToBranchedMatcher( + equalityElementMatcher(valueSelector)); } }; -var regexpElementSelector = function (regexp) { +// Given an element matcher (which evaluates a single value), returns a branched +// value (which evaluates the element matcher on all the branches and returns a +// more structured return value possibly including arrayIndex). +var convertElementMatcherToBranchedMatcher = function ( + elementMatcher, options) { + options = options || {}; + return function (branches) { + var expanded = branches; + if (!options.dontExpandLeafArrays) { + expanded = expandArraysInBranches( + branches, options.dontIncludeLeafArrays); + } + var result = _.any(expanded, function (element) { + // XXX arrayIndex! need to save the winner here + return elementMatcher(element.value); + }); + return {result: result}; + }; +}; + +// Takes a RegExp object and returns an element matcher. +var regexpElementMatcher = function (regexp) { return function (value) { if (value instanceof RegExp) { // Comparing two regexps means seeing if the regexps are identical @@ -111,25 +144,9 @@ var regexpElementSelector = function (regexp) { }; }; - -var convertElementSelectorToBranchedSelector = function ( - elementSelector, options) { - options = options || {}; - return function (branches) { - var expanded = branches; - if (!options.dontExpandLeafArrays) { - expanded = expandArraysInBranches( - branches, options.dontIncludeLeafArrays); - } - var result = _.any(expanded, function (element) { - // XXX arrayIndex! need to save the winner here - return elementSelector(element.value); - }); - return {result: result}; - }; -}; - -var equalityElementSelector = function (elementSelector) { +// Takes something that is not an operator object and returns an element matcher +// for equality with that thing. +var equalityElementMatcher = function (elementSelector) { if (isOperatorObject(elementSelector)) throw Error("Can't create equalityValueSelector for operator object"); @@ -147,31 +164,32 @@ var equalityElementSelector = function (elementSelector) { }; }; -var operatorValueSelector = function (valueSelector, selectorObjIfRoot) { +// Takes an operator object (an object with $ keys) and returns a branched +// matcher for it. +var operatorBranchedMatcher = function (valueSelector, matcherIfRoot) { // Each valueSelector works separately on the various branches. So one // operator can match one branch and another can match another branch. This // is OK. - var operatorFunctions = []; + var operatorMatchers = []; _.each(valueSelector, function (operand, operator) { if (_.has(VALUE_OPERATORS, operator)) { - operatorFunctions.push( - VALUE_OPERATORS[operator](operand, valueSelector, selectorObjIfRoot)); + operatorMatchers.push( + VALUE_OPERATORS[operator](operand, valueSelector, matcherIfRoot)); } else if (_.has(ELEMENT_OPERATORS, operator)) { - // XXX justify three arguments var options = ELEMENT_OPERATORS[operator]; if (typeof options === 'function') - options = {elementSelector: options}; - operatorFunctions.push( - convertElementSelectorToBranchedSelector( - options.elementSelector(operand, valueSelector), + options = {compileElementSelector: options}; + operatorMatchers.push( + convertElementMatcherToBranchedMatcher( + options.compileElementSelector(operand, valueSelector), options)); } else { throw new Error("Unrecognized operator: " + operator); } }); - return andBranchedSelectors(operatorFunctions); + return andBranchedMatchers(operatorMatchers); }; var compileArrayOfDocumentSelectors = function (selectors) { @@ -184,18 +202,17 @@ var compileArrayOfDocumentSelectors = function (selectors) { }); }; - -// XXX can factor out common logic below +// Operators that appear at the top level of a document selector. var LOGICAL_OPERATORS = { $and: function (subSelector) { - var selectors = compileArrayOfDocumentSelectors(subSelector); - return andDocumentMatchers(selectors); + var matchers = compileArrayOfDocumentSelectors(subSelector); + return andDocumentMatchers(matchers); }, $or: function (subSelector) { - var selectors = compileArrayOfDocumentSelectors(subSelector); + var matchers = compileArrayOfDocumentSelectors(subSelector); return function (doc) { - var result = _.any(selectors, function (f) { + var result = _.any(matchers, function (f) { return f(doc).result; }); // XXX arrayIndex! @@ -204,9 +221,9 @@ var LOGICAL_OPERATORS = { }, $nor: function (subSelector) { - var selectors = compileArrayOfDocumentSelectors(subSelector); + var matchers = compileArrayOfDocumentSelectors(subSelector); return function (doc) { - var result = _.all(selectors, function (f) { + var result = _.all(matchers, function (f) { return !f(doc).result; }); // Never set arrayIndex, because we only match if nothing in particular @@ -237,11 +254,12 @@ var LOGICAL_OPERATORS = { } }; -var invertBranchedSelector = function (branchedSelector) { - // Note that this implicitly "deMorganizes" the wrapped function. ie, it - // means that ALL branch values need to fail to match innerBranchedSelector. +// Returns a branched matcher that matches iff the given matcher does not. +// Note that this implicitly "deMorganizes" the wrapped function. ie, it +// means that ALL branch values need to fail to match innerBranchedMatcher. +var invertBranchedMatcher = function (branchedMatcher) { return function (branchValues) { - var invertMe = branchedSelector(branchValues); + var invertMe = branchedMatcher(branchValues); // We explicitly choose to strip arrayIndex here: it doesn't make sense to // say "update the array element that does not match something", at least // in mongo-land. @@ -249,61 +267,64 @@ var invertBranchedSelector = function (branchedSelector) { }; }; -// XXX doc +// Operators that (unlike LOGICAL_OPERATORS) pertain to individual paths in a +// document, but (unlike ELEMENT_OPERATORS) do not have a simple definition as +// "match each branched value independently and combine with +// convertElementMatcherToBranchedMatcher". var VALUE_OPERATORS = { - $not: function (operand, operator) { - return invertBranchedSelector(compileValueSelector(operand)); + $not: function (operand) { + return invertBranchedMatcher(compileValueSelector(operand)); }, $ne: function (operand) { - return invertBranchedSelector(convertElementSelectorToBranchedSelector( - equalityElementSelector(operand))); + return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( + equalityElementMatcher(operand))); }, $nin: function (operand) { - return invertBranchedSelector(convertElementSelectorToBranchedSelector( + return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( ELEMENT_OPERATORS.$in(operand))); }, $exists: function (operand) { - var exists = convertElementSelectorToBranchedSelector(function (value) { + var exists = convertElementMatcherToBranchedMatcher(function (value) { return value !== undefined; }); - return operand ? exists : invertBranchedSelector(exists); + return operand ? exists : invertBranchedMatcher(exists); }, // $options just provides options for $regex; its logic is inside $regex $options: function (operand, valueSelector) { if (!valueSelector.$regex) throw Error("$options needs a $regex"); - return matchesEverythingSelector; + return everythingMatcher; }, // $maxDistance is basically an argument to $near $maxDistance: function (operand, valueSelector) { if (!valueSelector.$near) throw Error("$maxDistance needs a $near"); - return matchesEverythingSelector; + return everythingMatcher; }, $all: function (operand) { if (!isArray(operand)) throw Error("$all requires array"); // Not sure why, but this seems to be what MongoDB does. if (_.isEmpty(operand)) - return matchesNothingSelector; + return nothingMatcher; - var branchedSelectors = []; + var branchedMatchers = []; _.each(operand, function (criterion) { // XXX handle $all/$elemMatch combination if (isOperatorObject(criterion)) throw Error("no $ expressions in $all"); // This is always a regexp or equality selector. - branchedSelectors.push(compileValueSelector(criterion)); + branchedMatchers.push(compileValueSelector(criterion)); }); - // andBranchedSelectors does NOT require all selectors to return true on the + // andBranchedMatchers does NOT require all selectors to return true on the // SAME branch. - return andBranchedSelectors(branchedSelectors); + return andBranchedMatchers(branchedMatchers); }, - $near: function (operand, valueSelector, selectorObjIfRoot) { - if (!selectorObjIfRoot) + $near: function (operand, valueSelector, matcherIfRoot) { + if (!matcherIfRoot) throw Error("$near can't be inside another $ operator"); - selectorObjIfRoot._isGeoQuery = true; + matcherIfRoot._isGeoQuery = true; // There are two kinds of geodata in MongoDB: coordinate pairs and // GeoJSON. They use different distance metrics, too. GeoJSON queries are @@ -369,6 +390,7 @@ var VALUE_OPERATORS = { } }; +// Helpers for $near. var distanceCoordinatePairs = function (a, b) { a = pointToArray(a); b = pointToArray(b); @@ -385,6 +407,7 @@ var pointToArray = function (point) { return _.map(point, _.identity); }; +// Helper for $lt/$gt/$lte/$gte. var makeInequality = function (cmpValueComparator) { return function (operand) { // Arrays never compare false with non-arrays for any inequality. @@ -413,12 +436,16 @@ var makeInequality = function (cmpValueComparator) { }; }; -// XXX redoc -// Each value operator is a function with args: -// - operand - Anything -// - operators - Object - operators on the same level (neighbours) -// returns a function with args: -// - value - a value the operator is tested against +// Each element selector is a function with args: +// - operand - the "right hand side" of the operator +// - valueSelector - the "context" for the operator (so that $regex can find +// $options) +// Or is an object with an compileElementSelector field (the above) and optional +// flags dontExpandLeafArrays and dontIncludeLeafArrays which control if +// expandArraysInBranches is called and if it takes an optional argument. +// +// An element selector compiler returns a function mapping a single value to +// bool. var ELEMENT_OPERATORS = { $lt: makeInequality(function (cmpValue) { return cmpValue < 0; @@ -449,21 +476,21 @@ var ELEMENT_OPERATORS = { if (!isArray(operand)) throw Error("$in needs an array"); - var elementSelectors = []; + var elementMatchers = []; _.each(operand, function (option) { if (option instanceof RegExp) - elementSelectors.push(regexpElementSelector(option)); + elementMatchers.push(regexpElementMatcher(option)); else if (isOperatorObject(option)) throw Error("cannot nest $ under $in"); else - elementSelectors.push(equalityElementSelector(option)); + elementMatchers.push(equalityElementMatcher(option)); }); return function (value) { // Allow {a: {$in: [null]}} to match when 'a' does not exist. if (value === undefined) value = null; - return _.any(elementSelectors, function (e) { + return _.any(elementMatchers, function (e) { return e(value); }); }; @@ -473,7 +500,7 @@ var ELEMENT_OPERATORS = { // don't want to consider the element [5,5] in the leaf array [[5,5]] as a // possible value. dontExpandLeafArrays: true, - elementSelector: function (operand) { + compileElementSelector: function (operand) { if (typeof operand === 'string') { // Don't ask me why, but by experimentation, this seems to be what Mongo // does. @@ -492,7 +519,7 @@ var ELEMENT_OPERATORS = { // {$type: 4}}. Thus, when we see a leaf array, we *should* expand it but // should *not* include it itself. dontIncludeLeafArrays: true, - elementSelector: function (operand) { + compileElementSelector: function (operand) { if (typeof operand !== 'number') throw Error("$type needs a number"); return function (value) { @@ -524,11 +551,11 @@ var ELEMENT_OPERATORS = { } else { regexp = new RegExp(operand); } - return regexpElementSelector(regexp); + return regexpElementMatcher(regexp); }, $elemMatch: { dontExpandLeafArrays: true, - elementSelector: function (operand, valueSelector) { + compileElementSelector: function (operand) { if (!isPlainObject(operand)) throw Error("$elemMatch need an object"); @@ -571,18 +598,6 @@ var ELEMENT_OPERATORS = { } }; -// For unit tests. True if the given document matches the given -// selector. -MinimongoTest.matches = function (selector, doc) { - return new Minimongo.Matcher(selector).documentMatches(doc).result; -}; - - -// string can be converted to integer -numericKey = function (s) { - return /^[0-9]+$/.test(s); -}; - // makeLookupFunction(key) returns a lookup function. // // A lookup function takes in a document and returns an array of matching @@ -621,7 +636,7 @@ numericKey = function (s) { makeLookupFunction = function (key) { var parts = key.split('.'); var firstPart = parts.length ? parts[0] : ''; - var firstPartIsNumeric = numericKey(firstPart); + var firstPartIsNumeric = isNumericKey(firstPart); var lookupRest; if (parts.length > 1) { lookupRest = makeLookupFunction(parts.slice(1).join('.')); @@ -743,22 +758,22 @@ expandArraysInBranches = function (branches, skipTheArrays) { return branchesOut; }; -var matchesNothingSelector = function (docOrBranchedValues) { +var nothingMatcher = function (docOrBranchedValues) { return {result: false}; }; -var matchesEverythingSelector = function (docOrBranchedValues) { +var everythingMatcher = function (docOrBranchedValues) { return {result: true}; }; // NB: We are cheating and using this function to implement "AND" for both -// "document selectors" and "branched selectors". They have the same return type +// "document matchers" and "branched matchers". They both return result objects // but the argument is different: for the former it's a whole doc, whereas for -// the latter it's an array of "branches" that match a given key path. -var andSomeSelectors = function (branchedSelectors) { +// the latter it's an array of "branched values". +var andSomeMatchers = function (branchedSelectors) { if (branchedSelectors.length === 0) - return matchesEverythingSelector; + return everythingMatcher; if (branchedSelectors.length === 1) return branchedSelectors[0]; @@ -768,7 +783,7 @@ var andSomeSelectors = function (branchedSelectors) { var distance; ret.result = _.all(branchedSelectors, function (f) { var subResult = f(branches); - // Copy a 'distance' number out of the first sub-selector that has + // Copy a 'distance' number out of the first sub-matcher that has // one. Yes, this means that if there are multiple $near fields in a // query, something arbitrary happens; this appears to be consistent with // Mongo. @@ -784,8 +799,8 @@ var andSomeSelectors = function (branchedSelectors) { }; }; -var andDocumentMatchers = andSomeSelectors; -var andBranchedSelectors = andSomeSelectors; +var andDocumentMatchers = andSomeMatchers; +var andBranchedMatchers = andSomeMatchers; // HELPER FUNCTIONS @@ -824,6 +839,12 @@ var isOperatorObject = function (valueSelector) { return !!theseAreOperators; // {} has no operators }; + +// string can be converted to integer +isNumericKey = function (s) { + return /^[0-9]+$/.test(s); +}; + // helpers used by compiled selector code LocalCollection._f = { // XXX for _all and _in, consider building 'inquery' at compile time.. diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index 877b8b338d..6ea0b28313 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -19,17 +19,17 @@ LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) { var i = 0, j = 0; while (i < sel.length && j < mod.length) { - if (numericKey(sel[i]) && numericKey(mod[j])) { + if (isNumericKey(sel[i]) && isNumericKey(mod[j])) { // foo.4.bar selector affected by foo.4 modifier // foo.3.bar selector unaffected by foo.4 modifier if (sel[i] === mod[j]) i++, j++; else return false; - } else if (numericKey(sel[i])) { + } else if (isNumericKey(sel[i])) { // foo.4.bar selector unaffected by foo.bar modifier return false; - } else if (numericKey(mod[j])) { + } else if (isNumericKey(mod[j])) { j++; } else if (sel[i] === mod[j]) i++, j++; @@ -45,7 +45,7 @@ LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) { getPathsWithoutNumericKeys = function (sel) { return _.map(getPaths(sel), function (path) { - return _.reject(path.split('.'), numericKey).join('.'); + return _.reject(path.split('.'), isNumericKey).join('.'); }); }; @@ -117,7 +117,7 @@ var getPaths = MinimongoTest.getSelectorPaths = function (sel) { }; function pathHasNumericKeys (path) { - return _.any(path.split('.'), numericKey); + return _.any(path.split('.'), isNumericKey); } function isLiteralSelector (selector) { From a5cd5eb43542c35b92fefcfbb7bdd338205318b8 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 31 Dec 2013 13:55:20 -0800 Subject: [PATCH 078/124] add comment about sort vs query --- packages/minimongo/sort.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index adc6708cee..dc2b838a5a 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -53,6 +53,13 @@ Sorter = function (spec) { // findMin). Each value can itself be an array, and we look at its values // too. (ie, we do a single level of flattening on branchValues, then find the // min/max.) + // + // XXX This is actually wrong! In fact, the whole attempt to compile sort + // functions independently of selectors is wrong. In MongoDB, if you have + // documents {_id: 'x', a: [1, 10]} and {_id: 'y', a: [5, 15]}, + // then C.find({}, {sort: {a: 1}}) puts x before y (1 comes before 5). + // But C.find({a: {$gt: 3}}, {sort: {a: 1}}) puts y before x (1 does not match + // the selector, and 5 comes before 10). var reduceValue = function (branchValues, findMin) { // Expand any leaf arrays that we find, and ignore those arrays themselves. branchValues = expandArraysInBranches(branchValues, true); From b44855d70d82309d5c8ddda2cc23e1fa9f885e3a Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Tue, 31 Dec 2013 18:38:03 -0800 Subject: [PATCH 079/124] Send user agent info with galaxy logins --- tools/auth.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/auth.js b/tools/auth.js index 88704073f9..8bb65091e5 100644 --- a/tools/auth.js +++ b/tools/auth.js @@ -309,7 +309,9 @@ var logInToGalaxy = function (galaxyName) { method: 'GET', strictSSL: true, headers: { - cookie: 'GALAXY_OAUTH_SESSION=' + session + cookie: 'GALAXY_OAUTH_SESSION=' + session + + '; GALAXY_USER_AGENT_TOOL=' + + encodeURIComponent(JSON.stringify(utils.getAgentInfo())) } }); var body = JSON.parse(galaxyResult.body); From fde0a90b9bfd05c931ac0acc4652634864d91e21 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 2 Jan 2014 14:45:43 -0800 Subject: [PATCH 080/124] Add _noYieldsAllowed safety belts to Deps functions. Because we want to start experimenting with using Deps.autorun on the server. --- packages/deps/deps.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/deps/deps.js b/packages/deps/deps.js index 3df6381b38..ef9629cbad 100644 --- a/packages/deps/deps.js +++ b/packages/deps/deps.js @@ -98,7 +98,9 @@ _.extend(Deps.Computation.prototype, { var g = function () { Deps.nonreactive(function () { - f(self); + return Meteor._noYieldsAllowed(function () { + f(self); + }); }); }; @@ -284,7 +286,9 @@ _.extend(Deps, { throw new Error('Deps.autorun requires a function argument'); constructingComputation = true; - var c = new Deps.Computation(f, Deps.currentComputation); + var c = new Deps.Computation(function (c) { + Meteor._noYieldsAllowed(_.bind(f, this, c)); + }, Deps.currentComputation); if (Deps.active) Deps.onInvalidate(function () { From 63b13ff6650096c3ec213217d7f9327a6edbd3b8 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 15:37:19 -0800 Subject: [PATCH 081/124] Clear distances passed into _getRawObjects Add comment suggested by Naomi --- packages/minimongo/minimongo.js | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 0ab739c57e..f1cf205d7f 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -375,6 +375,13 @@ _.extend(LocalCollection.Cursor.prototype, { // // If ordered is not set, returns an object mapping from ID to doc (sorter, skip // and limit should not be set). +// +// If ordered is set and this cursor is a $near geoquery, then this function +// will use an _IdMap to track each distance from the $near argument point in +// order to use it as a sort key. If an _IdMap is passed in the 'distances' +// argument, this function will clear it and use it for this purpose (otherwise +// it will just create its own _IdMap). The observeChanges implementation uses +// this to remember the distances after this function returns. LocalCollection.Cursor.prototype._getRawObjects = function (ordered, distances) { var self = this; @@ -404,8 +411,12 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered, // in the observeChanges case, distances is actually part of the "query" (ie, // live results set) object. in other cases, distances is only used inside // this function. - if (self.matcher.isGeoQuery() && ordered && !distances) - distances = new LocalCollection._IdMap(); + if (self.matcher.isGeoQuery() && ordered) { + if (distances) + distances.clear(); + else + distances = new LocalCollection._IdMap(); + } for (var idStringified in self.collection.docs) { var doc = self.collection.docs[idStringified]; From b383f2cd6a20ab21a1957afe08c0c7e53c79ea48 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 15:37:49 -0800 Subject: [PATCH 082/124] Use $near distances as lowest-priority sort key Previously, $near was only used in the absence of a sort specifier; now, it's also used as a tie-breaker when there is a sort specifier. (Tested: this matches MongoDB.) --- packages/minimongo/minimongo_tests.js | 15 +++++++- packages/minimongo/sort.js | 53 ++++++++++++++------------- 2 files changed, 41 insertions(+), 27 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index e7cc36a5b0..6af81b9c1a 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -2631,6 +2631,7 @@ Tinytest.add("minimongo - $near operator tests", function (test) { coll = new LocalCollection(); coll.insert({ _id: "x", + k: 9, a: [ {b: [ [100, 100], @@ -2638,6 +2639,7 @@ Tinytest.add("minimongo - $near operator tests", function (test) { {b: [150, 150]}]}); coll.insert({ _id: "y", + k: 9, a: {b: [5, 5]}}); var testNear = function (near, md, expected) { test.equal( @@ -2652,14 +2654,23 @@ Tinytest.add("minimongo - $near operator tests", function (test) { // 'y'. testNear([2, 2], 1000, ['x', 'y']); + // Ensure that distance is used as a tie-breaker for sort. + test.equal( + _.pluck(coll.find({'a.b': {$near: [1, 1]}}, {sort: {k: 1}}).fetch(), '_id'), + ['x', 'y']); + test.equal( + _.pluck(coll.find({'a.b': {$near: [5, 5]}}, {sort: {k: 1}}).fetch(), '_id'), + ['y', 'x']); + var operations = []; var cbs = log_callbacks(operations); var handle = coll.find({'a.b': {$near: [7,7]}}).observe(cbs); test.length(operations, 2); - test.equal(operations.shift(), ['added', {a:{b:[5,5]}}, 0, null]); + test.equal(operations.shift(), ['added', {k:9, a:{b:[5,5]}}, 0, null]); test.equal(operations.shift(), - ['added', {a:[{b:[[100,100],[1,1]]},{b:[150,150]}]}, 1, null]); + ['added', {k: 9, a:[{b:[[100,100],[1,1]]},{b:[150,150]}]}, + 1, null]); // This needs to be inserted in the MIDDLE of the two existing ones. coll.insert({a: {b: [3,3]}}); test.length(operations, 1); diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index dc2b838a5a..1faea9a455 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -13,7 +13,6 @@ Sorter = function (spec) { var self = this; - self._sortFunction = null; var sortSpecParts = []; @@ -42,12 +41,6 @@ Sorter = function (spec) { throw Error("Bad sort specification: ", JSON.stringify(spec)); } - // If there are no sorting rules specified, leave _sortFunction as null. This - // will allow us to have a special case where we sort on distances if query - // involved the $near operator. - if (sortSpecParts.length === 0) - return; - // reduceValue takes in all the possible values for the sort key along various // branches, and returns the min or max value (according to the bool // findMin). Each value can itself be an array, and we look at its values @@ -83,37 +76,47 @@ Sorter = function (spec) { return reduced; }; - self._sortFunction = function (a, b) { - for (var i = 0; i < sortSpecParts.length; ++i) { - var specPart = sortSpecParts[i]; + var comparators = _.map(sortSpecParts, function (specPart) { + return function (a, b) { var aValue = reduceValue(specPart.lookup(a), specPart.ascending); var bValue = reduceValue(specPart.lookup(b), specPart.ascending); var compare = LocalCollection._f._cmp(aValue, bValue); - if (compare !== 0) - return specPart.ascending ? compare : -compare; + return specPart.ascending ? compare : -compare; }; - return 0; - }; + }); + + self._baseComparator = composeComparators(comparators); }; Sorter.prototype.getComparator = function (options) { var self = this; - // If there was a sort specification, use it. - // XXX do we not use distance as a secondary sort key? - if (self._sortFunction) - return self._sortFunction; - // If there was no sort specification and we have no distances, everything is - // equal. + // If we have no distances, just use the comparator from the source + // specification (which defaults to "everything is equal". if (!options || !options.distances) { - return function (a, b) { - return 0; - }; + return self._baseComparator; } - return function (a, b) { + // Return a comparator which first tries the sort specification, and if that + // says "it's equal", breaks ties using $near distances. + return composeComparators([self._baseComparator, function (a, b) { return options.distances.get(a._id) - options.distances.get(b._id); - }; + }]); }; MinimongoTest.Sorter = Sorter; + +// Given an array of comparators +// (functions (a,b)->(negative or positive or zero)), returns a single +// comparator which uses each comparator in order and returns the first +// non-zero value. +var composeComparators = function (comparatorArray) { + return function (a, b) { + for (var i = 0; i < comparatorArray.length; ++i) { + var compare = comparatorArray[i](a, b); + if (compare !== 0) + return compare; + } + return 0; + }; +}; From e48f08cefc518f50de71383f623556bce445fe80 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 16:46:23 -0800 Subject: [PATCH 083/124] NOTES update. --- packages/minimongo/NOTES | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/NOTES b/packages/minimongo/NOTES index 7e7927d619..89ec82eb85 100644 --- a/packages/minimongo/NOTES +++ b/packages/minimongo/NOTES @@ -15,6 +15,16 @@ usually have sort key 5 (the minimum is used for ascending sorts and the maximum for descending sorts). But if the query's selector is {a: {$gt: 7}}, then Mongo will actually use 10 as the sort key because 5 does not match. +Unsupported selectors: + - $elemMatch inside $all + - geoqueries other than $near ($nearSphere, $geoIntersects, $geoWithin) + +In MongoDB, a query with $near that matches a document which has multiple points +matching the key (eg, `c.find({$near: 'a'})` with the document +`{a: [[1,1], [2, 2]]}` can return the document multiple times (perhaps not +even consecutively). In minimongo, queries never return a document multiple +times. + ## ON TYPES ## We don't implement these Mongo types completely: timestamp (but date works), @@ -52,10 +62,6 @@ hand, the underlying selector and mutator code is quite well tested.) We ignore the 'x' and 's' flags on regular expressions. -We don't do as much type checking as we could, especially in -selectors. If pass in something that's weirdly formed, you'll probably -just get a random exception or error. - "Natural order" isn't very well defined. We don't support capped collections. From cf78cefc8b71cc095887690e84780ee440f7d2f7 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 17:34:04 -0800 Subject: [PATCH 084/124] Refactor selector-vs-modifier/projection code Now they are methods on a compiled Matcher rather than doing their own operator parsing from scratch. This means less work is happening for each oplog entry, and it also localizes knowledge about selector parsing. --- packages/minimongo/minimongo_server_tests.js | 32 +++-- packages/minimongo/selector.js | 124 +++++++++++++----- packages/minimongo/selector_modifier.js | 54 +++----- packages/minimongo/selector_projection.js | 13 +- .../mongo-livedata/oplog_observe_driver.js | 6 +- 5 files changed, 139 insertions(+), 90 deletions(-) diff --git a/packages/minimongo/minimongo_server_tests.js b/packages/minimongo/minimongo_server_tests.js index afd0487c87..90e797c0aa 100644 --- a/packages/minimongo/minimongo_server_tests.js +++ b/packages/minimongo/minimongo_server_tests.js @@ -1,6 +1,7 @@ Tinytest.add("minimongo - modifier affects selector", function (test) { function testSelectorPaths (sel, paths, desc) { - test.isTrue(_.isEqual(MinimongoTest.getSelectorPaths(sel), paths), desc); + var matcher = new Minimongo.Matcher(sel); + test.equal(matcher._getPaths(), paths, desc); } testSelectorPaths({ @@ -49,18 +50,24 @@ Tinytest.add("minimongo - modifier affects selector", function (test) { } }, ['a', 'b.c'], "literal object"); + // Note that a and b do NOT end up in the path list, but x and y both do. + testSelectorPaths({ + $or: [ + {x: {$elemMatch: {a: 5}}}, + {y: {$elemMatch: {b: 7}}} + ] + }, ['x', 'y'], "$or and elemMatch"); + function testSelectorAffectedByModifier (sel, mod, yes, desc) { - if (yes) - test.isTrue(LocalCollection._isSelectorAffectedByModifier(sel, mod, desc)); - else - test.isFalse(LocalCollection._isSelectorAffectedByModifier(sel, mod, desc)); + var matcher = new Minimongo.Matcher(sel); + test.equal(matcher.affectedByModifier(mod), yes, desc); } function affected(sel, mod, desc) { - testSelectorAffectedByModifier(sel, mod, 1, desc); + testSelectorAffectedByModifier(sel, mod, true, desc); } function notAffected(sel, mod, desc) { - testSelectorAffectedByModifier(sel, mod, 0, desc); + testSelectorAffectedByModifier(sel, mod, false, desc); } notAffected({ foo: 0 }, { $set: { bar: 1 } }, "simplest"); @@ -89,7 +96,8 @@ Tinytest.add("minimongo - modifier affects selector", function (test) { Tinytest.add("minimongo - selector and projection combination", function (test) { function testSelProjectionComb (sel, proj, expected, desc) { - test.equal(LocalCollection._combineSelectorAndProjection(sel, proj), expected, desc); + var matcher = new Minimongo.Matcher(sel); + test.equal(matcher.combineIntoProjection(proj), expected, desc); } // Test with inclusive projection @@ -339,11 +347,15 @@ Tinytest.add("minimongo - selector and projection combination", function (test) var test = null; // set this global in the beginning of every test // T - should return true // F - should return false + var oneTest = function (sel, mod, expected, desc) { + var matcher = new Minimongo.Matcher(sel); + test.equal(matcher.canBecomeTrueByModifier(mod), expected, desc); + }; function T (sel, mod, desc) { - test.isTrue(LocalCollection._canSelectorBecomeTrueByModifier(sel, mod), desc); + oneTest(sel, mod, true, desc); } function F (sel, mod, desc) { - test.isFalse(LocalCollection._canSelectorBecomeTrueByModifier(sel, mod), desc); + oneTest(sel, mod, false, desc); } Tinytest.add("minimongo - can selector become true by modifier - literals (structured tests)", function (t) { diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 9004e83c7b..f9631b7b14 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -20,7 +20,17 @@ // if (matcher.documentMatches({a: 7})) ... Minimongo.Matcher = function (selector) { var self = this; - self._isGeoQuery = false; // can get overwritten by compilation + // A set (object mapping string -> *) of all of the document paths looked + // at by the selector. Also includes the empty string if it may look at any + // path (eg, $where). + self._paths = {}; + // Set to true if compilation finds a $near. + self._isGeoQuery = false; + // Set to false if compilation finds anything other than a simple equality on + // some fields. + self._isEquality = true; + // A clone of the original selector. Used by canBecomeTrueByModifier. + self._selector = null; self._docMatcher = self._compileSelector(selector); }; @@ -31,19 +41,28 @@ _.extend(Minimongo.Matcher.prototype, { isGeoQuery: function () { return this._isGeoQuery; }, + isEquality: function () { + return this._isEquality; + }, // Given a selector, return a function that takes one argument, a // document. It returns a result object. _compileSelector: function (selector) { var self = this; // you can pass a literal function instead of a selector - if (selector instanceof Function) + if (selector instanceof Function) { + self._isEquality = false; + self._selector = selector; + self._recordPathUsed(''); return function (doc) { return {result: !!selector.call(doc)}; }; + } // shorthand -- scalars match _id if (LocalCollection._selectorIsId(selector)) { + self._selector = {_id: selector}; + self._recordPathUsed('_id'); return function (doc) { return {result: EJSON.equals(doc._id, selector)}; }; @@ -52,15 +71,26 @@ _.extend(Minimongo.Matcher.prototype, { // protect against dangerous selectors. falsey and {_id: falsey} are both // likely programmer error, and not what you want, particularly for // destructive operations. - if (!selector || (('_id' in selector) && !selector._id)) + if (!selector || (('_id' in selector) && !selector._id)) { + self._isEquality = null; return nothingMatcher; + } // Top level can't be an array or true or binary. if (typeof(selector) === 'boolean' || isArray(selector) || EJSON.isBinary(selector)) throw new Error("Invalid selector: " + selector); - return compileDocumentSelector(selector, self); + self._selector = EJSON.clone(selector); + return compileDocumentSelector(selector, self, {isRoot: true}); + }, + _recordPathUsed: function (path) { + this._paths[path] = true; + }, + // Returns a list of key paths the given selector is looking for. It includes + // the empty string if there is a $where. + _getPaths: function () { + return _.keys(this._paths); } }); @@ -68,9 +98,12 @@ _.extend(Minimongo.Matcher.prototype, { // Takes in a selector that could match a full document (eg, the original // selector). Returns a function mapping document->result object. // +// matcher is the Matcher object we are compiling. +// // If this is the root document selector (ie, not wrapped in $and or the like), -// then matcherIfRoot is the Matcher object. (This is used by $near.) -var compileDocumentSelector = function (docSelector, matcherIfRoot) { +// then isRoot is true. (This is used by $near.) +var compileDocumentSelector = function (docSelector, matcher, options) { + options = options || {}; var docMatchers = []; _.each(docSelector, function (subSelector, key) { if (key.substr(0, 1) === '$') { @@ -78,11 +111,18 @@ var compileDocumentSelector = function (docSelector, matcherIfRoot) { // this function), or $where. if (!_.has(LOGICAL_OPERATORS, key)) throw new Error("Unrecognized logical operator: " + key); - docMatchers.push(LOGICAL_OPERATORS[key](subSelector)); + matcher._isEquality = false; + docMatchers.push(LOGICAL_OPERATORS[key](subSelector, matcher, + options.inElemMatch)); } else { + // Record this path, but only if we aren't in an elemMatcher, since in an + // elemMatch this is a path inside an object in an array, not in the doc + // root. + if (!options.inElemMatch) + matcher._recordPathUsed(key); var lookUpByIndex = makeLookupFunction(key); var valueMatcher = - compileValueSelector(subSelector, matcherIfRoot); + compileValueSelector(subSelector, matcher, options.isRoot); docMatchers.push(function (doc) { var branchValues = lookUpByIndex(doc); return valueMatcher(branchValues); @@ -97,13 +137,14 @@ var compileDocumentSelector = function (docSelector, matcherIfRoot) { // {$gt: 5, $lt: 9}, or a regular expression, or any non-expression object (to // indicate equality). Returns a branched matcher: a function mapping // [branched value]->result object. -var compileValueSelector = function (valueSelector, matcherIfRoot) { - if (valueSelector instanceof RegExp) +var compileValueSelector = function (valueSelector, matcher, isRoot) { + if (valueSelector instanceof RegExp) { + matcher._isEquality = false; return convertElementMatcherToBranchedMatcher( regexpElementMatcher(valueSelector)); - else if (isOperatorObject(valueSelector)) - return operatorBranchedMatcher(valueSelector, matcherIfRoot); - else { + } else if (isOperatorObject(valueSelector)) { + return operatorBranchedMatcher(valueSelector, matcher, isRoot); + } else { return convertElementMatcherToBranchedMatcher( equalityElementMatcher(valueSelector)); } @@ -166,23 +207,28 @@ var equalityElementMatcher = function (elementSelector) { // Takes an operator object (an object with $ keys) and returns a branched // matcher for it. -var operatorBranchedMatcher = function (valueSelector, matcherIfRoot) { +var operatorBranchedMatcher = function (valueSelector, matcher, isRoot) { // Each valueSelector works separately on the various branches. So one // operator can match one branch and another can match another branch. This // is OK. var operatorMatchers = []; _.each(valueSelector, function (operand, operator) { + // XXX we should actually implement $eq, which is new in 2.6 + if (operator !== '$eq') + matcher._isEquality = false; + if (_.has(VALUE_OPERATORS, operator)) { operatorMatchers.push( - VALUE_OPERATORS[operator](operand, valueSelector, matcherIfRoot)); + VALUE_OPERATORS[operator](operand, valueSelector, matcher, isRoot)); } else if (_.has(ELEMENT_OPERATORS, operator)) { var options = ELEMENT_OPERATORS[operator]; if (typeof options === 'function') options = {compileElementSelector: options}; operatorMatchers.push( convertElementMatcherToBranchedMatcher( - options.compileElementSelector(operand, valueSelector), + options.compileElementSelector( + operand, valueSelector, matcher), options)); } else { throw new Error("Unrecognized operator: " + operator); @@ -192,25 +238,29 @@ var operatorBranchedMatcher = function (valueSelector, matcherIfRoot) { return andBranchedMatchers(operatorMatchers); }; -var compileArrayOfDocumentSelectors = function (selectors) { +var compileArrayOfDocumentSelectors = function ( + selectors, matcher, inElemMatch) { if (!isArray(selectors) || _.isEmpty(selectors)) throw Error("$and/$or/$nor must be nonempty array"); return _.map(selectors, function (subSelector) { if (!isPlainObject(subSelector)) throw Error("$or/$and/$nor entries need to be full objects"); - return compileDocumentSelector(subSelector); + return compileDocumentSelector( + subSelector, matcher, {inElemMatch: inElemMatch}); }); }; // Operators that appear at the top level of a document selector. var LOGICAL_OPERATORS = { - $and: function (subSelector) { - var matchers = compileArrayOfDocumentSelectors(subSelector); + $and: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); return andDocumentMatchers(matchers); }, - $or: function (subSelector) { - var matchers = compileArrayOfDocumentSelectors(subSelector); + $or: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); return function (doc) { var result = _.any(matchers, function (f) { return f(doc).result; @@ -220,8 +270,9 @@ var LOGICAL_OPERATORS = { }; }, - $nor: function (subSelector) { - var matchers = compileArrayOfDocumentSelectors(subSelector); + $nor: function (subSelector, matcher, inElemMatch) { + var matchers = compileArrayOfDocumentSelectors( + subSelector, matcher, inElemMatch); return function (doc) { var result = _.all(matchers, function (f) { return !f(doc).result; @@ -232,7 +283,9 @@ var LOGICAL_OPERATORS = { }; }, - $where: function (selectorValue) { + $where: function (selectorValue, matcher) { + // Record that *any* path may be used. + matcher._recordPathUsed(''); if (!(selectorValue instanceof Function)) { // XXX MongoDB seems to have more complex logic to decide where or or not // to add "return"; not sure exactly what it is. @@ -272,8 +325,8 @@ var invertBranchedMatcher = function (branchedMatcher) { // "match each branched value independently and combine with // convertElementMatcherToBranchedMatcher". var VALUE_OPERATORS = { - $not: function (operand) { - return invertBranchedMatcher(compileValueSelector(operand)); + $not: function (operand, valueSelector, matcher) { + return invertBranchedMatcher(compileValueSelector(operand, matcher)); }, $ne: function (operand) { return invertBranchedMatcher(convertElementMatcherToBranchedMatcher( @@ -301,7 +354,7 @@ var VALUE_OPERATORS = { throw Error("$maxDistance needs a $near"); return everythingMatcher; }, - $all: function (operand) { + $all: function (operand, valueSelector, matcher) { if (!isArray(operand)) throw Error("$all requires array"); // Not sure why, but this seems to be what MongoDB does. @@ -314,17 +367,17 @@ var VALUE_OPERATORS = { if (isOperatorObject(criterion)) throw Error("no $ expressions in $all"); // This is always a regexp or equality selector. - branchedMatchers.push(compileValueSelector(criterion)); + branchedMatchers.push(compileValueSelector(criterion, matcher)); }); // andBranchedMatchers does NOT require all selectors to return true on the // SAME branch. return andBranchedMatchers(branchedMatchers); }, - $near: function (operand, valueSelector, matcherIfRoot) { - if (!matcherIfRoot) + $near: function (operand, valueSelector, matcher, isRoot) { + if (!isRoot) throw Error("$near can't be inside another $ operator"); - matcherIfRoot._isGeoQuery = true; + matcher._isGeoQuery = true; // There are two kinds of geodata in MongoDB: coordinate pairs and // GeoJSON. They use different distance metrics, too. GeoJSON queries are @@ -555,20 +608,21 @@ var ELEMENT_OPERATORS = { }, $elemMatch: { dontExpandLeafArrays: true, - compileElementSelector: function (operand) { + compileElementSelector: function (operand, valueSelector, matcher) { if (!isPlainObject(operand)) throw Error("$elemMatch need an object"); var matcher, isDocMatcher; if (isOperatorObject(operand)) { - matcher = compileValueSelector(operand); + matcher = compileValueSelector(operand, matcher); isDocMatcher = false; } else { // This is NOT the same as compileValueSelector(operand), and not just // because of the slightly different calling convention. // {$elemMatch: {x: 3}} means "an element has a field x:3", not // "consists only of a field x:3". Also, regexps and sub-$ are allowed. - matcher = compileDocumentSelector(operand); + matcher = compileDocumentSelector(operand, matcher, + {inElemMatch: true}); isDocMatcher = true; } diff --git a/packages/minimongo/selector_modifier.js b/packages/minimongo/selector_modifier.js index 6ea0b28313..100ac007ac 100644 --- a/packages/minimongo/selector_modifier.js +++ b/packages/minimongo/selector_modifier.js @@ -6,11 +6,12 @@ // - 'foo.bar': 42 // - $unset // - 'abc.d': 1 -LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) { +Minimongo.Matcher.prototype.affectedByModifier = function (modifier) { + var self = this; // safe check for $set/$unset being objects modifier = _.extend({ $set: {}, $unset: {} }, modifier); var modifiedPaths = _.keys(modifier.$set).concat(_.keys(modifier.$unset)); - var meaningfulPaths = getPaths(selector); + var meaningfulPaths = self._getPaths(); return _.any(modifiedPaths, function (path) { var mod = path.split('.'); @@ -43,44 +44,33 @@ LocalCollection._isSelectorAffectedByModifier = function (selector, modifier) { }); }; -getPathsWithoutNumericKeys = function (sel) { - return _.map(getPaths(sel), function (path) { - return _.reject(path.split('.'), isNumericKey).join('.'); - }); -}; - -// @param selector - Object: MongoDB selector. Currently doesn't support -// $-operators and arrays well. // @param modifier - Object: MongoDB-styled modifier with `$set`s and `$unsets` // only. (assumed to come from oplog) // @returns - Boolean: if after applying the modifier, selector can start // accepting the modified value. -LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) -{ - if (!LocalCollection._isSelectorAffectedByModifier(selector, modifier)) +// Currently doesn't support $-operators and numeric indices precisely. +Minimongo.Matcher.prototype.canBecomeTrueByModifier = function (modifier) { + var self = this; + if (!this.affectedByModifier(modifier)) return false; modifier = _.extend({$set:{}, $unset:{}}, modifier); - if (_.any(_.keys(selector), pathHasNumericKeys) || - _.any(_.keys(modifier.$unset), pathHasNumericKeys) || - _.any(_.keys(modifier.$set), pathHasNumericKeys)) + if (!self.isEquality()) return true; - if (!isLiteralSelector(selector)) + if (_.any(self._getPaths(), pathHasNumericKeys) || + _.any(_.keys(modifier.$unset), pathHasNumericKeys) || + _.any(_.keys(modifier.$set), pathHasNumericKeys)) return true; // convert a selector into an object matching the selector // { 'a.b': { ans: 42 }, 'foo.bar': null, 'foo.baz': "something" } // => { a: { b: { ans: 42 } }, foo: { bar: null, baz: "something" } } - var doc = pathsToTree(_.keys(selector), - function (path) { return selector[path]; }, + var doc = pathsToTree(self._getPaths(), + function (path) { return self._selector[path]; }, _.identity /*conflict resolution is no resolution*/); - // XXX we should move this function to being a method on Matcher so we aren't - // recompiling over and over - var matcher = new Minimongo.Matcher(selector); - try { LocalCollection._modify(doc, modifier); } catch (e) { @@ -99,11 +89,11 @@ LocalCollection._canSelectorBecomeTrueByModifier = function (selector, modifier) throw e; } - return matcher.documentMatches(doc).result; + return self.documentMatches(doc).result; }; -// Returns a list of key paths the given selector is looking for -var getPaths = MinimongoTest.getSelectorPaths = function (sel) { +var getPaths = function (sel) { + return _.keys(new Minimongo.Matcher(sel)._paths); return _.chain(sel).map(function (v, k) { // we don't know how to handle $where because it can be anything if (k === "$where") @@ -120,15 +110,3 @@ function pathHasNumericKeys (path) { return _.any(path.split('.'), isNumericKey); } -function isLiteralSelector (selector) { - return _.all(selector, function (subSelector, keyPath) { - if (keyPath.substr(0, 1) === "$" || _.isRegExp(subSelector)) - return false; - if (!_.isObject(subSelector) || _.isArray(subSelector)) - return true; - return _.all(subSelector, function (value, key) { - return key.substr(0, 1) !== "$"; - }); - }); -} - diff --git a/packages/minimongo/selector_projection.js b/packages/minimongo/selector_projection.js index ece29b8470..73a2727a8e 100644 --- a/packages/minimongo/selector_projection.js +++ b/packages/minimongo/selector_projection.js @@ -1,9 +1,9 @@ // Knows how to combine a mongo selector and a fields projection to a new fields // projection taking into account active fields from the passed selector. // @returns Object - projection object (same as fields option of mongo cursor) -LocalCollection._combineSelectorAndProjection = function (selector, projection) -{ - var selectorPaths = getPathsWithoutNumericKeys(selector); +Minimongo.Matcher.prototype.combineIntoProjection = function (projection) { + var self = this; + var selectorPaths = self._getPathsElidingNumericKeys(); // Special case for $where operator in the selector - projection should depend // on all fields of the document. getSelectorPaths returns a list of paths @@ -40,6 +40,13 @@ LocalCollection._combineSelectorAndProjection = function (selector, projection) } }; +Minimongo.Matcher.prototype._getPathsElidingNumericKeys = function () { + var self = this; + return _.map(self._getPaths(), function (path) { + return _.reject(path.split('.'), isNumericKey).join('.'); + }); +}; + // Returns a set of key paths similar to // { 'foo.bar': 1, 'a.b.c': 1 } var treeToPaths = function (tree, prefix) { diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 60be922a16..109a5dca37 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -38,8 +38,7 @@ OplogObserveDriver = function (options) { self._projectionFn = LocalCollection._compileProjection(projection); // Projection function, result of combining important fields for selector and // existing fields projection - self._sharedProjection = LocalCollection._combineSelectorAndProjection( - selector, projection); + self._sharedProjection = self._matcher.combineIntoProjection(projection); self._sharedProjectionFn = LocalCollection._compileProjection( self._sharedProjection); @@ -263,8 +262,7 @@ _.extend(OplogObserveDriver.prototype, { LocalCollection._modify(newDoc, op.o); self._handleDoc(id, self._sharedProjectionFn(newDoc)); } else if (!canDirectlyModifyDoc || - LocalCollection._canSelectorBecomeTrueByModifier( - self._cursorDescription.selector, op.o)) { + self._matcher.canBecomeTrueByModifier(op.o)) { self._needToFetch.set(id, op.ts.toString()); if (self._phase === PHASE.STEADY) self._fetchModifiedDocuments(); From 7d448eb0d880d6b2154954f092e2d1ed26babfa3 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 17:58:07 -0800 Subject: [PATCH 085/124] Throw on missing distance --- packages/minimongo/sort.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/minimongo/sort.js b/packages/minimongo/sort.js index 1faea9a455..718ec1537e 100644 --- a/packages/minimongo/sort.js +++ b/packages/minimongo/sort.js @@ -97,10 +97,16 @@ Sorter.prototype.getComparator = function (options) { return self._baseComparator; } + var distances = options.distances; + // Return a comparator which first tries the sort specification, and if that // says "it's equal", breaks ties using $near distances. return composeComparators([self._baseComparator, function (a, b) { - return options.distances.get(a._id) - options.distances.get(b._id); + if (!distances.has(a._id)) + throw Error("Missing distance for " + a._id); + if (!distances.has(b._id)) + throw Error("Missing distance for " + b._id); + return distances.get(a._id) - distances.get(b._id); }]); }; From 83fde36e00fb4cf17145b2045e2f61ec04dc61e7 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 2 Jan 2014 18:23:34 -0800 Subject: [PATCH 086/124] Fix hashed login token test failures caused by merge mishap. --- packages/accounts-base/accounts_server.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/accounts-base/accounts_server.js b/packages/accounts-base/accounts_server.js index 931d49c33b..59c82eac56 100644 --- a/packages/accounts-base/accounts_server.js +++ b/packages/accounts-base/accounts_server.js @@ -74,12 +74,23 @@ Accounts.createToken = function (options) { throw new Meteor.Error(400, "Unrecognized options for login request"); }; -// Deletes the given loginToken from the database. This will cause all -// connections associated with the token to be closed. +// Deletes the given loginToken from the database. +// +// For new-style hashed token, this will cause all connections +// associated with the token to be closed. +// +// Any connections associated with old-style unhashed tokens will be +// in the process of becoming associated with hashed tokens and then +// they'll get closed. Accounts.destroyToken = function (userId, loginToken) { Meteor.users.update(userId, { $pull: { - "services.resume.loginTokens": { "token": loginToken } + "services.resume.loginTokens": { + $or: [ + { hashedToken: loginToken }, + { token: loginToken } + ] + } } }); }; @@ -108,7 +119,7 @@ Meteor.methods({ Meteor._noYieldsAllowed(function () { Accounts._setLoginToken( result.id, - this.connection, + self.connection, Accounts._hashLoginToken(result.token) ); }); From a1627071a7d0e42e9a716c1996994d864adf4da5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 21:35:00 -0800 Subject: [PATCH 087/124] set arrayIndex in most places (not $near or $elemMatch yet) --- packages/minimongo/selector.js | 48 ++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index f9631b7b14..40803932ec 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -162,11 +162,18 @@ var convertElementMatcherToBranchedMatcher = function ( expanded = expandArraysInBranches( branches, options.dontIncludeLeafArrays); } - var result = _.any(expanded, function (element) { - // XXX arrayIndex! need to save the winner here - return elementMatcher(element.value); + var ret = {}; + ret.result = _.any(expanded, function (element) { + var matched = elementMatcher(element.value); + + // If some element matched, and it's tagged with an array index, include + // that index in our result object. + if (matched && element.arrayIndex !== undefined) + ret.arrayIndex = element.arrayIndex; + + return matched; }); - return {result: result}; + return ret; }; }; @@ -261,11 +268,18 @@ var LOGICAL_OPERATORS = { $or: function (subSelector, matcher, inElemMatch) { var matchers = compileArrayOfDocumentSelectors( subSelector, matcher, inElemMatch); + + // Special case: if there is only one matcher, use it directly, *preserving* + // any arrayIndex it returns. + if (matchers.length === 1) + return matchers[0]; + return function (doc) { var result = _.any(matchers, function (f) { return f(doc).result; }); - // XXX arrayIndex! + // $or does NOT set arrayIndex when it has multiple + // sub-expressions. (Tested against MongoDB.) return {result: result}; }; }, @@ -278,7 +292,7 @@ var LOGICAL_OPERATORS = { return !f(doc).result; }); // Never set arrayIndex, because we only match if nothing in particular - // "matched". + // "matched" (and because this is consistent with MongoDB). return {result: result}; }; }, @@ -804,6 +818,9 @@ expandArraysInBranches = function (branches, skipTheArrays) { _.each(branch.value, function (leaf, i) { branchesOut.push({ value: leaf, + // arrayIndex always defaults to the outermost array, but if we didn't + // need to use an array to get to this branch, we mark the index we + // just used as the arrayIndex. arrayIndex: branch.arrayIndex === undefined ? i : branch.arrayIndex }); }); @@ -834,7 +851,6 @@ var andSomeMatchers = function (branchedSelectors) { return function (branches) { // XXX arrayIndex! var ret = {}; - var distance; ret.result = _.all(branchedSelectors, function (f) { var subResult = f(branches); // Copy a 'distance' number out of the first sub-matcher that has @@ -842,13 +858,23 @@ var andSomeMatchers = function (branchedSelectors) { // query, something arbitrary happens; this appears to be consistent with // Mongo. if (subResult.result && subResult.distance !== undefined - && distance === undefined) { - distance = subResult.distance; + && ret.distance === undefined) { + ret.distance = subResult.distance; + } + // Similarly, propagate arrayIndex from sub-matchers... but to match + // MongoDB behavior, this time the *last* sub-matcher with an arrayIndex + // wins. + if (subResult.result && subResult.arrayIndex !== undefined) { + ret.arrayIndex = subResult.arrayIndex; } return subResult.result; }); - if (ret.result && distance !== undefined) - ret.distance = distance; + + // If we didn't actually match, forget any extra metadata we came up with. + if (!ret.result) { + delete ret.distance; + delete ret.arrayIndex; + } return ret; }; }; From 31d89599a72cb6d9583b19b5b3003abe083365cc Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 21:40:14 -0800 Subject: [PATCH 088/124] rename args in andSomeMatchers --- packages/minimongo/selector.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 40803932ec..f73db6fa01 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -842,17 +842,17 @@ var everythingMatcher = function (docOrBranchedValues) { // "document matchers" and "branched matchers". They both return result objects // but the argument is different: for the former it's a whole doc, whereas for // the latter it's an array of "branched values". -var andSomeMatchers = function (branchedSelectors) { - if (branchedSelectors.length === 0) +var andSomeMatchers = function (subMatchers) { + if (subMatchers.length === 0) return everythingMatcher; - if (branchedSelectors.length === 1) - return branchedSelectors[0]; + if (subMatchers.length === 1) + return subMatchers[0]; - return function (branches) { + return function (docOrBranches) { // XXX arrayIndex! var ret = {}; - ret.result = _.all(branchedSelectors, function (f) { - var subResult = f(branches); + ret.result = _.all(subMatchers, function (f) { + var subResult = f(docOrBranches); // Copy a 'distance' number out of the first sub-matcher that has // one. Yes, this means that if there are multiple $near fields in a // query, something arbitrary happens; this appears to be consistent with From 02aad697d7b22d01de904e97b56baa0244864233 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 21:42:51 -0800 Subject: [PATCH 089/124] remove random type tag in test --- packages/minimongo/minimongo_tests.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 6af81b9c1a..93b8db85eb 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -284,8 +284,7 @@ Tinytest.add("minimongo - selector_compiler", function (test) { var doesMatch = new Minimongo.Matcher(selector).documentMatches(doc).result; if (doesMatch != shouldMatch) { // XXX super janky - test.fail({type: "minimongo-ordering", - message: "minimongo match failure: document " + + test.fail({message: "minimongo match failure: document " + (shouldMatch ? "should match, but doesn't" : "shouldn't match, but does"), selector: JSON.stringify(selector), From a8d1798e880256b0ea0374b75371e8d0df4488e1 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 21:45:13 -0800 Subject: [PATCH 090/124] JSON -> EJSON in test --- packages/minimongo/minimongo_tests.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 93b8db85eb..e5a912fa98 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1710,10 +1710,10 @@ Tinytest.add("minimongo - modify", function (test) { // XXX super janky test.fail({type: "minimongo-modifier", message: "modifier test failure", - input_doc: JSON.stringify(doc), - modifier: JSON.stringify(mod), - expected: JSON.stringify(result), - actual: JSON.stringify(copy) + input_doc: EJSON.stringify(doc), + modifier: EJSON.stringify(mod), + expected: EJSON.stringify(result), + actual: EJSON.stringify(copy) }); } else { test.ok(); From 0f7e0b54ca30bf7867095a3b3ca1daa4c1d13559 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 21:47:24 -0800 Subject: [PATCH 091/124] Refactor: optionify LocalCollection._modify arg --- packages/minimongo/minimongo.js | 2 +- packages/minimongo/modify.js | 14 ++++++++------ packages/mongo-livedata/mongo_driver.js | 2 +- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index f1cf205d7f..7ab5dbd2b5 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -646,7 +646,7 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { var insertedId; if (updateCount === 0 && options.upsert) { var newDoc = LocalCollection._removeDollarOperators(selector); - LocalCollection._modify(newDoc, mod, true); + LocalCollection._modify(newDoc, mod, {isInsert: true}); if (! newDoc._id && options.insertedId) newDoc._id = options.insertedId; insertedId = self.insert(newDoc); diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 3be2c9447d..05a2d8bf33 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -6,10 +6,12 @@ // XXX atomicity: if one modification fails, do we roll back the whole // change? // -// isInsert is set when _modify is being called to compute the document to -// insert as part of an upsert operation. We use this primarily to figure out -// when to set the fields in $setOnInsert, if present. -LocalCollection._modify = function (doc, mod, isInsert) { +// options: +// - isInsert is set when _modify is being called to compute the document to +// insert as part of an upsert operation. We use this primarily to figure +// out when to set the fields in $setOnInsert, if present. +LocalCollection._modify = function (doc, mod, options) { + options = options || {}; var is_modifier = false; for (var k in mod) { // IE7 doesn't support indexing into strings (eg, k[0]), so use substr. @@ -42,7 +44,7 @@ LocalCollection._modify = function (doc, mod, isInsert) { for (var op in mod) { var mod_func = LocalCollection._modifiers[op]; // Treat $setOnInsert as $set if this is an insert. - if (isInsert && op === '$setOnInsert') + if (options.isInsert && op === '$setOnInsert') mod_func = LocalCollection._modifiers['$set']; if (!mod_func) throw MinimongoError("Invalid modifier specified " + op); @@ -74,7 +76,7 @@ LocalCollection._modify = function (doc, mod, isInsert) { // isInsert: if we're constructing a document to insert (via upsert) // and we're in replacement mode, not modify mode, DON'T take the // _id from the query. This matches mongo's behavior. - if (k !== '_id' || isInsert) + if (k !== '_id' || options.isInsert) delete doc[k]; }); for (var k in new_doc) { diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index baaa2eeab5..036f236885 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -491,7 +491,7 @@ var simulateUpsertWithInsertedId = function (collection, selector, mod, // the behavior of modifiers is concerned, whether `_modify` // is run on EJSON or on mongo-converted EJSON. var selectorDoc = LocalCollection._removeDollarOperators(selector); - LocalCollection._modify(selectorDoc, mod, true); + LocalCollection._modify(selectorDoc, mod, {isInsert: true}); newDoc = selectorDoc; } else { newDoc = mod; From b2e3b08248b61328f4d373014aea8a52f8bf332d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 22:07:37 -0800 Subject: [PATCH 092/124] Cleanup/modernization of modify.js --- packages/minimongo/helpers.js | 39 +++++++++++++++ packages/minimongo/modify.js | 86 +++++++++++++--------------------- packages/minimongo/package.js | 1 + packages/minimongo/selector.js | 51 ++++---------------- 4 files changed, 82 insertions(+), 95 deletions(-) create mode 100644 packages/minimongo/helpers.js diff --git a/packages/minimongo/helpers.js b/packages/minimongo/helpers.js new file mode 100644 index 0000000000..d4046124f2 --- /dev/null +++ b/packages/minimongo/helpers.js @@ -0,0 +1,39 @@ +// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as +// arrays. +// XXX maybe this should be EJSON.isArray +isArray = function (x) { + return _.isArray(x) && !EJSON.isBinary(x); +}; + +// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about +// RegExp +// XXX note that _type(undefined) === 3!!!! +isPlainObject = function (x) { + return x && LocalCollection._f._type(x) === 3; +}; + +isIndexable = function (x) { + return isArray(x) || isPlainObject(x); +}; + +isOperatorObject = function (valueSelector) { + if (!isPlainObject(valueSelector)) + return false; + + var theseAreOperators = undefined; + _.each(valueSelector, function (value, selKey) { + var thisIsOperator = selKey.substr(0, 1) === '$'; + if (theseAreOperators === undefined) { + theseAreOperators = thisIsOperator; + } else if (theseAreOperators !== thisIsOperator) { + throw new Error("Inconsistent operator: " + valueSelector); + } + }); + return !!theseAreOperators; // {} has no operators +}; + + +// string can be converted to integer +isNumericKey = function (s) { + return /^[0-9]+$/.test(s); +}; diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 05a2d8bf33..444199a3bb 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -12,59 +12,49 @@ // out when to set the fields in $setOnInsert, if present. LocalCollection._modify = function (doc, mod, options) { options = options || {}; - var is_modifier = false; - for (var k in mod) { - // IE7 doesn't support indexing into strings (eg, k[0]), so use substr. - // Too bad -- it's far slower: - // http://jsperf.com/testing-the-first-character-of-a-string - is_modifier = k.substr(0, 1) === '$'; - break; // just check the first key. - } + if (!isPlainObject(mod)) + throw MinimongoError("Modifier must be an object"); + var isModifier = isOperatorObject(mod); - var new_doc; + var newDoc; - if (!is_modifier) { + if (!isModifier) { if (mod._id && !EJSON.equals(doc._id, mod._id)) throw MinimongoError("Cannot change the _id of a document"); // replace the whole document for (var k in mod) { - if (k.substr(0, 1) === '$') - throw MinimongoError( - "When replacing document, field name may not start with '$'"); if (/\./.test(k)) throw MinimongoError( "When replacing document, field name may not contain '.'"); } - new_doc = mod; + newDoc = mod; } else { - // apply modifiers - var new_doc = EJSON.clone(doc); + // apply modifiers to the doc. + newDoc = EJSON.clone(doc); - for (var op in mod) { - var mod_func = LocalCollection._modifiers[op]; + _.each(mod, function (operand, op) { + var modFunc = MODIFIERS[op]; // Treat $setOnInsert as $set if this is an insert. if (options.isInsert && op === '$setOnInsert') - mod_func = LocalCollection._modifiers['$set']; - if (!mod_func) + modFunc = MODIFIERS['$set']; + if (!modFunc) throw MinimongoError("Invalid modifier specified " + op); - for (var keypath in mod[op]) { + _.each(operand, function (arg, keypath) { // XXX mongo doesn't allow mod field names to end in a period, // but I don't see why.. it allows '' as a key, as does JS if (keypath.length && keypath[keypath.length-1] === '.') throw MinimongoError( "Invalid mod field name, may not end in a period"); - var arg = mod[op][keypath]; var keyparts = keypath.split('.'); - var no_create = !!LocalCollection._noCreateModifiers[op]; - var forbid_array = (op === "$rename"); - var target = LocalCollection._findModTarget(new_doc, keyparts, - no_create, forbid_array); + var noCreate = _.has(NO_CREATE_MODIFIERS, op); + var forbidArray = (op === "$rename"); + var target = findModTarget(newDoc, keyparts, noCreate, forbidArray); var field = keyparts.pop(); - mod_func(target, field, arg, keypath, new_doc); - } - } + modFunc(target, field, arg, keypath, newDoc); + }); + }); } // move new document into place. @@ -79,31 +69,30 @@ LocalCollection._modify = function (doc, mod, options) { if (k !== '_id' || options.isInsert) delete doc[k]; }); - for (var k in new_doc) { - doc[k] = new_doc[k]; - } + _.each(newDoc, function (v, k) { + doc[k] = v; + }); }; // for a.b.c.2.d.e, keyparts should be ['a', 'b', 'c', '2', 'd', 'e'], // and then you would operate on the 'e' property of the returned -// object. if no_create is falsey, creates intermediate levels of +// object. if noCreate is falsey, creates intermediate levels of // structure as necessary, like mkdir -p (and raises an exception if // that would mean giving a non-numeric property to an array.) if -// no_create is true, return undefined instead. may modify the last +// noCreate is true, return undefined instead. may modify the last // element of keyparts to signal to the caller that it needs to use a // different value to index into the returned object (for example, -// ['a', '01'] -> ['a', 1]). if forbid_array is true, return null if +// ['a', '01'] -> ['a', 1]). if forbidArray is true, return null if // the keypath goes through an array. -LocalCollection._findModTarget = function (doc, keyparts, no_create, - forbid_array) { +var findModTarget = function (doc, keyparts, noCreate, forbidArray) { for (var i = 0; i < keyparts.length; i++) { var last = (i === keyparts.length - 1); var keypart = keyparts[i]; var numeric = /^[0-9]+$/.test(keypart); - if (no_create && (!(typeof doc === "object") || !(keypart in doc))) + if (noCreate && (!(typeof doc === "object") || !(keypart in doc))) return undefined; if (doc instanceof Array) { - if (forbid_array) + if (forbidArray) return null; if (!numeric) throw MinimongoError( @@ -136,7 +125,7 @@ LocalCollection._findModTarget = function (doc, keyparts, no_create, // notreached }; -LocalCollection._noCreateModifiers = { +var NO_CREATE_MODIFIERS = { $unset: true, $pop: true, $rename: true, @@ -144,7 +133,7 @@ LocalCollection._noCreateModifiers = { $pullAll: true }; -LocalCollection._modifiers = { +var MODIFIERS = { $inc: function (target, field, arg) { if (typeof arg !== "number") throw MinimongoError("Modifier $inc allowed for numbers only"); @@ -307,7 +296,7 @@ LocalCollection._modifiers = { else if (!(x instanceof Array)) throw MinimongoError("Cannot apply $pull/pullAll modifier to non-array"); else { - var out = [] + var out = []; if (typeof arg === "object" && !(arg instanceof Array)) { // XXX would be much nicer to compile this once, rather than // for each document we modify.. but usually we're not @@ -341,7 +330,7 @@ LocalCollection._modifiers = { else if (!(x instanceof Array)) throw MinimongoError("Cannot apply $pull/pullAll modifier to non-array"); else { - var out = [] + var out = []; for (var i = 0; i < x.length; i++) { var exclude = false; for (var j = 0; j < arg.length; j++) { @@ -370,7 +359,7 @@ LocalCollection._modifiers = { delete target[field]; var keyparts = arg.split('.'); - var target2 = LocalCollection._findModTarget(doc, keyparts, false, true); + var target2 = findModTarget(doc, keyparts, false, true); if (target2 === null) throw MinimongoError("$rename target field invalid"); var field2 = keyparts.pop(); @@ -382,12 +371,3 @@ LocalCollection._modifiers = { throw MinimongoError("$bit is not supported"); } }; - -LocalCollection._removeDollarOperators = function (selector) { - var selectorDoc = {}; - for (var k in selector) - if (k.substr(0, 1) !== '$') - selectorDoc[k] = selector[k]; - return selectorDoc; -}; - diff --git a/packages/minimongo/package.js b/packages/minimongo/package.js index 5d16c82f3f..4ba9a25fc7 100644 --- a/packages/minimongo/package.js +++ b/packages/minimongo/package.js @@ -13,6 +13,7 @@ Package.on_use(function (api) { api.use('geojson-utils'); api.add_files([ 'minimongo.js', + 'helpers.js', 'selector.js', 'sort.js', 'projection.js', diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index f73db6fa01..488d18ea65 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -883,48 +883,6 @@ var andDocumentMatchers = andSomeMatchers; var andBranchedMatchers = andSomeMatchers; -// HELPER FUNCTIONS - -// Like _.isArray, but doesn't regard polyfilled Uint8Arrays on old browsers as -// arrays. -// XXX maybe this should be EJSON.isArray -isArray = function (x) { - return _.isArray(x) && !EJSON.isBinary(x); -}; - -// XXX maybe this should be EJSON.isObject, though EJSON doesn't know about -// RegExp -// XXX note that _type(undefined) === 3!!!! -var isPlainObject = function (x) { - return x && LocalCollection._f._type(x) === 3; -}; - -var isIndexable = function (x) { - return isArray(x) || isPlainObject(x); -}; - -var isOperatorObject = function (valueSelector) { - if (!isPlainObject(valueSelector)) - return false; - - var theseAreOperators = undefined; - _.each(valueSelector, function (value, selKey) { - var thisIsOperator = selKey.substr(0, 1) === '$'; - if (theseAreOperators === undefined) { - theseAreOperators = thisIsOperator; - } else if (theseAreOperators !== thisIsOperator) { - throw new Error("Inconsistent selector: " + valueSelector); - } - }); - return !!theseAreOperators; // {} has no operators -}; - - -// string can be converted to integer -isNumericKey = function (s) { - return /^[0-9]+$/.test(s); -}; - // helpers used by compiled selector code LocalCollection._f = { // XXX for _all and _in, consider building 'inquery' at compile time.. @@ -1089,3 +1047,12 @@ LocalCollection._f = { throw Error("Unknown type to sort"); } }; + +// Oddball function used by upsert. +LocalCollection._removeDollarOperators = function (selector) { + var selectorDoc = {}; + for (var k in selector) + if (k.substr(0, 1) !== '$') + selectorDoc[k] = selector[k]; + return selectorDoc; +}; From 5c9e58f2af5b0304ef2fc6b603c3fab80bc45291 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 22:38:50 -0800 Subject: [PATCH 093/124] clean up findModTarget convert arguments to options --- packages/minimongo/minimongo_tests.js | 41 ++++++++++------------- packages/minimongo/modify.js | 47 ++++++++++++++++++--------- 2 files changed, 50 insertions(+), 38 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index e5a912fa98..bd865de7c6 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1703,25 +1703,23 @@ Tinytest.add("minimongo - binary search", function (test) { }); Tinytest.add("minimongo - modify", function (test) { - var modify = function (doc, mod, result) { - var copy = EJSON.clone(doc); - LocalCollection._modify(copy, mod); - if (!LocalCollection._f._equal(copy, result)) { - // XXX super janky - test.fail({type: "minimongo-modifier", - message: "modifier test failure", - input_doc: EJSON.stringify(doc), - modifier: EJSON.stringify(mod), - expected: EJSON.stringify(result), - actual: EJSON.stringify(copy) - }); - } else { - test.ok(); - } + var modifyWithQuery = function (doc, query, mod, expected) { + var coll = new LocalCollection; + coll.insert(doc); + // The query is relevant for 'a.$.b'. + coll.update(query, mod); + var actual = coll.findOne(); + delete actual._id; // added by insert + test.equal(actual, expected, {input: doc, mod: mod}); + }; + var modify = function (doc, mod, expected) { + modifyWithQuery(doc, {}, mod, expected); }; var exception = function (doc, mod) { + var coll = new LocalCollection; + coll.insert(doc); test.throws(function () { - LocalCollection._modify(EJSON.clone(doc), mod); + coll.update({}, mod); }); }; @@ -1748,18 +1746,13 @@ Tinytest.add("minimongo - modify", function (test) { modify({a: [null, null, null]}, {$set: {'a.3.b': 12}}, { a: [null, null, null, {b: 12}]}); exception({a: []}, {$set: {'a.b': 12}}); - test.expect_fail(); exception({a: 12}, {$set: {'a.b': 99}}); // tested on mongo - test.expect_fail(); exception({a: 'x'}, {$set: {'a.b': 99}}); - test.expect_fail(); exception({a: true}, {$set: {'a.b': 99}}); - test.expect_fail(); exception({a: null}, {$set: {'a.b': 99}}); modify({a: {}}, {$set: {'a.3': 12}}, {a: {'3': 12}}); modify({a: []}, {$set: {'a.3': 12}}, {a: [null, null, null, 12]}); modify({}, {$set: {'': 12}}, {'': 12}); // tested on mongo - test.expect_fail(); exception({}, {$set: {'.': 12}}); // tested on mongo modify({}, {$set: {'. ': 12}}, {'': {' ': 12}}); // tested on mongo modify({}, {$inc: {'... ': 12}}, {'': {'': {'': {' ': 12}}}}); // tested @@ -1952,9 +1945,11 @@ Tinytest.add("minimongo - modify", function (test) { exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2'}}); // tested exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2.r'}}); // tested test.expect_fail(); - exception({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}}); // tested + modify({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}}, + {a: {b: 12}, x: []}); // tested test.expect_fail(); - exception({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}}); // tested + modify({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}}, + {a: {b: 12}, x: []}); // tested exception({}, {$rename: {'a': 'a'}}); exception({}, {$rename: {'a.b': 'a.b'}}); modify({a: 12, b: 13}, {$rename: {a: 'b'}}, {b: 12}); diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 444199a3bb..1ccb3c0964 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -50,7 +50,10 @@ LocalCollection._modify = function (doc, mod, options) { var keyparts = keypath.split('.'); var noCreate = _.has(NO_CREATE_MODIFIERS, op); var forbidArray = (op === "$rename"); - var target = findModTarget(newDoc, keyparts, noCreate, forbidArray); + var target = findModTarget(newDoc, keyparts, { + noCreate: NO_CREATE_MODIFIERS[op], + forbidArray: (op === "$rename") + }); var field = keyparts.pop(); modFunc(target, field, arg, keypath, newDoc); }); @@ -76,29 +79,42 @@ LocalCollection._modify = function (doc, mod, options) { // for a.b.c.2.d.e, keyparts should be ['a', 'b', 'c', '2', 'd', 'e'], // and then you would operate on the 'e' property of the returned -// object. if noCreate is falsey, creates intermediate levels of +// object. +// +// if options.noCreate is falsey, creates intermediate levels of // structure as necessary, like mkdir -p (and raises an exception if // that would mean giving a non-numeric property to an array.) if -// noCreate is true, return undefined instead. may modify the last -// element of keyparts to signal to the caller that it needs to use a -// different value to index into the returned object (for example, -// ['a', '01'] -> ['a', 1]). if forbidArray is true, return null if -// the keypath goes through an array. -var findModTarget = function (doc, keyparts, noCreate, forbidArray) { +// options.noCreate is true, return undefined instead. +// +// may modify the last element of keyparts to signal to the caller that it needs +// to use a different value to index into the returned object (for example, +// ['a', '01'] -> ['a', 1]). +// +// if forbidArray is true, return null if the keypath goes through an array. +var findModTarget = function (doc, keyparts, options) { + options = options || {}; for (var i = 0; i < keyparts.length; i++) { var last = (i === keyparts.length - 1); var keypart = keyparts[i]; - var numeric = /^[0-9]+$/.test(keypart); - if (noCreate && (!(typeof doc === "object") || !(keypart in doc))) + var indexable = isIndexable(doc); + if (options.noCreate && !(indexable && keypart in doc)) return undefined; + if (!indexable) { + var e = MinimongoError( + "cannot use the part '" + keypart + "' to traverse " + doc); + e.setPropertyError = true; + throw e; + } if (doc instanceof Array) { - if (forbidArray) + if (options.forbidArray) return null; - if (!numeric) + if (isNumericKey(keypart)) { + keypart = parseInt(keypart); + } else { throw MinimongoError( "can't append to array using string field name [" + keypart + "]"); - keypart = parseInt(keypart); + } if (last) // handle 'a.01' keyparts[i] = keypart; @@ -112,7 +128,8 @@ var findModTarget = function (doc, keyparts, noCreate, forbidArray) { "' of list value " + JSON.stringify(doc[keypart])); } } else { - // XXX check valid fieldname (no $ at start, no .) + if (keypart.length && keypart.substr(0, 1) === '$') + throw MinimongoError("can't set field named " + keypart); if (!last && !(keypart in doc)) doc[keypart] = {}; } @@ -359,7 +376,7 @@ var MODIFIERS = { delete target[field]; var keyparts = arg.split('.'); - var target2 = findModTarget(doc, keyparts, false, true); + var target2 = findModTarget(doc, keyparts, {forbidArray: true}); if (target2 === null) throw MinimongoError("$rename target field invalid"); var field2 = keyparts.pop(); From e3e9cca12a7f6786d154416482051047f2d49d4a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 23:00:18 -0800 Subject: [PATCH 094/124] Implement 'a.$.b' modifier Does not yet work with $near or $elemMatch --- packages/minimongo/NOTES | 3 -- packages/minimongo/minimongo.js | 7 ++-- packages/minimongo/minimongo_tests.js | 59 +++++++++++++++++++++++---- packages/minimongo/modify.js | 33 ++++++++++++--- 4 files changed, 80 insertions(+), 22 deletions(-) diff --git a/packages/minimongo/NOTES b/packages/minimongo/NOTES index 89ec82eb85..c62d11c679 100644 --- a/packages/minimongo/NOTES +++ b/packages/minimongo/NOTES @@ -5,9 +5,6 @@ In update, $pull can't take a selector like {$gt: 3} (but it can take can be used, but selectors that are intended to match non-document values won't work.) -In update, we don't support '$' to indicate the matched array object -as in {$set: {'a.$.x': 12}}. - Sort does not correctly implement the Mongo behavior where only array entries that match the query's selector are used as sort keys. Specifically, when sorting a document with sort specifier {a:1}, the document {a: [5, 10]} will diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 7ab5dbd2b5..16918ae980 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -623,9 +623,8 @@ LocalCollection.prototype.update = function (selector, mod, options, callback) { var queryResult = matcher.documentMatches(doc); if (queryResult.result) { // XXX Should we save the original even if mod ends up being a no-op? - // XXX queryResult should have arrayIndex on it, useful for '$' self._saveOriginal(id, doc); - self._modifyAndNotify(doc, mod, recomputeQids); + self._modifyAndNotify(doc, mod, recomputeQids, queryResult.arrayIndex); ++updateCount; if (!options.multi) break; @@ -690,7 +689,7 @@ LocalCollection.prototype.upsert = function (selector, mod, options, callback) { }; LocalCollection.prototype._modifyAndNotify = function ( - doc, mod, recomputeQids) { + doc, mod, recomputeQids, arrayIndex) { var self = this; var matched_before = {}; @@ -708,7 +707,7 @@ LocalCollection.prototype._modifyAndNotify = function ( var old_doc = EJSON.clone(doc); - LocalCollection._modify(doc, mod); + LocalCollection._modify(doc, mod, {arrayIndex: arrayIndex}); for (qid in self.queries) { query = self.queries[qid]; diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index bd865de7c6..c75099b765 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1710,18 +1710,21 @@ Tinytest.add("minimongo - modify", function (test) { coll.update(query, mod); var actual = coll.findOne(); delete actual._id; // added by insert - test.equal(actual, expected, {input: doc, mod: mod}); + test.equal(actual, expected, EJSON.stringify({input: doc, mod: mod})); }; var modify = function (doc, mod, expected) { modifyWithQuery(doc, {}, mod, expected); }; - var exception = function (doc, mod) { + var exceptionWithQuery = function (doc, query, mod) { var coll = new LocalCollection; coll.insert(doc); test.throws(function () { - coll.update({}, mod); + coll.update(query, mod); }); }; + var exception = function (doc, mod) { + exceptionWithQuery(doc, {}, mod); + }; // document replacement modify({}, {}, {}); @@ -1764,6 +1767,45 @@ Tinytest.add("minimongo - modify", function (test) { modify({x: [null, null]}, {$set: {'x.2.a': 1}}, {x: [null, null, {a: 1}]}); exception({x: [null, null]}, {$set: {'x.1.a': 1}}); + // a.$.b + modifyWithQuery({a: [{x: 2}, {x: 4}]}, {'a.x': 4}, {$set: {'a.$.z': 9}}, + {a: [{x: 2}, {x: 4, z: 9}]}); + exception({a: [{x: 2}, {x: 4}]}, {$set: {'a.$.z': 9}}); + exceptionWithQuery({a: [{x: 2}, {x: 4}], b: 5}, {b: 5}, {$set: {'a.$.z': 9}}); + modifyWithQuery({a: [5, 6, 7]}, {a: 6}, {$set: {'a.$': 9}}, {a: [5, 9, 7]}); + modifyWithQuery({a: [{b: [{c: 9}, {c: 10}]}, {b: {c: 11}}]}, {'a.b.c': 10}, + {$unset: {'a.$.b': 1}}, {a: [{}, {b: {c: 11}}]}); + modifyWithQuery({a: [{b: [{c: 9}, {c: 10}]}, {b: {c: 11}}]}, {'a.b.c': 11}, + {$unset: {'a.$.b': 1}}, + {a: [{b: [{c: 9}, {c: 10}]}, {}]}); + modifyWithQuery({a: [1]}, {'a.0': 1}, {$set: {'a.$': 5}}, {a: [5]}); + modifyWithQuery({a: [9]}, {a: {$mod: [2, 1]}}, {$set: {'a.$': 5}}, {a: [5]}); + // Negatives don't set '$'. + exceptionWithQuery({a: [1]}, {$not: {a: 2}}, {$set: {'a.$': 5}}); + exceptionWithQuery({a: [1]}, {'a.0': {$ne: 2}}, {$set: {'a.$': 5}}); + // One $or clause works. + modifyWithQuery({a: [{x: 2}, {x: 4}]}, + {$or: [{'a.x': 4}]}, {$set: {'a.$.z': 9}}, + {a: [{x: 2}, {x: 4, z: 9}]}); + // More $or clauses throw. + exceptionWithQuery({a: [{x: 2}, {x: 4}]}, + {$or: [{'a.x': 4}, {'a.x': 4}]}, + {$set: {'a.$.z': 9}}); + // $and uses the last one. + modifyWithQuery({a: [{x: 1}, {x: 3}]}, + {$and: [{'a.x': 1}, {'a.x': 3}]}, + {$set: {'a.$.x': 5}}, + {a: [{x: 1}, {x: 5}]}); + modifyWithQuery({a: [{x: 1}, {x: 3}]}, + {$and: [{'a.x': 3}, {'a.x': 1}]}, + {$set: {'a.$.x': 5}}, + {a: [{x: 5}, {x: 3}]}); + // Same goes for the implicit AND of a document selector. + modifyWithQuery({a: [{x: 1}, {y: 3}]}, + {'a.x': 1, 'a.y': 3}, + {$set: {'a.$.z': 5}}, + {a: [{x: 1}, {y: 3, z: 5}]}); + // $inc modify({a: 1, b: 2}, {$inc: {a: 10}}, {a: 11, b: 2}); modify({a: 1, b: 2}, {$inc: {c: 10}}, {a: 1, b: 2, c: 10}); @@ -1944,12 +1986,11 @@ Tinytest.add("minimongo - modify", function (test) { {a: {}, q: {2: {r: 12}}}); exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2'}}); // tested exception({a: {b: 12}, q: []}, {$rename: {'a.b': 'q.2.r'}}); // tested - test.expect_fail(); - modify({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}}, - {a: {b: 12}, x: []}); // tested - test.expect_fail(); - modify({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}}, - {a: {b: 12}, x: []}); // tested + // These strange MongoDB behaviors throw. + // modify({a: {b: 12}, q: []}, {$rename: {'q.1': 'x'}}, + // {a: {b: 12}, x: []}); // tested + // modify({a: {b: 12}, q: []}, {$rename: {'q.1.j': 'x'}}, + // {a: {b: 12}, x: []}); // tested exception({}, {$rename: {'a': 'a'}}); exception({}, {$rename: {'a.b': 'a.b'}}); modify({a: 12, b: 13}, {$rename: {a: 'b'}}, {b: 12}); diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 1ccb3c0964..ec768fd11b 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -52,7 +52,8 @@ LocalCollection._modify = function (doc, mod, options) { var forbidArray = (op === "$rename"); var target = findModTarget(newDoc, keyparts, { noCreate: NO_CREATE_MODIFIERS[op], - forbidArray: (op === "$rename") + forbidArray: (op === "$rename"), + arrayIndex: options.arrayIndex }); var field = keyparts.pop(); modFunc(target, field, arg, keypath, newDoc); @@ -91,15 +92,18 @@ LocalCollection._modify = function (doc, mod, options) { // ['a', '01'] -> ['a', 1]). // // if forbidArray is true, return null if the keypath goes through an array. +// +// if options.arrayIndex is defined, use this for the (first) '$' in the path. var findModTarget = function (doc, keyparts, options) { options = options || {}; + var usedArrayIndex = false; for (var i = 0; i < keyparts.length; i++) { var last = (i === keyparts.length - 1); var keypart = keyparts[i]; var indexable = isIndexable(doc); - if (options.noCreate && !(indexable && keypart in doc)) - return undefined; if (!indexable) { + if (options.noCreate) + return undefined; var e = MinimongoError( "cannot use the part '" + keypart + "' to traverse " + doc); e.setPropertyError = true; @@ -108,9 +112,20 @@ var findModTarget = function (doc, keyparts, options) { if (doc instanceof Array) { if (options.forbidArray) return null; - if (isNumericKey(keypart)) { + if (keypart === '$') { + if (usedArrayIndex) + throw MinimongoError("Too many positional (i.e. '$') elements"); + if (options.arrayIndex === undefined) { + throw MinimongoError("The positional operator did not find the " + + "match needed from the query"); + } + keypart = options.arrayIndex; + usedArrayIndex = true; + } else if (isNumericKey(keypart)) { keypart = parseInt(keypart); } else { + if (options.noCreate) + return undefined; throw MinimongoError( "can't append to array using string field name [" + keypart + "]"); @@ -118,6 +133,8 @@ var findModTarget = function (doc, keyparts, options) { if (last) // handle 'a.01' keyparts[i] = keypart; + if (options.noCreate && keypart >= doc.length) + return undefined; while (doc.length < keypart) doc.push(null); if (!last) { @@ -130,8 +147,12 @@ var findModTarget = function (doc, keyparts, options) { } else { if (keypart.length && keypart.substr(0, 1) === '$') throw MinimongoError("can't set field named " + keypart); - if (!last && !(keypart in doc)) - doc[keypart] = {}; + if (!(keypart in doc)) { + if (options.noCreate) + return undefined; + if (!last) + doc[keypart] = {}; + } } if (last) From 2063999ce05df92bad420ebb63e6db5cd56b4fa2 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 23:36:26 -0800 Subject: [PATCH 095/124] Implement '$' update for $near --- packages/minimongo/minimongo_tests.js | 7 +++++++ packages/minimongo/selector.js | 17 +++++++++-------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index c75099b765..c392c46be5 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1805,6 +1805,13 @@ Tinytest.add("minimongo - modify", function (test) { {'a.x': 1, 'a.y': 3}, {$set: {'a.$.z': 5}}, {a: [{x: 1}, {y: 3, z: 5}]}); + // with $near, make sure it finds the closest one + modifyWithQuery({a: [{b: [1,1]}, + {b: [ [3,3], [4,4] ]}, + {b: [9,9]}]}, + {'a.b': {$near: [5, 5]}}, + {$set: {'a.$.b': 'k'}}, + {a: [{b: [1,1]}, {b: 'k'}, {b: [9,9]}]}); // $inc modify({a: 1, b: 2}, {$inc: {a: 10}}, {a: 11, b: 2}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 488d18ea65..2b746cb1ba 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -437,22 +437,23 @@ var VALUE_OPERATORS = { // actually show up *multiple times* in the result set, with one entry for // each within-$maxDistance branching point. branchedValues = expandArraysInBranches(branchedValues); - var minDistance = null; + var result = {result: false}; _.each(branchedValues, function (branch) { var curDistance = distance(branch.value); // Skip branches that aren't real points or are too far away. if (curDistance === null || curDistance > maxDistance) return; // Skip anything that's a tie. - if (minDistance !== null && minDistance <= curDistance) + if (result.distance !== undefined && result.distance <= curDistance) return; - minDistance = curDistance; + result.result = true; + result.distance = curDistance; + if (branch.arrayIndex === undefined) + delete result.arrayIndex; + else + result.arrayIndex = branch.arrayIndex; }); - if (minDistance !== null) { - // XXX arrayIndex! - return {result: true, distance: minDistance}; - } - return {result: false}; + return result; }; } }; From 82739804b8a482f96d8fcee48954c61ea63a7a09 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 2 Jan 2014 23:49:51 -0800 Subject: [PATCH 096/124] Implement '$' update for $elemMatch --- packages/minimongo/minimongo_tests.js | 10 +++++++++ packages/minimongo/selector.js | 29 ++++++++++++++++++--------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index c392c46be5..666362ffcc 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1772,6 +1772,8 @@ Tinytest.add("minimongo - modify", function (test) { {a: [{x: 2}, {x: 4, z: 9}]}); exception({a: [{x: 2}, {x: 4}]}, {$set: {'a.$.z': 9}}); exceptionWithQuery({a: [{x: 2}, {x: 4}], b: 5}, {b: 5}, {$set: {'a.$.z': 9}}); + // can't have two $ + exceptionWithQuery({a: [{x: [2]}]}, {'a.x': 2}, {$set: {'a.$.x.$': 9}}); modifyWithQuery({a: [5, 6, 7]}, {a: 6}, {$set: {'a.$': 9}}, {a: [5, 9, 7]}); modifyWithQuery({a: [{b: [{c: 9}, {c: 10}]}, {b: {c: 11}}]}, {'a.b.c': 10}, {$unset: {'a.$.b': 1}}, {a: [{}, {b: {c: 11}}]}); @@ -1812,6 +1814,14 @@ Tinytest.add("minimongo - modify", function (test) { {'a.b': {$near: [5, 5]}}, {$set: {'a.$.b': 'k'}}, {a: [{b: [1,1]}, {b: 'k'}, {b: [9,9]}]}); + modifyWithQuery({a: [{x: 1}, {y: 1}, {x: 1, y: 1}]}, + {a: {$elemMatch: {x: 1, y: 1}}}, + {$set: {'a.$.x': 2}}, + {a: [{x: 1}, {y: 1}, {x: 2, y: 1}]}); + modifyWithQuery({a: [{b: [{x: 1}, {y: 1}, {x: 1, y: 1}]}]}, + {'a.b': {$elemMatch: {x: 1, y: 1}}}, + {$set: {'a.$.b': 3}}, + {a: [{b: 3}]}); // $inc modify({a: 1, b: 2}, {$inc: {a: 10}}, {a: 11, b: 2}); diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 2b746cb1ba..d85bf28ae4 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -166,6 +166,14 @@ var convertElementMatcherToBranchedMatcher = function ( ret.result = _.any(expanded, function (element) { var matched = elementMatcher(element.value); + // Special case for $elemMatch: it means "true, and use this arrayIndex if + // I didn't already have one". + if (typeof matched === 'number') { + if (element.arrayIndex === undefined) + element.arrayIndex = matched; + matched = true; + } + // If some element matched, and it's tagged with an array index, include // that index in our result object. if (matched && element.arrayIndex !== undefined) @@ -387,7 +395,6 @@ var VALUE_OPERATORS = { // SAME branch. return andBranchedMatchers(branchedMatchers); }, - $near: function (operand, valueSelector, matcher, isRoot) { if (!isRoot) throw Error("$near can't be inside another $ operator"); @@ -627,26 +634,25 @@ var ELEMENT_OPERATORS = { if (!isPlainObject(operand)) throw Error("$elemMatch need an object"); - var matcher, isDocMatcher; + var subMatcher, isDocMatcher; if (isOperatorObject(operand)) { - matcher = compileValueSelector(operand, matcher); + subMatcher = compileValueSelector(operand, matcher); isDocMatcher = false; } else { // This is NOT the same as compileValueSelector(operand), and not just // because of the slightly different calling convention. // {$elemMatch: {x: 3}} means "an element has a field x:3", not // "consists only of a field x:3". Also, regexps and sub-$ are allowed. - matcher = compileDocumentSelector(operand, matcher, - {inElemMatch: true}); + subMatcher = compileDocumentSelector(operand, matcher, + {inElemMatch: true}); isDocMatcher = true; } return function (value) { if (!isArray(value)) return false; - return _.any(value, function (arrayElement) { - // XXX arrayIndex! - // XXX nesting geo stuff in here! + for (var i = 0; i < value.length; ++i) { + var arrayElement = value[i]; var arg; if (isDocMatcher) { // We can only match {$elemMatch: {b: 3}} against objects. @@ -660,8 +666,11 @@ var ELEMENT_OPERATORS = { // {a: [8]} but not {a: [[8]]} arg = [{value: arrayElement, dontIterate: true}]; } - return matcher(arg).result; - }); + // XXX support $near in $elemMatch by propagating $distance? + if (subMatcher(arg).result) + return i; // specially understood to mean "use my arrayIndex" + } + return false; }; } } From 85d8d5300c8a548f99d199a918f4c86d129c81fa Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 3 Jan 2014 13:27:14 -0800 Subject: [PATCH 097/124] Confirm that we hit the right URL when revoking tokens. Require token revoke endpoints to return JSON with a `tokenRevoked` key, to avoid being fooled by endpoints that don't understand token revocation but just happened to return 200 status codes. --- tools/auth.js | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tools/auth.js b/tools/auth.js index 8bb65091e5..e912bf0b19 100644 --- a/tools/auth.js +++ b/tools/auth.js @@ -196,15 +196,28 @@ var tryRevokeOldTokens = function (options) { } var response = result.response; - if (response.statusCode === 200) { - // Server confirms that the tokens have been revoked - // (Be careful to reread session data in case httpHelpers changed it) - data = readSessionData(); - var session = getSession(data, domain); - session.pendingRevoke = _.difference(session.pendingRevoke, tokenIds); - if (! session.pendingRevoke.length) - delete session.pendingRevoke; - writeSessionData(data); + if (response.statusCode === 200 && + response.body) { + try { + var body = JSON.parse(response.body); + if (body.tokenRevoked) { + // Server confirms that the tokens have been revoked. Checking for a + // `tokenRevoked` key in the response confirms that we hit an actual + // accounts server that understands that we were trying to revoke some + // tokens, not just a random URL that happened to return a 200 + // response. + + // (Be careful to reread session data in case httpHelpers changed it) + data = readSessionData(); + var session = getSession(data, domain); + session.pendingRevoke = _.difference(session.pendingRevoke, tokenIds); + if (! session.pendingRevoke.length) + delete session.pendingRevoke; + writeSessionData(data); + } + } catch (e) { + logoutFailWarning(domain); + } } else { logoutFailWarning(domain); } From f4e3a08baeecc9a5e0febab3aa58a9c9bc4c0295 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 3 Jan 2014 14:31:18 -0800 Subject: [PATCH 098/124] Print human-friendly error messages for common login failures. --- tools/auth.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/auth.js b/tools/auth.js index e912bf0b19..2afa6c324c 100644 --- a/tools/auth.js +++ b/tools/auth.js @@ -329,7 +329,7 @@ var logInToGalaxy = function (galaxyName) { }); var body = JSON.parse(galaxyResult.body); } catch (e) { - return { error: 'no-galaxy' }; + return { error: (body && body.error) || 'no-galaxy' }; } response = galaxyResult.response; @@ -339,7 +339,7 @@ var logInToGalaxy = function (galaxyName) { if (response.statusCode !== 200 || ! body || ! _.has(galaxyResult.setCookie, 'GALAXY_AUTH')) - return { error: 'access-denied' }; + return { error: (body && body.error) || 'access-denied' }; return { token: galaxyResult.setCookie.GALAXY_AUTH, @@ -438,8 +438,19 @@ exports.loginCommand = function (argv, showUsage) { var galaxyLoginResult = logInToGalaxy(galaxy); if (galaxyLoginResult.error) { // XXX add human readable error messages - process.stdout.write('\nLogin to ' + galaxy + ' failed: ' + - galaxyLoginResult.error + '\n'); + process.stdout.write('\nLogin to ' + galaxy + ' failed. '); + + if (galaxyLoginResult.error === 'unauthorized') { + process.stdout.write('You are not authorized for this galaxy.\n'); + } else if (galaxyLoginResult.error === 'no_oauth_server') { + process.stdout.write('The galaxy could not ' + + 'contact Meteor Accounts.\n'); + } else if (galaxyLoginResult.error === 'no_identity') { + process.stdout.write('Your login information could not be found.\n'); + } else { + process.stdout.write('Error: ' + galaxyLoginResult.error + '\n'); + } + process.exit(1); } data = readSessionData(); // be careful to reread data file after RPC From 4f7d14c1f29088fbe1eb26bd22e5ba0d7aac5385 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 3 Jan 2014 17:19:59 -0800 Subject: [PATCH 099/124] Upgrade websocket-driver to 0.3.2 This lowers the max websocket frame length from 1GB to 64MB. Note that due to #1648, this may not immediately affect existing checkouts of meteor (but will get into all release builds). --- packages/livedata/.npm/package/npm-shrinkwrap.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/livedata/.npm/package/npm-shrinkwrap.json b/packages/livedata/.npm/package/npm-shrinkwrap.json index 7e31b07e80..f48e03a46b 100644 --- a/packages/livedata/.npm/package/npm-shrinkwrap.json +++ b/packages/livedata/.npm/package/npm-shrinkwrap.json @@ -10,7 +10,7 @@ "version": "0.7.0", "dependencies": { "websocket-driver": { - "version": "0.3.1" + "version": "0.3.2" } } } From c74dd9aa622fd5e9ea9b2decbe56fdae6969c92d Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 6 Jan 2014 15:13:48 -0800 Subject: [PATCH 100/124] Add missing 'random' dependency to retry --- packages/retry/package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/retry/package.js b/packages/retry/package.js index 9c21873dcf..1ecff6d3cb 100644 --- a/packages/retry/package.js +++ b/packages/retry/package.js @@ -4,7 +4,7 @@ Package.describe({ }); Package.on_use(function (api) { - api.use('underscore', ['client', 'server']); + api.use(['underscore', 'random'], ['client', 'server']); api.export('Retry'); api.add_files('retry.js', ['client', 'server']); }); From 8800564e8009a8894ee6aa6dd7efdc606faa3572 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 6 Jan 2014 16:59:48 -0800 Subject: [PATCH 101/124] Use OplogObserveDriver for most selectors. Previously OplogObserveDriver was only used for selectors which performed equality checks against scalars. Now that we believe minimongo to be more robust in the face of more MongoDB edge cases, we use OplogObserveDriver (if configured) for any selector that minimongo can compile except those containing $near or $where. (We still do not use OplogObserveDriver for cursors with skip or limit.) --- packages/minimongo/minimongo.js | 6 ++-- packages/minimongo/selector.js | 14 ++++++--- packages/mongo-livedata/mongo_driver.js | 18 ++++++++++-- .../mongo-livedata/oplog_observe_driver.js | 29 ++++++------------- packages/mongo-livedata/oplog_tests.js | 23 ++++++++++----- 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 16918ae980..460ef06a9d 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -96,7 +96,7 @@ LocalCollection.Cursor = function (collection, selector, options) { } else { self.selector_id = undefined; self.matcher = new Minimongo.Matcher(selector, self); - self.sorter = (self.matcher.isGeoQuery() || options.sort) ? + self.sorter = (self.matcher.hasGeoQuery() || options.sort) ? new Sorter(options.sort || []) : null; } self.skip = options.skip; @@ -274,7 +274,7 @@ _.extend(LocalCollection.Cursor.prototype, { matcher: self.matcher, // not fast pathed sorter: ordered && self.sorter, distances: ( - self.matcher.isGeoQuery() && ordered && new LocalCollection._IdMap), + self.matcher.hasGeoQuery() && ordered && new LocalCollection._IdMap), results_snapshot: null, ordered: ordered, cursor: self, @@ -411,7 +411,7 @@ LocalCollection.Cursor.prototype._getRawObjects = function (ordered, // in the observeChanges case, distances is actually part of the "query" (ie, // live results set) object. in other cases, distances is only used inside // this function. - if (self.matcher.isGeoQuery() && ordered) { + if (self.matcher.hasGeoQuery() && ordered) { if (distances) distances.clear(); else diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index d85bf28ae4..83212f3466 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -25,7 +25,9 @@ Minimongo.Matcher = function (selector) { // path (eg, $where). self._paths = {}; // Set to true if compilation finds a $near. - self._isGeoQuery = false; + self._hasGeoQuery = false; + // Set to true if compilation finds a $where. + self._hasWhere = false; // Set to false if compilation finds anything other than a simple equality on // some fields. self._isEquality = true; @@ -38,8 +40,11 @@ _.extend(Minimongo.Matcher.prototype, { documentMatches: function (doc) { return this._docMatcher(doc); }, - isGeoQuery: function () { - return this._isGeoQuery; + hasGeoQuery: function () { + return this._hasGeoQuery; + }, + hasWhere: function () { + return this._hasWhere; }, isEquality: function () { return this._isEquality; @@ -308,6 +313,7 @@ var LOGICAL_OPERATORS = { $where: function (selectorValue, matcher) { // Record that *any* path may be used. matcher._recordPathUsed(''); + matcher._hasWhere = true; if (!(selectorValue instanceof Function)) { // XXX MongoDB seems to have more complex logic to decide where or or not // to add "return"; not sure exactly what it is. @@ -398,7 +404,7 @@ var VALUE_OPERATORS = { $near: function (operand, valueSelector, matcher, isRoot) { if (!isRoot) throw Error("$near can't be inside another $ operator"); - matcher._isGeoQuery = true; + matcher._hasGeoQuery = true; // There are two kinds of geodata in MongoDB: coordinate pairs and // GeoJSON. They use different distance metrics, too. GeoJSON queries are diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 036f236885..bdb9207571 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -987,15 +987,27 @@ MongoConnection.prototype._observeChanges = function ( if (firstHandle) { var driverClass = PollingObserveDriver; - if (self._oplogHandle && !ordered && !callbacks._testOnlyPollCallback - && OplogObserveDriver.cursorSupported(cursorDescription)) { - driverClass = OplogObserveDriver; + var matcher; + if (self._oplogHandle && !ordered && !callbacks._testOnlyPollCallback) { + try { + matcher = new Minimongo.Matcher(cursorDescription.selector); + } catch (e) { + // Ignore and avoid oplog driver. eg, maybe we're trying to compile some + // newfangled $selector that minimongo doesn't support yet. + // XXX make all compilation errors MinimongoError or something + // so that this doesn't ignore unrelated exceptions + } + if (matcher + && OplogObserveDriver.cursorSupported(cursorDescription, matcher)) { + driverClass = OplogObserveDriver; + } } observeDriver = new driverClass({ cursorDescription: cursorDescription, mongoHandle: self, multiplexer: multiplexer, ordered: ordered, + matcher: matcher, // ignored by polling _testOnlyPollCallback: callbacks._testOnlyPollCallback }); diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index 109a5dca37..fc0d47c012 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -32,8 +32,7 @@ OplogObserveDriver = function (options) { self._published = new LocalCollection._IdMap; var selector = self._cursorDescription.selector; - self._matcher = new Minimongo.Matcher( - self._cursorDescription.selector); + self._matcher = options.matcher; var projection = self._cursorDescription.options.fields || {}; self._projectionFn = LocalCollection._compileProjection(projection); // Projection function, result of combining important fields for selector and @@ -460,7 +459,7 @@ _.extend(OplogObserveDriver.prototype, { // Does our oplog tailing code support this cursor? For now, we are being very // conservative and allowing only simple queries with simple options. // (This is a "static method".) -OplogObserveDriver.cursorSupported = function (cursorDescription) { +OplogObserveDriver.cursorSupported = function (cursorDescription, matcher) { // First, check the options. var options = cursorDescription.options; @@ -486,23 +485,13 @@ OplogObserveDriver.cursorSupported = function (cursorDescription) { } } - // For now, we're just dealing with equality queries: no $operators, regexps, - // or $and/$or/$where/etc clauses. We can expand the scope of what we're - // comfortable processing later. ($where will get pretty scary since it will - // allow selector processing to yield!) - return _.all(cursorDescription.selector, function (value, field) { - // No logical operators like $and. - if (field.substr(0, 1) === '$') - return false; - // We only allow scalars, not sub-documents or $operators or RegExp. - // XXX Date would be easy too, though I doubt anyone is doing equality - // lookups on dates - return typeof value === "string" || - typeof value === "number" || - typeof value === "boolean" || - value === null || - value instanceof Meteor.Collection.ObjectID; - }); + // We don't allow the following selectors: + // - $where (not confident that we provide the same JS environment + // as Mongo, and can yield!) + // - $near (has "interesting" properties in MongoDB, like the possibility + // of returning an ID multiple times, though even polling maybe + // have a bug there + return !matcher.hasWhere() && !matcher.hasGeoQuery(); }; var modifierCanBeDirectlyApplied = function (modifier) { diff --git a/packages/mongo-livedata/oplog_tests.js b/packages/mongo-livedata/oplog_tests.js index dc403c3766..a94527f322 100644 --- a/packages/mongo-livedata/oplog_tests.js +++ b/packages/mongo-livedata/oplog_tests.js @@ -3,9 +3,9 @@ var OplogCollection = new Meteor.Collection("oplog-" + Random.id()); Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) { var supported = function (expected, selector) { var cursor = OplogCollection.find(selector); - test.equal( - MongoTest.OplogObserveDriver.cursorSupported(cursor._cursorDescription), - expected); + var handle = cursor.observeChanges({added: function () {}}); + test.equal(!!handle._multiplexer._observeDriver._usesOplog, expected); + handle.stop(); }; supported(true, "asdf"); @@ -25,8 +25,17 @@ Tinytest.add("mongo-livedata - oplog - cursorSupported", function (test) { supported(true, {}); - supported(false, {$and: [{foo: "asdf"}, {bar: "baz"}]}); - supported(false, {foo: {x: 1}}); - supported(false, {foo: {$gt: 1}}); - supported(false, {foo: [1, 2, 3]}); + supported(true, {$and: [{foo: "asdf"}, {bar: "baz"}]}); + supported(true, {foo: {x: 1}}); + supported(true, {foo: {$gt: 1}}); + supported(true, {foo: [1, 2, 3]}); + + // No $where. + supported(false, {$where: "xxx"}); + supported(false, {$and: [{foo: "adsf"}, {$where: "xxx"}]}); + // No geoqueries. + supported(false, {x: {$near: [1,1]}}); + // Nothing Minimongo doesn't understand. (Minimongo happens to fail to + // implement $elemMatch inside $all which MongoDB supports.) + supported(false, {x: {$all: [{$elemMatch: {y: 2}}]}}); }); From add4f6e0155faa31e1e2a67454a423c387fab709 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 6 Jan 2014 17:54:16 -0800 Subject: [PATCH 102/124] Disallow {fields:{_id:0}} in observeChanges This implies it is not allowed in `observe` either, or in cursors returned from publish functions, or in cursors used in {{#each}} Why? observeChanges and DDP publication use the ID as part of the callback/message, and eliding it completely breaks them. Meteor UI uses the ID with {{#each}} to properly move nodes around instead of re-rendering. We could try to allow it for `observe` outside of {{#each}}, but it would feel somewhat inconsistent. --- History.md | 4 +++ docs/client/api.html | 25 +++++++++++++------ packages/minimongo/minimongo.js | 3 +++ packages/minimongo/minimongo_tests.js | 8 ++++++ packages/mongo-livedata/mongo_driver.js | 8 ++++++ .../mongo-livedata/observe_changes_tests.js | 7 ++++++ 6 files changed, 48 insertions(+), 7 deletions(-) diff --git a/History.md b/History.md index 05d090bf21..1cbdd9428f 100644 --- a/History.md +++ b/History.md @@ -2,6 +2,10 @@ * Hash login tokens before storing them in the database. +* Cursors with a field specifier containing `{_id: 0}` can no longer be used + with `observeChanges` or `observe`. This includes the implicit calls to these + functions that are done when returning a cursor from a publish function or + using `{{#each}}`. ## v0.7.0.1 diff --git a/docs/client/api.html b/docs/client/api.html index 8a63b56bc9..8435a3952e 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -1319,21 +1319,32 @@ it's up to you to be sure. Queries can specify a particular set of fields to include or exclude from the result object. -To exclude certain fields from the result objects, the field specifier -is a dictionary whose keys are field names and whose values are `0`. +To exclude specific fields from the result objects, the field specifier is a +dictionary whose keys are field names and whose values are `0`. All unspecified +fields are included. // Users.find({}, {fields: {password: 0, hash: 0}}) -To return an object that only includes the specified field, use `1` as +To include only specific fields in the result documents, use `1` as the value. The `_id` field is still included in the result. // Users.find({}, {fields: {firstname: 1, lastname: 1}}) -It is not possible to mix inclusion and exclusion styles (except for the cases -when `_id` is included by default or explicitly excluded). Field operators such -as `$` and `$elemMatch` are not available on the client side yet. +With one exception, it is not possible to mix inclusion and exclusion styles: +the keys must either be all 1 or all 0. The exception is that you may specify +`_id: 0` in an inclusion specifier, which will leave `_id` out of the result +object as well. However, such field specifiers can not be used with +[`observeChanges`](#observe_changes), [`observe`](#observe), cursors returned +from a [publish function](#meteor_publish), or cursors used in +`{{dstache}}#each}}` in a template. They may be used with [`fetch`](#fetch), +[`findOne`](#findone), [`forEach`](#foreach), and [`map`](#map). -More advanced example: + +Field +operators such as `$` and `$elemMatch` are not available on the client side +yet. + +A more advanced example: Users.insert({ alterEgos: [{ name: "Kira", alliance: "murderer" }, { name: "L", alliance: "police" }], diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index 460ef06a9d..aa5a8b60cd 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -270,6 +270,9 @@ _.extend(LocalCollection.Cursor.prototype, { if (!options._allow_unordered && !ordered && (self.skip || self.limit)) throw new Error("must use ordered observe with skip or limit"); + if (self.fields && (self.fields._id === 0 || self.fields._id === false)) + throw Error("You may not observe a cursor with {fields: {_id: 0}}"); + var query = { matcher: self.matcher, // not fast pathed sorter: ordered && self.sorter, diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 666362ffcc..cd726a5f52 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -1432,6 +1432,14 @@ Tinytest.add("minimongo - observe ordered with projection", function (test) { c.insert({_id: idA2, a:2}); test.equal(operations.shift(), undefined); + var cursor = c.find({}, {fields: {a: 1, _id: 0}}); + test.throws(function () { + cursor.observeChanges({added: function () {}}); + }); + test.throws(function () { + cursor.observe({added: function () {}}); + }); + // test initial inserts (and backwards sort) handle = c.find({}, {sort: {a: -1}, fields: { a: 1 } }).observe(cbs); test.equal(operations.shift(), ['added', {a:2}, 0, null]); diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index bdb9207571..2036f5cd16 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -957,6 +957,14 @@ MongoConnection.prototype._observeChanges = function ( return self._observeChangesTailable(cursorDescription, ordered, callbacks); } + // You may not filter out _id when observing changes, because the id is a core + // part of the observeChanges API. + if (cursorDescription.options.fields && + (cursorDescription.options.fields._id === 0 || + cursorDescription.options.fields._id === false)) { + throw Error("You may not observe a cursor with {fields: {_id: 0}}"); + } + var observeKey = JSON.stringify( _.extend({ordered: ordered}, cursorDescription)); diff --git a/packages/mongo-livedata/observe_changes_tests.js b/packages/mongo-livedata/observe_changes_tests.js index 1831718afc..db6d54d4c2 100644 --- a/packages/mongo-livedata/observe_changes_tests.js +++ b/packages/mongo-livedata/observe_changes_tests.js @@ -25,6 +25,7 @@ _.each ([{added:'added', forceOrdered: true}, function (logger) { var barid = c.insert({thing: "stuff"}); var fooid = c.insert({noodles: "good", bacon: "bad", apples: "ok"}); + var handle = c.find(fooid).observeChanges(logger); if (added === 'added') logger.expectResult(added, [fooid, {noodles: "good", bacon: "bad",apples: "ok"}]); @@ -43,6 +44,12 @@ _.each ([{added:'added', forceOrdered: true}, c.insert({noodles: "good", bacon: "bad", apples: "ok"}); logger.expectNoResult(); handle.stop(); + + var badCursor = c.find({}, {fields: {noodles: 1, _id: false}}); + test.throws(function () { + badCursor.observeChanges(logger); + }); + onComplete(); }); }); From f7c3e7621c5ec95c56bd1802013cbe4f55da6764 Mon Sep 17 00:00:00 2001 From: Denis Gorbachev Date: Thu, 26 Dec 2013 09:32:29 +0300 Subject: [PATCH 103/124] Update concepts.html --- docs/client/concepts.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 4fb6ffa178..e93cd6b133 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -558,8 +558,8 @@ database cursor is passed to `#each`, it will wire up all of the machinery to efficiently add and move DOM nodes as new results enter the query. -Helpers can take arguments, and they receive the current template data -in `this`: +Helpers can take arguments, and they receive the current template context data +in `this`. Note that some block helpers change context (notably `each`): // in a JavaScript file Template.players.leagueIs = function (league) { From cc1d47b5c5950e9b40da5c92c7504fa01034af07 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 16:34:18 -0800 Subject: [PATCH 104/124] Wording tweak. --- docs/client/concepts.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index e93cd6b133..0475ddb6e5 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -559,7 +559,8 @@ machinery to efficiently add and move DOM nodes as new results enter the query. Helpers can take arguments, and they receive the current template context data -in `this`. Note that some block helpers change context (notably `each`): +in `this`. Note that some block helpers change the current context (notably +`each` and `with`): // in a JavaScript file Template.players.leagueIs = function (league) { From ce77adc22ecbc433c25d3a82eef07e4f4a865bbf Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 17:00:18 -0800 Subject: [PATCH 105/124] Remove warning about update/$ --- docs/client/api.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index 8435a3952e..972f2e60ad 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -678,8 +678,6 @@ In this release, Minimongo has some limitations: * `$pull` in modifiers can only accept certain kinds of selectors. -* `$` to denote the matched array position is not -supported in modifier. * `findAndModify`, aggregate functions, and map/reduce aren't supported. From 53de3f21ba8921ac3c613e6abdddff2630364874 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 17:15:06 -0800 Subject: [PATCH 106/124] Improve docs for server-to-server collections Fixes #1723. --- docs/client/api.html | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index 972f2e60ad..1e8fcaf4b0 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -586,23 +586,30 @@ access the same collection using the same API. Specifically, when you pass a `name`, here's what happens: -* On the server, a collection with that name is created on a backend -Mongo server. When you call methods on that collection on the server, -they translate directly into normal Mongo operations (after checking that -they match your [access control rules](#allow)). +* On the server (if you do not specify a `connection`), a collection with that +name is created on a backend Mongo server. When you call methods on that +collection on the server, they translate directly into normal Mongo operations +(after checking that they match your [access control rules](#allow)). -* On the client, a Minimongo instance is -created. Minimongo is essentially an in-memory, non-persistent -implementation of Mongo in pure JavaScript. It serves as a local cache -that stores just the subset of the database that this client is working -with. Queries on the client ([`find`](#find)) are served directly out of -this cache, without talking to the server. +* On the client (and on the server if you specify a `connection`), a Minimongo +instance is created. Minimongo is essentially an in-memory, non-persistent +implementation of Mongo in pure JavaScript. It serves as a local cache that +stores just the subset of the database that this client is working with. Queries +([`find`](#find)) on these collections are served directly out of this cache, +without talking to the server. * When you write to the database on the client ([`insert`](#insert), -[`update`](#update), [`remove`](#remove)), the command is executed -immediately on the client, and, simultaneously, it's shipped up to the -server and executed there too. The `livedata` package is -responsible for this. +[`update`](#update), [`remove`](#remove)), the command is executed locally +immediately, and, simultaneously, it's sent to the server and executed +there too. This happens via [stubs](#meteor_methods), because writes are +implemented as methods. + +{{#note}} +When, on the server, you write to a collection which has a specified +`connection` to another server, it sends the corresponding method to the other +server and receives the changed values back from it over DDP. Unlike on the +client, it does not execute the write locally first. +{{/note}} If you pass `null` as the `name`, then you're creating a local collection. It's not synchronized anywhere; it's just a local scratchpad From c6bea042b7a0a39bea6dfa529cfb3a1ac0fe3d15 Mon Sep 17 00:00:00 2001 From: Maxime Quandalle Date: Fri, 3 Jan 2014 14:34:43 +0100 Subject: [PATCH 107/124] Update coffeescript.html --- docs/client/packages/coffeescript.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/client/packages/coffeescript.html b/docs/client/packages/coffeescript.html index d01284cf4f..107690da33 100644 --- a/docs/client/packages/coffeescript.html +++ b/docs/client/packages/coffeescript.html @@ -8,8 +8,8 @@ braces and parentheses. The code compiles one-to-one into the equivalent JS, and there is no interpretation at runtime. CoffeeScript is supported on both the client and the server. Files -ending with `.coffee` or `.litcoffee` are automatically compiled to -JavaScript. +ending with `.coffee`, `.litcoffee` or `.coffee.md` are automatically +compiled to JavaScript. ### Namespacing and CoffeeScript From 8a3f24765b4413d2ce7179ba2906332f2693cb36 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 17:23:49 -0800 Subject: [PATCH 108/124] Add Oxford comma http://www.washingtontimes.com/news/2013/dec/11/comma-twitter-erupts-over-obama-castro-marriage/ --- docs/client/packages/coffeescript.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client/packages/coffeescript.html b/docs/client/packages/coffeescript.html index 107690da33..efc55c82ab 100644 --- a/docs/client/packages/coffeescript.html +++ b/docs/client/packages/coffeescript.html @@ -8,7 +8,7 @@ braces and parentheses. The code compiles one-to-one into the equivalent JS, and there is no interpretation at runtime. CoffeeScript is supported on both the client and the server. Files -ending with `.coffee`, `.litcoffee` or `.coffee.md` are automatically +ending with `.coffee`, `.litcoffee`, or `.coffee.md` are automatically compiled to JavaScript. ### Namespacing and CoffeeScript From bab936eac99004b6cd14b4ed8b279862e2d7e606 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 20:17:47 -0800 Subject: [PATCH 109/124] Patch _.each to not treat {length: 5} as an array Specifically, in all Underscore "collection" functions which treat their arguments polymorphically as either "object-like" or "array-like", don't treat arguments with `x.constructor === Object` as arrays (except for the 'arguments' object). Fixes #594. Fixes #1737. --- docs/client/packages/underscore.html | 11 +++++++ .../mongo-livedata/mongo_livedata_tests.js | 32 +++++++++++++++++-- packages/underscore/package.js | 5 +++ packages/underscore/underscore.js | 18 ++++++++--- 4 files changed, 60 insertions(+), 6 deletions(-) diff --git a/docs/client/packages/underscore.html b/docs/client/packages/underscore.html index bad129e09d..cdf2fc1d33 100644 --- a/docs/client/packages/underscore.html +++ b/docs/client/packages/underscore.html @@ -18,6 +18,17 @@ if you do use underscore in your application, you should still add the package as we will remove the default underscore in the future. {{/warning}} +{{#warning}} +We have slightly modified the way Underscore differentiates between +objects and arrays in [collection functions](http://underscorejs.org/#each). +The original Underscore logic is to treat any object with a numeric `length` +property as an array (which helps it work properly on +[`NodeList`s](https://developer.mozilla.org/en-US/docs/Web/API/NodeList)). +In Meteor's version of Underscore, objects with a numeric `length` property +are treated as objects if they have no prototype (specifically, if +`x.constructor === Object`. +{{/warning}} + {{/better_markdown}} diff --git a/packages/mongo-livedata/mongo_livedata_tests.js b/packages/mongo-livedata/mongo_livedata_tests.js index dbb4a65d36..db770a6179 100644 --- a/packages/mongo-livedata/mongo_livedata_tests.js +++ b/packages/mongo-livedata/mongo_livedata_tests.js @@ -757,18 +757,46 @@ testAsyncMulti('mongo-livedata - empty documents, ' + idGeneration, [ } var coll = new Meteor.Collection(collectionName, collectionOptions); - var docId; coll.insert({}, expect(function (err, id) { test.isFalse(err); test.isTrue(id); - docId = id; var cursor = coll.find(); test.equal(cursor.count(), 1); })); } ]); +// See https://github.com/meteor/meteor/issues/594. +testAsyncMulti('mongo-livedata - document with length, ' + idGeneration, [ + function (test, expect) { + var self = this; + var collectionName = Random.id(); + if (Meteor.isClient) { + Meteor.call('createInsecureCollection', collectionName); + Meteor.subscribe('c-' + collectionName); + } + + self.coll = new Meteor.Collection(collectionName, collectionOptions); + + self.coll.insert({foo: 'x', length: 0}, expect(function (err, id) { + test.isFalse(err); + test.isTrue(id); + self.docId = id; + test.equal(self.coll.findOne(self.docId), + {_id: self.docId, foo: 'x', length: 0}); + })); + }, + function (test, expect) { + var self = this; + self.coll.update(self.docId, {$set: {length: 5}}, expect(function (err) { + test.isFalse(err); + test.equal(self.coll.findOne(self.docId), + {_id: self.docId, foo: 'x', length: 5}); + })); + } +]); + testAsyncMulti('mongo-livedata - document with a date, ' + idGeneration, [ function (test, expect) { var collectionName = Random.id(); diff --git a/packages/underscore/package.js b/packages/underscore/package.js index c070df7437..0d0c0459c3 100644 --- a/packages/underscore/package.js +++ b/packages/underscore/package.js @@ -20,5 +20,10 @@ Package.on_use(function (api) { api.export('_'); + // NOTE: we patch _.each and various other functions that polymorphically take + // objects, arrays, and array-like objects (such as the querySelectorAll + // return value, document.images, and 'arguments') such that objects with a + // numeric length field whose constructor === Object are still treated as + // objects, not as arrays. Search for looksLikeArray. api.add_files(['pre.js', 'underscore.js', 'post.js']); }); diff --git a/packages/underscore/underscore.js b/packages/underscore/underscore.js index b50115df5c..12a74d5f55 100644 --- a/packages/underscore/underscore.js +++ b/packages/underscore/underscore.js @@ -70,6 +70,16 @@ // Collection Functions // -------------------- + // METEOR CHANGE: _.each({length: 5}) should be treated like an object, not an + // array. This looksLikeArray function is introduced by Meteor, and replaces + // all instances of `obj.length === +obj.length`. + // https://github.com/meteor/meteor/issues/594 + // https://github.com/jashkenas/underscore/issues/770 + var looksLikeArray = function (obj) { + return (obj.length === +obj.length + && (_.isArguments(obj) || obj.constructor !== Object)); + }; + // The cornerstone, an `each` implementation, aka `forEach`. // Handles objects with the built-in `forEach`, arrays, and raw objects. // Delegates to **ECMAScript 5**'s native `forEach` if available. @@ -77,7 +87,7 @@ if (obj == null) return; if (nativeForEach && obj.forEach === nativeForEach) { obj.forEach(iterator, context); - } else if (obj.length === +obj.length) { + } else if (looksLikeArray(obj)) { for (var i = 0, length = obj.length; i < length; i++) { if (iterator.call(context, obj[i], i, obj) === breaker) return; } @@ -134,7 +144,7 @@ return initial ? obj.reduceRight(iterator, memo) : obj.reduceRight(iterator); } var length = obj.length; - if (length !== +length) { + if (!looksLikeArray(obj)) { var keys = _.keys(obj); length = keys.length; } @@ -381,14 +391,14 @@ _.toArray = function(obj) { if (!obj) return []; if (_.isArray(obj)) return slice.call(obj); - if (obj.length === +obj.length) return _.map(obj, _.identity); + if (looksLikeArray(obj)) return _.map(obj, _.identity); return _.values(obj); }; // Return the number of elements in an object. _.size = function(obj) { if (obj == null) return 0; - return (obj.length === +obj.length) ? obj.length : _.keys(obj).length; + return (looksLikeArray(obj)) ? obj.length : _.keys(obj).length; }; // Array Functions From 6a6df0bff72098b19651f0c7efad83e375bbe6cc Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 21:17:36 -0800 Subject: [PATCH 110/124] Add #594 fix to History --- History.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/History.md b/History.md index 1cbdd9428f..485db81e2e 100644 --- a/History.md +++ b/History.md @@ -7,6 +7,10 @@ functions that are done when returning a cursor from a publish function or using `{{#each}}`. +* Patch Underscore to not treat plain objects (`x.constructor === Object`) + with numeric `length` fields as arrays. Among other things, this allows you + to use documents with numeric `length` fields with Mongo. #594 #1737 + ## v0.7.0.1 * Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the From 837f842e7b229a351917f6285477e0cd299aa7c5 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 7 Jan 2014 22:53:36 -0800 Subject: [PATCH 111/124] Fix $type:4 queries and sorts with numeric indices Add lots of sort tests. All new tests in this commit have been verified against MongoDB (2.5). --- packages/minimongo/minimongo_tests.js | 89 ++++++++++++++++++++++++++- packages/minimongo/selector.js | 6 +- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index cd726a5f52..e4ed903a33 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -624,6 +624,9 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$type: 11}}, {a: 'x'}); nomatch({a: {$type: 11}}, {}); + // The normal rule for {$type:4} (4 means array) is that it NOT good enough to + // just have an array that's the leaf that matches the path. (An array inside + // that array is good, though.) nomatch({a: {$type: 4}}, {a: []}); nomatch({a: {$type: 4}}, {a: [1]}); // tested against mongodb match({a: {$type: 1}}, {a: [1]}); @@ -635,6 +638,10 @@ Tinytest.add("minimongo - selector_compiler", function (test) { nomatch({a: {$type: 1}}, {a: ["1", []]}); match({a: {$type: 2}}, {a: ["1", []]}); match({a: {$type: 4}}, {a: ["1", []]}); // tested against mongodb + // An exception to the normal rule is that an array found via numeric index is + // examined itself, and its elements are not. + match({'a.0': {$type: 4}}, {a: [[0]]}); + nomatch({'a.0': {$type: 1}}, {a: [[0]]}); // regular expressions match({a: /a/}, {a: 'cat'}); @@ -1526,7 +1533,7 @@ Tinytest.add("minimongo - ordering", function (test) { // document ordering under a sort specification var verify = function (sorts, docs) { - _.each(sorts, function (sort) { + _.each(_.isArray(sorts) ? sorts : [sorts], function (sort) { var sorter = new MinimongoTest.Sorter(sort); assert_ordering(test, sorter.getComparator(), docs); }); @@ -1562,7 +1569,87 @@ Tinytest.add("minimongo - ordering", function (test) { new MinimongoTest.Sorter(123); }); + // No sort spec implies everything equal. test.equal(new MinimongoTest.Sorter({}).getComparator()({a:1}, {a:2}), 0); + + // All sorts of array edge cases! + // Increasing sort sorts by the smallest element it finds; 1 < 2. + verify({a: 1}, [ + {a: [1, 10, 20]}, + {a: [5, 2, 99]} + ]); + // Decreasing sorts by largest it finds; 99 > 20. + verify({a: -1}, [ + {a: [5, 2, 99]}, + {a: [1, 10, 20]} + ]); + // Can also sort by specific array indices. + verify({'a.1': 1}, [ + {a: [5, 2, 99]}, + {a: [1, 10, 20]} + ]); + // We do NOT expand sub-arrays, so the minimum in the second doc is 5, not + // -20. (Numbers always sort before arrays.) + verify({a: 1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [-5, -20], 18]} + ]); + // The maximum in each of these is the array, since arrays are "greater" than + // numbers. And [10, 15] is greater than [-5, -20]. + verify({a: -1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [-5, -20], 18]} + ]); + // 'a.0' here ONLY means "first element of a", not "first element of something + // found in a", so it CANNOT find the 10 or -5. + verify({'a.0': 1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [-5, -20], 18]} + ]); + verify({'a.0': -1}, [ + {a: [5, [-5, -20], 18]}, + {a: [1, [10, 15], 20]} + ]); + // Similarly, this is just comparing [-5,-20] to [10, 15]. + verify({'a.1': 1}, [ + {a: [5, [-5, -20], 18]}, + {a: [1, [10, 15], 20]} + ]); + verify({'a.1': -1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [-5, -20], 18]} + ]); + // Here we are just comparing [10,15] directly to [19,3] (and NOT also + // iterating over the numbers; this is implemented by setting dontIterate in + // makeLookupFunction). So [10,15]<[19,3] even though 3 is the smallest + // number you can find there. + verify({'a.1': 1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [19, 3], 18]} + ]); + verify({'a.1': -1}, [ + {a: [5, [19, 3], 18]}, + {a: [1, [10, 15], 20]} + ]); + // Minimal elements are 1 and 5. + verify({a: 1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [19, 3], 18]} + ]); + // Maximal elements are [19,3] and [10,15] (because arrays sort higher than + // numbers), even though there's a 20 floating around. + verify({a: -1}, [ + {a: [5, [19, 3], 18]}, + {a: [1, [10, 15], 20]} + ]); + // Maximal elements are [10,15] and [3,19]. [10,15] is bigger even though 19 + // is the biggest number in them, because array comparison is lexicographic. + verify({a: -1}, [ + {a: [1, [10, 15], 20]}, + {a: [5, [3, 19], 18]} + ]); + + }); Tinytest.add("minimongo - sort", function (test) { diff --git a/packages/minimongo/selector.js b/packages/minimongo/selector.js index 83212f3466..f308d5160e 100644 --- a/packages/minimongo/selector.js +++ b/packages/minimongo/selector.js @@ -824,7 +824,11 @@ expandArraysInBranches = function (branches, skipTheArrays) { var branchesOut = []; _.each(branches, function (branch) { var thisIsArray = isArray(branch.value); - if (!skipTheArrays || !thisIsArray) { + // We include the branch itself, *UNLESS* we it's an array that we're going + // to iterate and we're told to skip arrays. (That's right, we include some + // arrays even skipTheArrays is true: these are arrays that were found via + // explicit numerical indices.) + if (!(skipTheArrays && thisIsArray && !branch.dontIterate)) { branchesOut.push({ value: branch.value, arrayIndex: branch.arrayIndex From 8d6e2c72e5ef076606b345145b11bcd09bfc891b Mon Sep 17 00:00:00 2001 From: Sashko Stubailo Date: Wed, 8 Jan 2014 11:31:31 -0800 Subject: [PATCH 112/124] Fix typo in callback name in Facebook package --- packages/facebook/facebook_client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/facebook/facebook_client.js b/packages/facebook/facebook_client.js index c695c0305a..7eb09fe52a 100644 --- a/packages/facebook/facebook_client.js +++ b/packages/facebook/facebook_client.js @@ -34,6 +34,6 @@ Facebook.requestCredential = function (options, credentialRequestCompleteCallbac Oauth.showPopup( loginUrl, - _.bind(credentialRequestCompleteCallack, null, credentialToken) + _.bind(credentialRequestCompleteCallback, null, credentialToken) ); }; From bd9e5d805776448fe435ba8e002dc7cb7a8578a6 Mon Sep 17 00:00:00 2001 From: Sashko Stubailo Date: Wed, 8 Jan 2014 14:42:01 -0800 Subject: [PATCH 113/124] Add log warning when there is an error with OAuth --- packages/oauth/oauth_server.js | 5 ++++- packages/oauth/package.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/oauth/oauth_server.js b/packages/oauth/oauth_server.js index 23cc1f3e18..736748e8ee 100644 --- a/packages/oauth/oauth_server.js +++ b/packages/oauth/oauth_server.js @@ -162,7 +162,10 @@ var ensureConfigured = function(serviceName) { Oauth._renderOauthResults = function(res, query) { // We support ?close and ?redirect=URL. Any other query should // just serve a blank page - if ('close' in query) { // check with 'in' because we don't set a value + if (query.error) { + Log.warn("Error in Oauth Server: " + query.error); + closePopup(res); + } else if ('close' in query) { // check with 'in' because we don't set a value closePopup(res); } else if (query.redirect) { // Only redirect to URLs on the same domain as this app. diff --git a/packages/oauth/package.js b/packages/oauth/package.js index 85659c45fc..4c2fdd115d 100644 --- a/packages/oauth/package.js +++ b/packages/oauth/package.js @@ -6,7 +6,7 @@ Package.describe({ Package.on_use(function (api) { api.use('routepolicy', 'server'); api.use('webapp', 'server'); - api.use(['underscore', 'service-configuration'], 'server'); + api.use(['underscore', 'service-configuration', 'logging'], 'server'); api.export('Oauth'); api.export('OauthTest', 'server', {testOnly: true}); From 1ee8f2aff6aef7599e5a12f134bf1048ea166c1e Mon Sep 17 00:00:00 2001 From: Sashko Stubailo Date: Wed, 8 Jan 2014 11:26:57 -0800 Subject: [PATCH 114/124] Fix issues with meteorid popup --- packages/meteorid/meteorid_client.js | 5 ++++- packages/meteorid/meteorid_common.js | 2 +- packages/meteorid/meteorid_configure.html | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/meteorid/meteorid_client.js b/packages/meteorid/meteorid_client.js index bf935b1742..46a57e867d 100644 --- a/packages/meteorid/meteorid_client.js +++ b/packages/meteorid/meteorid_client.js @@ -26,6 +26,9 @@ MeteorId.requestCredential = function (credentialRequestCompleteCallback) { Oauth.showPopup( loginUrl, _.bind(credentialRequestCompleteCallback, null, credentialToken), - { height: 406 } + { + width: 430, + height: 406 + } ); }; diff --git a/packages/meteorid/meteorid_common.js b/packages/meteorid/meteorid_common.js index 21a65afeed..034221f57a 100644 --- a/packages/meteorid/meteorid_common.js +++ b/packages/meteorid/meteorid_common.js @@ -1,2 +1,2 @@ // XXX fill me in! -METEORID_URL = ""; +METEORID_URL = "http://10.0.2.2:3000"; diff --git a/packages/meteorid/meteorid_configure.html b/packages/meteorid/meteorid_configure.html index 5d04c620d3..5961a5cca4 100644 --- a/packages/meteorid/meteorid_configure.html +++ b/packages/meteorid/meteorid_configure.html @@ -3,7 +3,7 @@ First, you'll need to get a MeteorId Client ID. Set Authorized Redirect URIs to: - {{siteUrl}}_oauth/meteor?close + {{siteUrl}}_oauth/meteorId?close

From 56d60907cc96eb5953aade03da1f70400356310e Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 18 Nov 2013 09:55:08 -0800 Subject: [PATCH 115/124] Simplify `logoutOtherClients` and fix `onReconnect` race. If we called `login` and then called `logoutOtherClients` before the login result was recieved, then we would end up with no `onReconnect` callback. Fixed by just leaving `onReconnect` as it is when calling `logoutOtherClients` -- we were only replacing `onReconnect` for the sake of tests that have since been rewritten much more cleanly. Fixes #1616. --- packages/accounts-base/accounts_client.js | 26 +++++++++++------------ 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/packages/accounts-base/accounts_client.js b/packages/accounts-base/accounts_client.js index 579ec34474..0e7df5582b 100644 --- a/packages/accounts-base/accounts_client.js +++ b/packages/accounts-base/accounts_client.js @@ -194,22 +194,20 @@ Meteor.logout = function (callback) { }; Meteor.logoutOtherClients = function (callback) { - // Our connection is going to be closed, but we don't want to call the - // onReconnect handler until the result comes back for this method, because - // the token will have been deleted on the server. Instead, wait until we get - // a new token and call the reconnect handler with that. - // XXX this is messy. - // XXX what if login gets called before the callback runs? - var origOnReconnect = Meteor.connection.onReconnect; - var userId = Meteor.userId(); - Meteor.connection.onReconnect = null; - Meteor.apply('logoutOtherClients', [], { wait: true }, + // Call the `logoutOtherClients` method and store the login token that we get + // back. The server is not supposed to close connections on the old token for + // 10 seconds, so we should have time to store our new token before being + // disconnected. If we get disconnected, then we'll immediately reconnect with + // the new token. If for some reason we get disconnected before storing the + // new token, then the worst that will happen is that we'll have a flicker + // from trying to log in with the old token before storing and logging in with + // the new one. + Meteor.apply('logoutOtherClients', [], function (error, result) { - Meteor.connection.onReconnect = origOnReconnect; - if (! error) + if (! error) { + var userId = Meteor.userId(); storeLoginToken(userId, result.token, result.tokenExpires); - Meteor.connection.onReconnect(); - callback && callback(error); + } }); }; From 44629cf80055a84f2ee55377dec795b363790659 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 18 Nov 2013 15:26:08 -0800 Subject: [PATCH 116/124] Avoid overwriting fresh tokens from other tabs. * Before wiping a bad token from storage on reconnect, make sure that we're wiping the token that we tried and failed to log in with. Avoids logging out another tab that might have gotten a fresh valid token while we were logging in with the old, invalid one (though it is still theoretically possible). * In the logoutOtherClients callback, try to log in with the token that we get in the response. Accounts for the situation where the server disconnects us before the callback runs. * If we fail to log in with a token found during a localStorage poll, make the client logged out. * Add a test that attempts to simulate one tab getting a fresh new token while another tab logs in with an old invalid token on reconnect. --- packages/accounts-base/accounts_client.js | 60 ++++++++++++++++---- packages/accounts-base/localstorage_token.js | 10 +++- packages/accounts-password/password_tests.js | 49 +++++++++++++++- 3 files changed, 102 insertions(+), 17 deletions(-) diff --git a/packages/accounts-base/accounts_client.js b/packages/accounts-base/accounts_client.js index 0e7df5582b..d21bdad805 100644 --- a/packages/accounts-base/accounts_client.js +++ b/packages/accounts-base/accounts_client.js @@ -115,8 +115,28 @@ Accounts.callLoginMethod = function (options) { // need to show a logging-in animation. _suppressLoggingIn: true, userCallback: function (error) { + var storedTokenNow = storedLoginToken(); if (error) { - makeClientLoggedOut(); + // If we had a login error AND the current stored token is the + // one that we tried to log in with, then declare ourselves + // logged out. If there's a token in storage but it's not the + // token that we tried to log in with, we don't know anything + // about whether that token is valid or not, so do nothing. The + // periodic localStorage poll will decide if we are logged in or + // out with this token, if it hasn't already. Of course, even + // with this check, another tab could insert a new valid token + // immediately before we clear localStorage here, which would + // lead to both tabs being logged out, but by checking the token + // in storage right now we hope to make that unlikely to happen. + // + // If there is no token in storage right now, we don't have to + // do anything; whatever code removed the token from storage was + // responsible for calling `makeClientLoggedOut()`, or the + // periodic localStorage poll will call `makeClientLoggedOut` + // eventually if another tab wiped the token from storage. + if (storedTokenNow && storedTokenNow === result.token) { + makeClientLoggedOut(); + } } // Possibly a weird callback to call, but better than nothing if // there is a reconnect between "login result received" and "data @@ -194,23 +214,41 @@ Meteor.logout = function (callback) { }; Meteor.logoutOtherClients = function (callback) { - // Call the `logoutOtherClients` method and store the login token that we get - // back. The server is not supposed to close connections on the old token for - // 10 seconds, so we should have time to store our new token before being - // disconnected. If we get disconnected, then we'll immediately reconnect with - // the new token. If for some reason we get disconnected before storing the - // new token, then the worst that will happen is that we'll have a flicker - // from trying to log in with the old token before storing and logging in with - // the new one. - Meteor.apply('logoutOtherClients', [], + // Call the `logoutOtherClients` method. Store the login token that we get + // back and use it to log in again. The server is not supposed to close + // connections on the old token for 10 seconds, so we should have time to + // store our new token and log in with it before being disconnected. If we get + // disconnected, then we'll immediately reconnect with the new token. If for + // some reason we get disconnected before storing the new token, then the + // worst that will happen is that we'll have a flicker from trying to log in + // with the old token before storing and logging in with the new one. + Meteor.apply('logoutOtherClients', [], { wait: true }, function (error, result) { - if (! error) { + if (error) { + callback && callback(error); + } else { var userId = Meteor.userId(); storeLoginToken(userId, result.token, result.tokenExpires); + // If the server hasn't disconnected us yet by deleting our + // old token, then logging in now with the new valid token + // will prevent us from getting disconnected. If the server + // has already disconnected us due to our old invalid token, + // then we would have already tried and failed to login with + // the old token on reconnect, and we have to make sure a + // login method gets sent here with the new token. + Meteor.loginWithToken(result.token, function (err) { + if (err && + storedLoginToken() && + storedLoginToken().token === result.token) { + makeClientLoggedOut(); + } + callback && callback(err); + }); } }); }; + /// /// LOGIN SERVICES /// diff --git a/packages/accounts-base/localstorage_token.js b/packages/accounts-base/localstorage_token.js index 24c6e25318..e30d4514c7 100644 --- a/packages/accounts-base/localstorage_token.js +++ b/packages/accounts-base/localstorage_token.js @@ -115,10 +115,14 @@ var pollStoredLoginToken = function() { // != instead of !== just to make sure undefined and null are treated the same if (lastLoginTokenWhenPolled != currentLoginToken) { - if (currentLoginToken) - Meteor.loginWithToken(currentLoginToken); // XXX should we pass a callback here? - else + if (currentLoginToken) { + Meteor.loginWithToken(currentLoginToken, function (err) { + if (err) + makeClientLoggedOut(); + }); + } else { Meteor.logout(); + } } lastLoginTokenWhenPolled = currentLoginToken; }; diff --git a/packages/accounts-password/password_tests.js b/packages/accounts-password/password_tests.js index 59ae584d98..73f7ea1db0 100644 --- a/packages/accounts-password/password_tests.js +++ b/packages/accounts-password/password_tests.js @@ -1,5 +1,13 @@ Accounts._noConnectionCloseDelayForTest = true; +if (Meteor.isServer) { + Meteor.methods({ + getUserId: function () { + return this.userId; + } + }); +} + if (Meteor.isClient) (function () { // XXX note, only one test can do login/logout things at once! for @@ -404,10 +412,10 @@ if (Meteor.isClient) (function () { var self = this; // copied from livedata/client_convenience.js - var ddpUrl = '/'; + self.ddpUrl = '/'; if (typeof __meteor_runtime_config__ !== "undefined") { if (__meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL) - ddpUrl = __meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL; + self.ddpUrl = __meteor_runtime_config__.DDP_DEFAULT_CONNECTION_URL; } // XXX can we get the url from the existing connection somehow // instead? @@ -416,7 +424,7 @@ if (Meteor.isClient) (function () { // connection while leaving Meteor.connection logged in. var token; var userId; - self.secondConn = DDP.connect(ddpUrl); + self.secondConn = DDP.connect(self.ddpUrl); var expectLoginError = expect(function (err) { test.isTrue(err); @@ -477,6 +485,41 @@ if (Meteor.isClient) (function () { ); }, logoutStep, + function (test, expect) { + var self = this; + // Test that, when we call logoutOtherClients, if the server disconnects + // us before the logoutOtherClients callback runs, then we still end up + // logged in. + var expectServerLoggedIn = expect(function (err, result) { + test.isFalse(err); + test.isTrue(Meteor.userId()); + test.equal(result, Meteor.userId()); + }); + + Meteor.loginWithPassword( + self.username, + self.password, + expect(function (err) { + test.isFalse(err); + test.isTrue(Meteor.userId()); + + // The test is only useful if things interleave in the following order: + // - logoutOtherClients runs on the server + // - onReconnect fires and sends a login method with the old token, + // which results in an error + // - logoutOtherClients callback runs and stores the new token and + // logs in with it + // In practice they seem to interleave this way, but I'm not sure how + // to make sure that they do. + + Meteor.logoutOtherClients(function (err) { + test.isFalse(err); + Meteor.call("getUserId", expectServerLoggedIn); + }); + }) + ); + }, + logoutStep, function (test, expect) { var self = this; // Test that deleting a user logs out that user's connections. From d9d4f2139e13c2ff6f35b5c910c00cc156b2c172 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Wed, 8 Jan 2014 22:24:01 -0800 Subject: [PATCH 117/124] Add History.md entry for #1616 fix --- History.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/History.md b/History.md index 485db81e2e..23e49d79a2 100644 --- a/History.md +++ b/History.md @@ -11,6 +11,10 @@ with numeric `length` fields as arrays. Among other things, this allows you to use documents with numeric `length` fields with Mongo. #594 #1737 +* Fix races when calling login and/or logoutOtherClients from multiple + tabs. #1616 + + ## v0.7.0.1 * Two fixes to `meteor run` Mongo startup bugs that could lead to hangs with the From 32768a8c057e6871e774aec92351e88cc33e2bd4 Mon Sep 17 00:00:00 2001 From: Michael Bishop Date: Thu, 9 Jan 2014 12:47:14 -0800 Subject: [PATCH 118/124] typo: http => https --- packages/meteor/url_common.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/meteor/url_common.js b/packages/meteor/url_common.js index 9fa045a3d5..c2da364104 100644 --- a/packages/meteor/url_common.js +++ b/packages/meteor/url_common.js @@ -20,7 +20,7 @@ Meteor.absoluteUrl = function (path, options) { if (path) url += path; - // turn http to http if secure option is set, and we're not talking + // turn http to https if secure option is set, and we're not talking // to localhost. if (options.secure && /^http:/.test(url) && // url starts with 'http:' From 0b76997e3dc815fdd02466786c059ebd811cd0e0 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 9 Jan 2014 15:17:37 -0800 Subject: [PATCH 119/124] Be careful not to send a null ADMIN_APP env var --- packages/ctl-helper/ctl-helper.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/ctl-helper/ctl-helper.js b/packages/ctl-helper/ctl-helper.js index 4806ddcc7a..c071cb6487 100644 --- a/packages/ctl-helper/ctl-helper.js +++ b/packages/ctl-helper/ctl-helper.js @@ -45,6 +45,7 @@ _.extend(Ctl, { Ctl.findGalaxy(), 'getAppConfiguration', [Ctl.myAppName()]); if (typeof admin == 'undefined') admin = appConfig.admin; + admin = !!admin; var jobId = null; var rootUrl = Ctl.rootUrl; @@ -65,13 +66,15 @@ _.extend(Ctl, { }); // XXX args? env? + var env = { + ROOT_URL: rootUrl, + METEOR_SETTINGS: appConfig.settings || appConfig.METEOR_SETTINGS + }; + if (admin) + env.ADMIN_APP = 'true'; jobId = Ctl.prettyCall(Ctl.findGalaxy(), 'run', [Ctl.myAppName(), program, { exitPolicy: 'restart', - env: { - ROOT_URL: rootUrl, - METEOR_SETTINGS: appConfig.settings || appConfig.METEOR_SETTINGS, - ADMIN_APP: admin - }, + env: env, ports: { "main": { bindEnv: "PORT", From e445a4af6aaa39aabce12edbe4a2a93fc30e6745 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 9 Jan 2014 16:57:49 -0800 Subject: [PATCH 120/124] Changes to how Meteor.settings is filled on Galaxy (a) Prefer $APP_CONFIG over $METEOR_SETTINGS (b) Allow $APP_CONFIG's settings field to be a string which we parse (which will be the default soon) --- packages/meteor/server_environment.js | 31 +++++++++++++++++++-------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/packages/meteor/server_environment.js b/packages/meteor/server_environment.js index 18cbbaa550..6d6a2ff4e7 100644 --- a/packages/meteor/server_environment.js +++ b/packages/meteor/server_environment.js @@ -4,21 +4,34 @@ Meteor = { }; Meteor.settings = {}; -if (process.env.METEOR_SETTINGS) { + +if (process.env.APP_CONFIG) { + // put settings from the app configuration in the settings. Don't depend on + // the Galaxy package for now, to avoid silly loops. + try { + var appConfig = JSON.parse(process.env.APP_CONFIG); + if (!appConfig.settings) { + Meteor.settings = {}; + } else if (typeof appConfig.settings === "string") { + Meteor.settings = JSON.parse(appConfig.settings); + } else { + // Old versions of Galaxy may store settings in MongoDB as objects. Newer + // versions store it as strings (so that we aren't restricted to + // MongoDB-compatible objects). This line makes it work on older Galaxies. + // XXX delete this eventually + Meteor.settings = appConfig.settings; + } + } catch (e) { + throw new Error("Settings from app config are not valid JSON"); + } +} else if (process.env.METEOR_SETTINGS) { try { Meteor.settings = JSON.parse(process.env.METEOR_SETTINGS); } catch (e) { throw new Error("Settings are not valid JSON"); } -} else if ( process.env.APP_CONFIG) { - // put settings from the app configuration in the settings. Don't depend on - // the Galaxy package for now, to avoid silly loops. - try { - Meteor.settings = JSON.parse(process.env.APP_CONFIG).settings || {}; - } catch (e) { - throw new Error("Settings are not valid JSON"); - } } + // Push a subset of settings to the client. if (Meteor.settings && Meteor.settings.public && typeof __meteor_runtime_config__ === "object") { From 4b34feb0a9569312277565b3875369d2833e8aa2 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 10 Jan 2014 16:44:45 -0800 Subject: [PATCH 121/124] Fix tiny comment typo --- packages/accounts-base/accounts_server.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/accounts-base/accounts_server.js b/packages/accounts-base/accounts_server.js index 59c82eac56..7c5dc3dae7 100644 --- a/packages/accounts-base/accounts_server.js +++ b/packages/accounts-base/accounts_server.js @@ -62,7 +62,7 @@ loginHandlers = []; // { user: { username: }, password: }, or // { user: { email: }, password: }. Accounts.createToken = function (options) { - // Try all of the registered login handlers until one of them doesn' return + // Try all of the registered login handlers until one of them doesn't return // `undefined`, meaning it handled this call to `login`. Return that return // value, which ought to be a {id/token} pair. for (var i = 0; i < loginHandlers.length; ++i) { From bc736e4695bc85ab1a1f01723e33a5ff1685c2c0 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Mon, 13 Jan 2014 11:50:53 -0800 Subject: [PATCH 122/124] Upgrade jquery-waypoints to 2.0.3 --- History.md | 2 + packages/jquery-waypoints/waypoints.js | 1196 +++++++++++------------- 2 files changed, 522 insertions(+), 676 deletions(-) diff --git a/History.md b/History.md index 23e49d79a2..d567729837 100644 --- a/History.md +++ b/History.md @@ -14,6 +14,8 @@ * Fix races when calling login and/or logoutOtherClients from multiple tabs. #1616 +* Upgrade `jquery-waypoints` package from 1.1.7 to 2.0.3. (Contains + backward-incompatible changes). ## v0.7.0.1 diff --git a/packages/jquery-waypoints/waypoints.js b/packages/jquery-waypoints/waypoints.js index 61734a36d9..81da414443 100644 --- a/packages/jquery-waypoints/waypoints.js +++ b/packages/jquery-waypoints/waypoints.js @@ -1,676 +1,520 @@ -/*! -jQuery Waypoints - v1.1.7 -Copyright (c) 2011-2012 Caleb Troughton -Dual licensed under the MIT license and GPL license. -https://github.com/imakewebthings/jquery-waypoints/blob/master/MIT-license.txt -https://github.com/imakewebthings/jquery-waypoints/blob/master/GPL-license.txt -*/ - -/* -Waypoints is a small jQuery plugin that makes it easy to execute a function -whenever you scroll to an element. - -GitHub Repository: https://github.com/imakewebthings/jquery-waypoints -Documentation and Examples: http://imakewebthings.github.com/jquery-waypoints - -Changelog: - v1.1.7 - - Actually fix the post-load bug in Issue #28 from v1.1.3. - v1.1.6 - - Fix potential memory leak by unbinding events on empty context elements. - v1.1.5 - - Make plugin compatible with Browserify/RequireJS. (Thanks @cjroebuck) - v1.1.4 - - Add handler option to give alternate binding method. (Issue #34) - v1.1.3 - - Fix cases where waypoints are added post-load and should be triggered - immediately. (Issue #28) - v1.1.2 - - Fixed error thrown by waypoints with triggerOnce option that were - triggered via resize refresh. - v1.1.1 - - Fixed bug in initialization where all offsets were being calculated - as if set to 0 initially, causing unwarranted triggers during the - subsequent refresh. - - Added onlyOnScroll, an option for individual waypoints that disables - triggers due to an offset refresh that crosses the current scroll - point. (All credit to @knuton on this one.) - v1.1 - - Moved the continuous option out of global settings and into the options - object for individual waypoints. - - Added the context option, which allows for using waypoints within any - scrollable element, not just the window. - v1.0.2 - - Moved scroll and resize handler bindings out of load. Should play nicer - with async loaders like Head JS and LABjs. - - Fixed a 1px off error when using certain % offsets. - - Added unit tests. - v1.0.1 - - Added $.waypoints('viewportHeight'). - - Fixed iOS bug (using the new viewportHeight method). - - Added offset function alias: 'bottom-in-view'. - v1.0 - - Initial release. - -Support: - - jQuery versions 1.4.3+ - - IE6+, FF3+, Chrome 6+, Safari 4+, Opera 11 - - Other versions and browsers may work, these are just the ones I've looked at. -*/ - -(function($, wp, wps, window, undefined){ - '$:nomunge'; - - var $w = $(window), - - // Keeping common strings as variables = better minification - eventName = 'waypoint.reached', - - /* - For the waypoint and direction passed in, trigger the waypoint.reached - event and deal with the triggerOnce option. - */ - triggerWaypoint = function(way, dir) { - way.element.trigger(eventName, dir); - if (way.options.triggerOnce) { - way.element[wp]('destroy'); - } - }, - - /* - Given a jQuery element and Context, returns the index of that element in the waypoints - array. Returns the index, or -1 if the element is not a waypoint. - */ - waypointIndex = function(el, context) { - if (!context) return -1; - var i = context.waypoints.length - 1; - while (i >= 0 && context.waypoints[i].element[0] !== el[0]) { - i -= 1; - } - return i; - }, - - // Private list of all elements used as scrolling contexts for waypoints. - contexts = [], - - /* - Context Class - represents a scrolling context. Properties include: - element: jQuery object containing a single HTML element. - waypoints: Array of waypoints operating under this scroll context. - oldScroll: Keeps the previous scroll position to determine scroll direction. - didScroll: Flag used in scrolling the context's scroll event. - didResize: Flag used in scrolling the context's resize event. - doScroll: Function that checks for crossed waypoints. Called from throttler. - */ - Context = function(context) { - $.extend(this, { - element: $(context), - oldScroll: 0, - - /* - List of all elements that have been registered as waypoints. - Each object in the array contains: - element: jQuery object containing a single HTML element. - offset: The window scroll offset, in px, that triggers the waypoint event. - options: Options object that was passed to the waypoint fn function. - */ - 'waypoints': [], - - didScroll: false, - didResize: false, - - doScroll: $.proxy(function() { - var newScroll = this.element.scrollTop(), - - // Are we scrolling up or down? Used for direction argument in callback. - isDown = newScroll > this.oldScroll, - that = this, - - // Get a list of all waypoints that were crossed since last scroll move. - pointsHit = $.grep(this.waypoints, function(el, i) { - return isDown ? - (el.offset > that.oldScroll && el.offset <= newScroll) : - (el.offset <= that.oldScroll && el.offset > newScroll); - }), - len = pointsHit.length; - - // iOS adjustment - if (!this.oldScroll || !newScroll) { - $[wps]('refresh'); - } - - // Done with scroll comparisons, store new scroll before ejection - this.oldScroll = newScroll; - - // No waypoints crossed? Eject. - if (!len) return; - - // If several waypoints triggered, need to do so in reverse order going up - if (!isDown) pointsHit.reverse(); - - /* - One scroll move may cross several waypoints. If the waypoint's continuous - option is true it should fire even if it isn't the last waypoint. If false, - it will only fire if it's the last one. - */ - $.each(pointsHit, function(i, point) { - if (point.options.continuous || i === len - 1) { - triggerWaypoint(point, [isDown ? 'down' : 'up']); - } - }); - }, this) - }); - - // Setup scroll and resize handlers. Throttled at the settings-defined rate limits. - $(context).bind('scroll.waypoints', $.proxy(function() { - if (!this.didScroll) { - this.didScroll = true; - window.setTimeout($.proxy(function() { - this.doScroll(); - this.didScroll = false; - }, this), $[wps].settings.scrollThrottle); - } - }, this)).bind('resize.waypoints', $.proxy(function() { - if (!this.didResize) { - this.didResize = true; - window.setTimeout($.proxy(function() { - $[wps]('refresh'); - this.didResize = false; - }, this), $[wps].settings.resizeThrottle); - } - }, this)); - - $w.load($.proxy(function() { - /* - Fire a scroll check, should the page be loaded at a non-zero scroll value, - as with a fragment id link or a page refresh. - */ - this.doScroll(); - }, this)); - }, - - /* Returns a Context object from the contexts array, given the raw HTML element - for that context. */ - getContextByElement = function(element) { - var found = null; - - $.each(contexts, function(i, c) { - if (c.element[0] === element) { - found = c; - return false; - } - }); - - return found; - }, - - // Methods exposed to the effin' object - methods = { - /* - jQuery.fn.waypoint([handler], [options]) - - handler - function, optional - A callback function called when the user scrolls past the element. - The function signature is function(event, direction) where event is - a standard jQuery Event Object and direction is a string, either 'down' - or 'up' indicating which direction the user is scrolling. - - options - object, optional - A map of options to apply to this set of waypoints, including where on - the browser window the waypoint is triggered. For a full list of - options and their defaults, see $.fn.waypoint.defaults. - - This is how you register an element as a waypoint. When the user scrolls past - that element it triggers waypoint.reached, a custom event. Since the - parameters for creating a waypoint are optional, we have a few different - possible signatures. Let’s look at each of them. - - someElements.waypoint(); - - Calling .waypoint with no parameters will register the elements as waypoints - using the default options. The elements will fire the waypoint.reached event, - but calling it in this way does not bind any handler to the event. You can - bind to the event yourself, as with any other event, like so: - - someElements.bind('waypoint.reached', function(event, direction) { - // make it rain - }); - - You will usually want to create a waypoint and immediately bind a function to - waypoint.reached, and can do so by passing a handler as the first argument to - .waypoint: - - someElements.waypoint(function(event, direction) { - if (direction === 'down') { - // do this on the way down - } - else { - // do this on the way back up through the waypoint - } - }); - - This will still use the default options, which will trigger the waypoint when - the top of the element hits the top of the window. We can pass .waypoint an - options object to customize things: - - someElements.waypoint(function(event, direction) { - // do something amazing - }, { - offset: '50%' // middle of the page - }); - - You can also pass just an options object. - - someElements.waypoint({ - offset: 100 // 100px from the top - }); - - This behaves like .waypoint(), in that it registers the elements as waypoints - but binds no event handlers. - - Calling .waypoint on an existing waypoint will extend the previous options. - If the call includes a handler, it will be bound to waypoint.reached without - unbinding any other handlers. - */ - init: function(f, options) { - // Register each element as a waypoint, add to array. - this.each(function() { - var cElement = $.fn[wp].defaults.context, - context, - $this = $(this); - - // Default window context or a specific element? - if (options && options.context) { - cElement = options.context; - } - - // Find the closest element that matches the context - if (!$.isWindow(cElement)) { - cElement = $this.closest(cElement)[0]; - } - context = getContextByElement(cElement); - - // Not a context yet? Create and push. - if (!context) { - context = new Context(cElement); - contexts.push(context); - } - - // Extend default and preexisting options - var ndx = waypointIndex($this, context), - base = ndx < 0 ? $.fn[wp].defaults : context.waypoints[ndx].options, - opts = $.extend({}, base, options); - - // Offset aliases - opts.offset = opts.offset === "bottom-in-view" ? - function() { - var cHeight = $.isWindow(cElement) ? $[wps]('viewportHeight') - : $(cElement).height(); - return cHeight - $(this).outerHeight(); - } : opts.offset; - - // Update, or create new waypoint - if (ndx < 0) { - context.waypoints.push({ - 'element': $this, - 'offset': null, - 'options': opts - }); - } - else { - context.waypoints[ndx].options = opts; - } - - // Bind the function if it was passed in. - if (f) { - $this.bind(eventName, f); - } - // Bind the function in the handler option if it exists. - if (options && options.handler) { - $this.bind(eventName, options.handler); - } - }); - - // Need to re-sort+refresh the waypoints array after new elements are added. - $[wps]('refresh'); - - return this; - }, - - - /* - jQuery.fn.waypoint('remove') - - Passing the string 'remove' to .waypoint unregisters the elements as waypoints - and wipes any custom options, but leaves the waypoint.reached events bound. - Calling .waypoint again in the future would reregister the waypoint and the old - handlers would continue to work. - */ - remove: function() { - return this.each(function(i, el) { - var $el = $(el); - - $.each(contexts, function(i, c) { - var ndx = waypointIndex($el, c); - - if (ndx >= 0) { - c.waypoints.splice(ndx, 1); - - if (!c.waypoints.length) { - c.element.unbind('scroll.waypoints resize.waypoints'); - contexts.splice(i, 1); - } - } - }); - }); - }, - - /* - jQuery.fn.waypoint('destroy') - - Passing the string 'destroy' to .waypoint will unbind all waypoint.reached - event handlers on those elements and unregisters them as waypoints. - */ - destroy: function() { - return this.unbind(eventName)[wp]('remove'); - } - }, - - /* - Methods used by the jQuery object extension. - */ - jQMethods = { - - /* - jQuery.waypoints('refresh') - - This will force a recalculation of each waypoint’s trigger point based on - its offset option and context. This is called automatically whenever the window - (or other defined context) is resized, new waypoints are added, or a waypoint’s - options are modified. If your project is changing the DOM or page layout without - doing one of these things, you may want to manually call this refresh. - */ - refresh: function() { - $.each(contexts, function(i, c) { - var isWin = $.isWindow(c.element[0]), - contextOffset = isWin ? 0 : c.element.offset().top, - contextHeight = isWin ? $[wps]('viewportHeight') : c.element.height(), - contextScroll = isWin ? 0 : c.element.scrollTop(); - - $.each(c.waypoints, function(j, o) { - /* $.each isn't safe from element removal due to triggerOnce. - Should rewrite the loop but this is way easier. */ - if (!o) return; - - // Adjustment is just the offset if it's a px value - var adjustment = o.options.offset, - oldOffset = o.offset; - - // Set adjustment to the return value if offset is a function. - if (typeof o.options.offset === "function") { - adjustment = o.options.offset.apply(o.element); - } - // Calculate the adjustment if offset is a percentage. - else if (typeof o.options.offset === "string") { - var amount = parseFloat(o.options.offset); - adjustment = o.options.offset.indexOf("%") ? - Math.ceil(contextHeight * (amount / 100)) : amount; - } - - /* - Set the element offset to the window scroll offset, less - all our adjustments. - */ - o.offset = o.element.offset().top - contextOffset - + contextScroll - adjustment; - - /* - An element offset change across the current scroll point triggers - the event, just as if we scrolled past it unless prevented by an - optional flag. - */ - if (o.options.onlyOnScroll) return; - - if (oldOffset !== null && c.oldScroll > oldOffset && c.oldScroll <= o.offset) { - triggerWaypoint(o, ['up']); - } - else if (oldOffset !== null && c.oldScroll < oldOffset && c.oldScroll >= o.offset) { - triggerWaypoint(o, ['down']); - } - /* For new waypoints added after load, check that down should have - already been triggered */ - else if (!oldOffset && c.element.scrollTop() > o.offset) { - triggerWaypoint(o, ['down']); - } - }); - - // Keep waypoints sorted by offset value. - c.waypoints.sort(function(a, b) { - return a.offset - b.offset; - }); - }); - }, - - - /* - jQuery.waypoints('viewportHeight') - - This will return the height of the viewport, adjusting for inconsistencies - that come with calling $(window).height() in iOS. Recommended for use - within any offset functions. - */ - viewportHeight: function() { - return (window.innerHeight ? window.innerHeight : $w.height()); - }, - - - /* - jQuery.waypoints() - - This will return a jQuery object with a collection of all registered waypoint - elements. - - $('.post').waypoint(); - $('.ad-unit').waypoint(function(event, direction) { - // Passed an ad unit - }); - console.log($.waypoints()); - - The example above would log a jQuery object containing all .post and .ad-unit - elements. - */ - aggregate: function() { - var points = $(); - $.each(contexts, function(i, c) { - $.each(c.waypoints, function(i, e) { - points = points.add(e.element); - }); - }); - return points; - } - }; - - - /* - fn extension. Delegates to appropriate method. - */ - $.fn[wp] = function(method) { - - if (methods[method]) { - return methods[method].apply(this, Array.prototype.slice.call(arguments, 1)); - } - else if (typeof method === "function" || !method) { - return methods.init.apply(this, arguments); - } - else if (typeof method === "object") { - return methods.init.apply(this, [null, method]); - } - else { - $.error( 'Method ' + method + ' does not exist on jQuery ' + wp ); - } - }; - - - /* - The default options object that is extended when calling .waypoint. It has the - following properties: - - context - string | element | jQuery* - default: window - The context defines which scrollable element the waypoint belongs to and acts - within. The default, window, means the waypoint offset is calculated with relation - to the whole viewport. You can set this to another element to use the waypoints - within that element. Accepts a selector string, *but if you use jQuery 1.6+ it - also accepts a raw HTML element or jQuery object. - - continuous - boolean - default: true - If true, and multiple waypoints are triggered in one scroll, this waypoint will - trigger even if it is not the last waypoint reached. If false, it will only - trigger if it is the last waypoint. - - handler - function - default: undefined - An alternative way to bind functions to the waypoint, without using the function - as the first argument to the waypoint function. - - offset - number | string | function - default: 0 - Determines how far the top of the element must be from the top of the browser - window to trigger a waypoint. It can be a number, which is taken as a number - of pixels, a string representing a percentage of the viewport height, or a - function that will return a number of pixels. - - onlyOnScroll - boolean - default: false - If true, this waypoint will not trigger if an offset change during a refresh - causes it to pass the current scroll point. - - triggerOnce - boolean - default: false - If true, the waypoint will be destroyed when triggered. - - An offset of 250 would trigger the waypoint when the top of the element is 250px - from the top of the viewport. Negative values for any offset work as you might - expect. A value of -100 would trigger the waypoint when the element is 100px above - the top of the window. - - offset: '100%' - - A string percentage will determine the pixel offset based on the height of the - window. When resizing the window, this offset will automatically be recalculated - without needing to call $.waypoints('refresh'). - - // The bottom of the element is in view - offset: function() { - return $.waypoints('viewportHeight') - $(this).outerHeight(); - } - - Offset can take a function, which must return a number of pixels from the top of - the window. The this value will always refer to the raw HTML element of the - waypoint. As with % values, functions are recalculated automatically when the - window resizes. For more on recalculating offsets, see $.waypoints('refresh'). - - An offset value of 'bottom-in-view' will act as an alias for the function in the - example above, as this is a common usage. - - offset: 'bottom-in-view' - - You can see this alias in use on the Scroll Analytics example page. - - The triggerOnce flag, if true, will destroy the waypoint after the first trigger. - This is just a shortcut for calling .waypoint('destroy') within the waypoint - handler. This is useful in situations such as scroll analytics, where you only - want to record an event once for each page visit. - - The context option lets you use Waypoints within an element other than the window. - You can define the context with a selector string and the waypoint will act within - the nearest ancestor that matches this selector. - - $('.something-scrollable .waypoint').waypoint({ - context: '.something-scrollable' - }); - - You can see this in action on the Dial Controls example. - - The handler option gives authors an alternative way to bind functions when - creating a waypoint. In place of: - - $('.item').waypoint(function(event, direction) { - // make things happen - }); - - You may instead write: - - $('.item').waypoint({ - handler: function(event, direction) { - // make things happen - } - }); - - */ - $.fn[wp].defaults = { - continuous: true, - offset: 0, - triggerOnce: false, - context: window - }; - - - - - - /* - jQuery object extension. Delegates to appropriate methods above. - */ - $[wps] = function(method) { - if (jQMethods[method]) { - return jQMethods[method].apply(this); - } - else { - return jQMethods['aggregate'](); - } - }; - - - /* - $.waypoints.settings - - Settings object that determines some of the plugin’s behavior. - - resizeThrottle - number - default: 200 - For performance reasons, the refresh performed during resizes is - throttled. This value is the rate-limit in milliseconds between resize - refreshes. For more information on throttling, check out Ben Alman’s - throttle / debounce plugin. - http://benalman.com/projects/jquery-throttle-debounce-plugin/ - - scrollThrottle - number - default: 100 - For performance reasons, checking for any crossed waypoints during a - scroll event is throttled. This value is the rate-limit in milliseconds - between scroll checks. For more information on throttling, check out Ben - Alman’s throttle / debounce plugin. - http://benalman.com/projects/jquery-throttle-debounce-plugin/ - */ - $[wps].settings = { - resizeThrottle: 200, - scrollThrottle: 100 - }; - - $w.load(function() { - // Calculate everything once on load. - $[wps]('refresh'); - }); -})(jQuery, 'waypoint', 'waypoints', window); +// Generated by CoffeeScript 1.6.2 +/* +jQuery Waypoints - v2.0.3 +Copyright (c) 2011-2013 Caleb Troughton +Dual licensed under the MIT license and GPL license. +https://github.com/imakewebthings/jquery-waypoints/blob/master/licenses.txt +*/ + + +(function() { + var __indexOf = [].indexOf || function(item) { for (var i = 0, l = this.length; i < l; i++) { if (i in this && this[i] === item) return i; } return -1; }, + __slice = [].slice; + + (function(root, factory) { + if (typeof define === 'function' && define.amd) { + return define('waypoints', ['jquery'], function($) { + return factory($, root); + }); + } else { + return factory(root.jQuery, root); + } + })(this, function($, window) { + var $w, Context, Waypoint, allWaypoints, contextCounter, contextKey, contexts, isTouch, jQMethods, methods, resizeEvent, scrollEvent, waypointCounter, waypointKey, wp, wps; + + $w = $(window); + isTouch = __indexOf.call(window, 'ontouchstart') >= 0; + allWaypoints = { + horizontal: {}, + vertical: {} + }; + contextCounter = 1; + contexts = {}; + contextKey = 'waypoints-context-id'; + resizeEvent = 'resize.waypoints'; + scrollEvent = 'scroll.waypoints'; + waypointCounter = 1; + waypointKey = 'waypoints-waypoint-ids'; + wp = 'waypoint'; + wps = 'waypoints'; + Context = (function() { + function Context($element) { + var _this = this; + + this.$element = $element; + this.element = $element[0]; + this.didResize = false; + this.didScroll = false; + this.id = 'context' + contextCounter++; + this.oldScroll = { + x: $element.scrollLeft(), + y: $element.scrollTop() + }; + this.waypoints = { + horizontal: {}, + vertical: {} + }; + $element.data(contextKey, this.id); + contexts[this.id] = this; + $element.bind(scrollEvent, function() { + var scrollHandler; + + if (!(_this.didScroll || isTouch)) { + _this.didScroll = true; + scrollHandler = function() { + _this.doScroll(); + return _this.didScroll = false; + }; + return window.setTimeout(scrollHandler, $[wps].settings.scrollThrottle); + } + }); + $element.bind(resizeEvent, function() { + var resizeHandler; + + if (!_this.didResize) { + _this.didResize = true; + resizeHandler = function() { + $[wps]('refresh'); + return _this.didResize = false; + }; + return window.setTimeout(resizeHandler, $[wps].settings.resizeThrottle); + } + }); + } + + Context.prototype.doScroll = function() { + var axes, + _this = this; + + axes = { + horizontal: { + newScroll: this.$element.scrollLeft(), + oldScroll: this.oldScroll.x, + forward: 'right', + backward: 'left' + }, + vertical: { + newScroll: this.$element.scrollTop(), + oldScroll: this.oldScroll.y, + forward: 'down', + backward: 'up' + } + }; + if (isTouch && (!axes.vertical.oldScroll || !axes.vertical.newScroll)) { + $[wps]('refresh'); + } + $.each(axes, function(aKey, axis) { + var direction, isForward, triggered; + + triggered = []; + isForward = axis.newScroll > axis.oldScroll; + direction = isForward ? axis.forward : axis.backward; + $.each(_this.waypoints[aKey], function(wKey, waypoint) { + var _ref, _ref1; + + if ((axis.oldScroll < (_ref = waypoint.offset) && _ref <= axis.newScroll)) { + return triggered.push(waypoint); + } else if ((axis.newScroll < (_ref1 = waypoint.offset) && _ref1 <= axis.oldScroll)) { + return triggered.push(waypoint); + } + }); + triggered.sort(function(a, b) { + return a.offset - b.offset; + }); + if (!isForward) { + triggered.reverse(); + } + return $.each(triggered, function(i, waypoint) { + if (waypoint.options.continuous || i === triggered.length - 1) { + return waypoint.trigger([direction]); + } + }); + }); + return this.oldScroll = { + x: axes.horizontal.newScroll, + y: axes.vertical.newScroll + }; + }; + + Context.prototype.refresh = function() { + var axes, cOffset, isWin, + _this = this; + + isWin = $.isWindow(this.element); + cOffset = this.$element.offset(); + this.doScroll(); + axes = { + horizontal: { + contextOffset: isWin ? 0 : cOffset.left, + contextScroll: isWin ? 0 : this.oldScroll.x, + contextDimension: this.$element.width(), + oldScroll: this.oldScroll.x, + forward: 'right', + backward: 'left', + offsetProp: 'left' + }, + vertical: { + contextOffset: isWin ? 0 : cOffset.top, + contextScroll: isWin ? 0 : this.oldScroll.y, + contextDimension: isWin ? $[wps]('viewportHeight') : this.$element.height(), + oldScroll: this.oldScroll.y, + forward: 'down', + backward: 'up', + offsetProp: 'top' + } + }; + return $.each(axes, function(aKey, axis) { + return $.each(_this.waypoints[aKey], function(i, waypoint) { + var adjustment, elementOffset, oldOffset, _ref, _ref1; + + adjustment = waypoint.options.offset; + oldOffset = waypoint.offset; + elementOffset = $.isWindow(waypoint.element) ? 0 : waypoint.$element.offset()[axis.offsetProp]; + if ($.isFunction(adjustment)) { + adjustment = adjustment.apply(waypoint.element); + } else if (typeof adjustment === 'string') { + adjustment = parseFloat(adjustment); + if (waypoint.options.offset.indexOf('%') > -1) { + adjustment = Math.ceil(axis.contextDimension * adjustment / 100); + } + } + waypoint.offset = elementOffset - axis.contextOffset + axis.contextScroll - adjustment; + if ((waypoint.options.onlyOnScroll && (oldOffset != null)) || !waypoint.enabled) { + return; + } + if (oldOffset !== null && (oldOffset < (_ref = axis.oldScroll) && _ref <= waypoint.offset)) { + return waypoint.trigger([axis.backward]); + } else if (oldOffset !== null && (oldOffset > (_ref1 = axis.oldScroll) && _ref1 >= waypoint.offset)) { + return waypoint.trigger([axis.forward]); + } else if (oldOffset === null && axis.oldScroll >= waypoint.offset) { + return waypoint.trigger([axis.forward]); + } + }); + }); + }; + + Context.prototype.checkEmpty = function() { + if ($.isEmptyObject(this.waypoints.horizontal) && $.isEmptyObject(this.waypoints.vertical)) { + this.$element.unbind([resizeEvent, scrollEvent].join(' ')); + return delete contexts[this.id]; + } + }; + + return Context; + + })(); + Waypoint = (function() { + function Waypoint($element, context, options) { + var idList, _ref; + + options = $.extend({}, $.fn[wp].defaults, options); + if (options.offset === 'bottom-in-view') { + options.offset = function() { + var contextHeight; + + contextHeight = $[wps]('viewportHeight'); + if (!$.isWindow(context.element)) { + contextHeight = context.$element.height(); + } + return contextHeight - $(this).outerHeight(); + }; + } + this.$element = $element; + this.element = $element[0]; + this.axis = options.horizontal ? 'horizontal' : 'vertical'; + this.callback = options.handler; + this.context = context; + this.enabled = options.enabled; + this.id = 'waypoints' + waypointCounter++; + this.offset = null; + this.options = options; + context.waypoints[this.axis][this.id] = this; + allWaypoints[this.axis][this.id] = this; + idList = (_ref = $element.data(waypointKey)) != null ? _ref : []; + idList.push(this.id); + $element.data(waypointKey, idList); + } + + Waypoint.prototype.trigger = function(args) { + if (!this.enabled) { + return; + } + if (this.callback != null) { + this.callback.apply(this.element, args); + } + if (this.options.triggerOnce) { + return this.destroy(); + } + }; + + Waypoint.prototype.disable = function() { + return this.enabled = false; + }; + + Waypoint.prototype.enable = function() { + this.context.refresh(); + return this.enabled = true; + }; + + Waypoint.prototype.destroy = function() { + delete allWaypoints[this.axis][this.id]; + delete this.context.waypoints[this.axis][this.id]; + return this.context.checkEmpty(); + }; + + Waypoint.getWaypointsByElement = function(element) { + var all, ids; + + ids = $(element).data(waypointKey); + if (!ids) { + return []; + } + all = $.extend({}, allWaypoints.horizontal, allWaypoints.vertical); + return $.map(ids, function(id) { + return all[id]; + }); + }; + + return Waypoint; + + })(); + methods = { + init: function(f, options) { + var _ref; + + if (options == null) { + options = {}; + } + if ((_ref = options.handler) == null) { + options.handler = f; + } + this.each(function() { + var $this, context, contextElement, _ref1; + + $this = $(this); + contextElement = (_ref1 = options.context) != null ? _ref1 : $.fn[wp].defaults.context; + if (!$.isWindow(contextElement)) { + contextElement = $this.closest(contextElement); + } + contextElement = $(contextElement); + context = contexts[contextElement.data(contextKey)]; + if (!context) { + context = new Context(contextElement); + } + return new Waypoint($this, context, options); + }); + $[wps]('refresh'); + return this; + }, + disable: function() { + return methods._invoke(this, 'disable'); + }, + enable: function() { + return methods._invoke(this, 'enable'); + }, + destroy: function() { + return methods._invoke(this, 'destroy'); + }, + prev: function(axis, selector) { + return methods._traverse.call(this, axis, selector, function(stack, index, waypoints) { + if (index > 0) { + return stack.push(waypoints[index - 1]); + } + }); + }, + next: function(axis, selector) { + return methods._traverse.call(this, axis, selector, function(stack, index, waypoints) { + if (index < waypoints.length - 1) { + return stack.push(waypoints[index + 1]); + } + }); + }, + _traverse: function(axis, selector, push) { + var stack, waypoints; + + if (axis == null) { + axis = 'vertical'; + } + if (selector == null) { + selector = window; + } + waypoints = jQMethods.aggregate(selector); + stack = []; + this.each(function() { + var index; + + index = $.inArray(this, waypoints[axis]); + return push(stack, index, waypoints[axis]); + }); + return this.pushStack(stack); + }, + _invoke: function($elements, method) { + $elements.each(function() { + var waypoints; + + waypoints = Waypoint.getWaypointsByElement(this); + return $.each(waypoints, function(i, waypoint) { + waypoint[method](); + return true; + }); + }); + return this; + } + }; + $.fn[wp] = function() { + var args, method; + + method = arguments[0], args = 2 <= arguments.length ? __slice.call(arguments, 1) : []; + if (methods[method]) { + return methods[method].apply(this, args); + } else if ($.isFunction(method)) { + return methods.init.apply(this, arguments); + } else if ($.isPlainObject(method)) { + return methods.init.apply(this, [null, method]); + } else if (!method) { + return $.error("jQuery Waypoints needs a callback function or handler option."); + } else { + return $.error("The " + method + " method does not exist in jQuery Waypoints."); + } + }; + $.fn[wp].defaults = { + context: window, + continuous: true, + enabled: true, + horizontal: false, + offset: 0, + triggerOnce: false + }; + jQMethods = { + refresh: function() { + return $.each(contexts, function(i, context) { + return context.refresh(); + }); + }, + viewportHeight: function() { + var _ref; + + return (_ref = window.innerHeight) != null ? _ref : $w.height(); + }, + aggregate: function(contextSelector) { + var collection, waypoints, _ref; + + collection = allWaypoints; + if (contextSelector) { + collection = (_ref = contexts[$(contextSelector).data(contextKey)]) != null ? _ref.waypoints : void 0; + } + if (!collection) { + return []; + } + waypoints = { + horizontal: [], + vertical: [] + }; + $.each(waypoints, function(axis, arr) { + $.each(collection[axis], function(key, waypoint) { + return arr.push(waypoint); + }); + arr.sort(function(a, b) { + return a.offset - b.offset; + }); + waypoints[axis] = $.map(arr, function(waypoint) { + return waypoint.element; + }); + return waypoints[axis] = $.unique(waypoints[axis]); + }); + return waypoints; + }, + above: function(contextSelector) { + if (contextSelector == null) { + contextSelector = window; + } + return jQMethods._filter(contextSelector, 'vertical', function(context, waypoint) { + return waypoint.offset <= context.oldScroll.y; + }); + }, + below: function(contextSelector) { + if (contextSelector == null) { + contextSelector = window; + } + return jQMethods._filter(contextSelector, 'vertical', function(context, waypoint) { + return waypoint.offset > context.oldScroll.y; + }); + }, + left: function(contextSelector) { + if (contextSelector == null) { + contextSelector = window; + } + return jQMethods._filter(contextSelector, 'horizontal', function(context, waypoint) { + return waypoint.offset <= context.oldScroll.x; + }); + }, + right: function(contextSelector) { + if (contextSelector == null) { + contextSelector = window; + } + return jQMethods._filter(contextSelector, 'horizontal', function(context, waypoint) { + return waypoint.offset > context.oldScroll.x; + }); + }, + enable: function() { + return jQMethods._invoke('enable'); + }, + disable: function() { + return jQMethods._invoke('disable'); + }, + destroy: function() { + return jQMethods._invoke('destroy'); + }, + extendFn: function(methodName, f) { + return methods[methodName] = f; + }, + _invoke: function(method) { + var waypoints; + + waypoints = $.extend({}, allWaypoints.vertical, allWaypoints.horizontal); + return $.each(waypoints, function(key, waypoint) { + waypoint[method](); + return true; + }); + }, + _filter: function(selector, axis, test) { + var context, waypoints; + + context = contexts[$(selector).data(contextKey)]; + if (!context) { + return []; + } + waypoints = []; + $.each(context.waypoints[axis], function(i, waypoint) { + if (test(context, waypoint)) { + return waypoints.push(waypoint); + } + }); + waypoints.sort(function(a, b) { + return a.offset - b.offset; + }); + return $.map(waypoints, function(waypoint) { + return waypoint.element; + }); + } + }; + $[wps] = function() { + var args, method; + + method = arguments[0], args = 2 <= arguments.length ? __slice.call(arguments, 1) : []; + if (jQMethods[method]) { + return jQMethods[method].apply(null, args); + } else { + return jQMethods.aggregate.call(null, method); + } + }; + $[wps].settings = { + resizeThrottle: 100, + scrollThrottle: 30 + }; + return $w.load(function() { + return $[wps]('refresh'); + }); + }); + +}).call(this); From 9c1943c46750e3414df319f45b536c2a84f2fa10 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Mon, 13 Jan 2014 11:51:21 -0800 Subject: [PATCH 123/124] Adjust docs to waypoints 2.x and add a delay for deps flush to increase performance on scroll. --- docs/client/docs.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/client/docs.js b/docs/client/docs.js index 4ecb004ab5..c878bc9013 100644 --- a/docs/client/docs.js +++ b/docs/client/docs.js @@ -47,10 +47,17 @@ Meteor.startup(function () { } var ignore_waypoints = false; - $('body').delegate('h1, h2, h3', 'waypoint.reached', function (evt, dir) { + var lastTimeout = null; + $('h1, h2, h3').waypoint(function (evt, dir) { if (!ignore_waypoints) { var active = (dir === "up") ? this.prev : this; - Session.set("section", active.id); + if (active.id) { + if (lastTimeout) + Meteor.clearTimeout(lastTimeout); + lastTimeout = Meteor.setTimeout(function () { + Session.set("section", active.id); + }, 200); + } } }); From 8699a3af94dc497f15f5e3da3f3b4ce5e6fab1af Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 13 Jan 2014 20:26:51 -0800 Subject: [PATCH 124/124] Throw if $MONGO_OPLOG_URL is not a replset `new MongoConnection` can now yield. I can't remember why I thought this would be a problem when first implementing OplogHandle, and it does not seem to be from initial testing. --- packages/mongo-livedata/mongo_driver.js | 2 +- packages/mongo-livedata/oplog_tailing.js | 54 ++++++++++++------------ 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/packages/mongo-livedata/mongo_driver.js b/packages/mongo-livedata/mongo_driver.js index 2036f5cd16..a22d365244 100644 --- a/packages/mongo-livedata/mongo_driver.js +++ b/packages/mongo-livedata/mongo_driver.js @@ -165,7 +165,7 @@ MongoConnection = function (url, options) { self._withDb(function (db) { dbNameFuture.return(db.databaseName); }); - self._oplogHandle = new OplogHandle(options.oplogUrl, dbNameFuture); + self._oplogHandle = new OplogHandle(options.oplogUrl, dbNameFuture.wait()); } }; diff --git a/packages/mongo-livedata/oplog_tailing.js b/packages/mongo-livedata/oplog_tailing.js index 1a123b0035..a04fb0eee9 100644 --- a/packages/mongo-livedata/oplog_tailing.js +++ b/packages/mongo-livedata/oplog_tailing.js @@ -1,6 +1,7 @@ var Future = Npm.require('fibers/future'); var OPLOG_COLLECTION = 'oplog.rs'; +var REPLSET_COLLECTION = 'system.replset'; // Like Perl's quotemeta: quotes all regexp metacharacters. See // https://github.com/substack/quotemeta/blob/master/index.js @@ -27,10 +28,10 @@ idForOp = function (op) { throw Error("Unknown op: " + EJSON.stringify(op)); }; -OplogHandle = function (oplogUrl, dbNameFuture) { +OplogHandle = function (oplogUrl, dbName) { var self = this; self._oplogUrl = oplogUrl; - self._dbNameFuture = dbNameFuture; + self._dbName = dbName; self._oplogLastEntryConnection = null; self._oplogTailConnection = null; @@ -41,26 +42,17 @@ OplogHandle = function (oplogUrl, dbNameFuture) { factPackage: "mongo-livedata", factName: "oplog-watchers" }); self._lastProcessedTS = null; - // Lazily calculate the basic selector. Don't call _baseOplogSelector() at the - // top level of the constructor, because we don't want the constructor to - // block. Note that the _.once is per-handle. - self._baseOplogSelector = _.once(function () { - return { - ns: new RegExp('^' + quotemeta(self._dbNameFuture.wait()) + '\\.'), - $or: [ - { op: {$in: ['i', 'u', 'd']} }, - // If it is not db.collection.drop(), ignore it - { op: 'c', 'o.drop': { $exists: true } }] - }; - }); + self._baseOplogSelector = { + ns: new RegExp('^' + quotemeta(self._dbName) + '\\.'), + $or: [ + { op: {$in: ['i', 'u', 'd']} }, + // If it is not db.collection.drop(), ignore it + { op: 'c', 'o.drop': { $exists: true } }] + }; // XXX doc self._catchingUpFutures = []; - // Setting up the connections and tail handler is a blocking operation, so we - // do it "later". - Meteor.defer(function () { - self._startTailing(); - }); + self._startTailing(); }; _.extend(OplogHandle.prototype, { @@ -117,7 +109,7 @@ _.extend(OplogHandle.prototype, { // tailing selector (ie, we need to specify the DB name) or else we might // find a TS that won't show up in the actual tail stream. var lastEntry = self._oplogLastEntryConnection.findOne( - OPLOG_COLLECTION, self._baseOplogSelector(), + OPLOG_COLLECTION, self._baseOplogSelector, {fields: {ts: 1}, sort: {$natural: -1}}); if (!lastEntry) { @@ -168,13 +160,19 @@ _.extend(OplogHandle.prototype, { self._oplogLastEntryConnection = new MongoConnection( self._oplogUrl, {poolSize: 1}); - // Find the last oplog entry. Blocks until the connection is ready. + // First, make sure that there actually is a repl set here. If not, oplog + // tailing won't ever find anything! (Blocks until the connection is ready.) + var replSetInfo = self._oplogLastEntryConnection.findOne( + REPLSET_COLLECTION, {}); + if (!replSetInfo) + throw Error("$MONGO_OPLOG_URL must be set to the 'local' database of " + + "a Mongo replica set"); + + // Find the last oplog entry. var lastOplogEntry = self._oplogLastEntryConnection.findOne( OPLOG_COLLECTION, {}, {sort: {$natural: -1}}); - var dbName = self._dbNameFuture.wait(); - - var oplogSelector = _.clone(self._baseOplogSelector()); + var oplogSelector = _.clone(self._baseOplogSelector); if (lastOplogEntry) { // Start after the last entry that currently exists. oplogSelector.ts = {$gt: lastOplogEntry.ts}; @@ -189,11 +187,13 @@ _.extend(OplogHandle.prototype, { self._tailHandle = self._oplogTailConnection.tail( cursorDescription, function (doc) { - if (!(doc.ns && doc.ns.length > dbName.length + 1 && - doc.ns.substr(0, dbName.length + 1) === (dbName + '.'))) + if (!(doc.ns && doc.ns.length > self._dbName.length + 1 && + doc.ns.substr(0, self._dbName.length + 1) === + (self._dbName + '.'))) { throw new Error("Unexpected ns"); + } - var trigger = {collection: doc.ns.substr(dbName.length + 1), + var trigger = {collection: doc.ns.substr(self._dbName.length + 1), dropCollection: false, op: doc};