From 897cc78d8454996cb743efb1829263159cbdcb05 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 23 Jun 2014 16:35:55 -0700 Subject: [PATCH 001/126] Update docs for bcrypt dependency --- docs/client/concepts.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 0ed41a18b3..dcf141beb3 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -758,8 +758,9 @@ the native packages included in the bundle. To do that, make sure you have `npm` available, and run the following: $ cd bundle/programs/server/node_modules - $ rm -r fibers + $ rm -r fibers bcrypt $ npm install fibers@1.0.1 + $ npm install bcrypt@0.7.7 {{/warning}} {{/markdown}} From 72c3646cfe9213d20daa533581527ab0c92db77c Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 23 Jun 2014 16:38:35 -0700 Subject: [PATCH 002/126] Make docs `npm install` one line --- docs/client/concepts.html | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index dcf141beb3..a41d1694f3 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -759,8 +759,7 @@ have `npm` available, and run the following: $ cd bundle/programs/server/node_modules $ rm -r fibers bcrypt - $ npm install fibers@1.0.1 - $ npm install bcrypt@0.7.7 + $ npm install fibers@1.0.1 bcrypt@0.7.7 {{/warning}} {{/markdown}} From c5292257533fd899dde0d4f3849864d23090c26e Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Tue, 24 Jun 2014 08:33:02 -0700 Subject: [PATCH 003/126] Remove sentence about SRP from docs --- docs/client/api.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client/api.js b/docs/client/api.js index e1d5d24746..b389f4fad9 100644 --- a/docs/client/api.js +++ b/docs/client/api.js @@ -1087,7 +1087,7 @@ Template.api.loginWithPassword = { { name: "password", type: "String", - descr: "The user's password. This is __not__ sent in plain text over the wire — it is secured with [SRP](http://en.wikipedia.org/wiki/Secure_Remote_Password_protocol)." + descr: "The user's password." }, { name: "callback", From 7106caef4910749e048594f6a447013bff5314bc Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 7 Jul 2014 11:44:10 -0700 Subject: [PATCH 004/126] Remove more SRP from docs --- docs/client/api.html | 10 ++++------ docs/client/concepts.html | 3 +-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index adc97f191e..83580b0103 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -1886,12 +1886,10 @@ authentication. In addition to the basic username and password-based sign-in process, it also supports email-based sign-in including address verification and password recovery emails. -Unlike most web applications, the Meteor client does not send the user's -password directly to the server. It uses the [Secure Remote Password -protocol](http://en.wikipedia.org/wiki/Secure_Remote_Password_protocol) -to ensure the server never sees the user's plain-text password. This -helps protect against embarrassing password leaks if the server's -database is compromised. +The Meteor server stores passwords using the +[bcrypt](http://en.wikipedia.org/wiki/Bcrypt) algorithm. This helps +protect against embarrassing password leaks if the server's database is +compromised. To add password support to your application, run `$ meteor add accounts-password`. You can construct your own user interface using the diff --git a/docs/client/concepts.html b/docs/client/concepts.html index a41d1694f3..0dcd570c19 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -311,8 +311,7 @@ releases will include support for other databases. Meteor includes [Meteor Accounts](#accounts_api), a state-of-the-art authentication system. It features secure password login using the -[Secure Remote Password -protocol](http://en.wikipedia.org/wiki/Secure_Remote_Password_protocol), +[bcrypt](http://en.wikipedia.org/wiki/Bcrypt) algorithm, and integration with external services including Facebook, GitHub, Google, Meetup, Twitter, and Weibo. Meteor Accounts defines a [`Meteor.users`](#meteor_users) collection where developers can store From 5b80ea33b94acf1265dac057cf55f168249f94c5 Mon Sep 17 00:00:00 2001 From: Maria Pacana Date: Tue, 8 Jul 2014 17:59:31 -0700 Subject: [PATCH 005/126] Added links to Meteor Manual on docs page. --- docs/client/api.html | 15 ++++++++++++++- docs/client/introduction.html | 5 +++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/client/api.html b/docs/client/api.html index 83580b0103..f9f10bdf74 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -1994,7 +1994,7 @@ Override fields of the object by assigning to them: a `String` for the subject line of a reset password email. - `resetPassword.text`: A `Function` that takes a user object and a url, and returns the body text for a reset password email. - - `resetPassword.html`: An optional `Function` that takes a user object and a + - `resetPassword.html`: An optional `Function` that takes a user object and a url, and returns the body html for a reset password email. - `enrollAccount`: Same as `resetPassword`, but for initial password setup for new accounts. @@ -2558,6 +2558,11 @@ advanced facilities such as `Deps.Dependency` and `onInvalidate` callbacks are intended primarily for package authors implementing new reactive data sources. +To learn more about how Deps works and to explore advanced features of Deps, +visit the Deps chapter in the +Meteor Manual, which describes it in +complete detail. + {{> api_box deps_autorun }} `Deps.autorun` allows you to run a function that depends on reactive data @@ -2633,6 +2638,10 @@ after processing outstanding invalidations. It is illegal to call `flush` from inside a `flush` or from a running computation. +The Meteor Manual +describes the motivation for the flush cycle and the guarantees made by +`Deps.flush` and `Deps.afterFlush`. + {{> api_box deps_nonreactive }} Calls `func` with `Deps.currentComputation` temporarily set to `null` @@ -2820,6 +2829,10 @@ A Dependency's dependent computations are always valid (they have either by the Dependency itself or some other way, it is immediately removed. +See the +Meteor Manual to learn how to create a reactive data source using + Deps.Dependency. + {{> api_box dependency_changed }} {{> api_box dependency_depend }} diff --git a/docs/client/introduction.html b/docs/client/introduction.html index 7783fe4024..6a5220c5a4 100644 --- a/docs/client/introduction.html +++ b/docs/client/introduction.html @@ -130,6 +130,11 @@ with the project!
GitHub
The core code is on GitHub. If you're able to write code or file issues, we'd love to have your help. Please read Contributing to Meteor for how to get started. +
+ +
The Meteor Manual
+
In-depth articles about the core components of Meteor can be found on theMeteor Manual. We've published the first article, which is about Deps, our transparent reactivity framework. More articles (covering topics like Blaze, Unibuild, and DDP) coming soon! +
{{/markdown}} From 121a4adff2b9f315b6f6dae9501d69f27aa09934 Mon Sep 17 00:00:00 2001 From: Maria Pacana Date: Tue, 8 Jul 2014 18:55:21 -0700 Subject: [PATCH 006/126] Revised wording about Manual on docs page. --- docs/client/api.html | 2 +- docs/client/introduction.html | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index f9f10bdf74..76da6c15f1 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -2558,7 +2558,7 @@ advanced facilities such as `Deps.Dependency` and `onInvalidate` callbacks are intended primarily for package authors implementing new reactive data sources. -To learn more about how Deps works and to explore advanced features of Deps, +To learn more about how Deps works and to explore advanced ways to use it, visit the Deps chapter in the Meteor Manual, which describes it in complete detail. diff --git a/docs/client/introduction.html b/docs/client/introduction.html index 6a5220c5a4..6c4d9e2e30 100644 --- a/docs/client/introduction.html +++ b/docs/client/introduction.html @@ -133,7 +133,7 @@ with the project!
The Meteor Manual
-
In-depth articles about the core components of Meteor can be found on theMeteor Manual. We've published the first article, which is about Deps, our transparent reactivity framework. More articles (covering topics like Blaze, Unibuild, and DDP) coming soon! +
In-depth articles about the core components of Meteor can be found on the Meteor Manual. The first article is about Deps, our transparent reactivity framework. More articles (covering topics like Blaze, Unibuild, and DDP) are coming soon!
From 0ba651a79bd35574151b4ff1e4e4bf9b8f20fe02 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Thu, 17 Jul 2014 19:10:54 -0700 Subject: [PATCH 007/126] =?UTF-8?q?Don=E2=80=99t=20create=20Connection=20u?= =?UTF-8?q?ntil=20livedata=20test=20runs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit for “livedata - publisher errors” --- packages/livedata/livedata_tests.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/livedata/livedata_tests.js b/packages/livedata/livedata_tests.js index 07bd42e779..6cf075a703 100644 --- a/packages/livedata/livedata_tests.js +++ b/packages/livedata/livedata_tests.js @@ -526,16 +526,18 @@ if (Meteor.isClient) { ]); testAsyncMulti("livedata - publisher errors", (function () { - // Use a separate connection so that we can safely check to see if - // conn._subscriptions is empty. - var conn = new LivedataTest.Connection('/', - {reloadWithOutstanding: true}); - var collName = Random.id(); - var coll = new Meteor.Collection(collName, {connection: conn}); + var conn, collName, coll; var errorFromRerun; var gotErrorFromStopper = false; return [ function (test, expect) { + // Use a separate connection so that we can safely check to see if + // conn._subscriptions is empty. + conn = new LivedataTest.Connection('/', + {reloadWithOutstanding: true}); + collName = Random.id(); + coll = new Meteor.Collection(collName, {connection: conn}); + var testSubError = function (options) { conn.subscribe("publisherErrors", collName, options, { onReady: expect(), From d3d3e129ed0b68278fabb26e19eaee0afba540c8 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 18 Jul 2014 10:09:50 -0700 Subject: [PATCH 008/126] History pass --- History.md | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/History.md b/History.md index fc4ae4180f..b9a12f2564 100644 --- a/History.md +++ b/History.md @@ -1,8 +1,70 @@ ## v.NEXT + +## v0.8.3 + +#### XXX blaze-refactor + +* Create `` tags as SVG elements when they have `xlink:href` + attributes. (Previously, `` tags inside SVGs were never created as + SVG elements.) #2178 + +* Throw an error in `{{foo bar}}` if `foo` is missing or not a function. + +* Check that arguments to `UI.insert` have the right types. + +* XXX 3c6c8e5 + + +#### Meteor Accounts + +* Fix OAuth popup flow in mobile apps that don't support `window.opener` + (such as iOS Chrome). #2302 + +* Fix regression in 0.8.2 where an exception would be thrown if + `Meteor.loginWithPassword` didn't have a callback. Callbacks to + `Meteor.loginWithPassword` are now optional again. #2255 + +* Fix "Email already exists" error with MongoDB 2.6. #2238 + + +#### mongo-livedata and minimongo + * Fix performance issue where a large batch of oplog updates could block the node event loop for long periods. #2299. +* Fix oplog bug resulting in error message "Buffer inexplicably empty". #2274 + +* Fix regression from 0.8.2 that caused collections to appear empty in + reactive `findOne()` or `fetch` queries that run before a mutator + returns. #2275 + + +#### Miscellaneous + +* Stop including code by default that automatically refreshes the page + if JavaScript and CSS don't load correctly. While this code is useful + in some multi-server deployments, it can cause infinite refresh loops + if there are errors on the page. Add the `reload-safetybelt` package + to your app if you want to include this code. + +* Add `WebAppInternals.addStaticJs()` for adding static JavaScript code + to be served in the app, inline if allowed by `browser-policy`. + +* On the server, `Meteor.startup(c)` now calls `c` immediately if the + server has already started up, matching the client behavior. #2239 + +* Fix `Meteor._inherits` to copy static properties of the parent + function to the child function. + +* Make the `tinytest/run` method return immediately, so that `wait` + method calls from client tests don't block on server tests completing. + +* Add support for source maps for server-side code. + +* Log errors from method invocations on the client if there is no + callback provided. + * Upgraded dependencies: - less: 1.7.1 (from 1.6.1) From c287a6b3ab0907fb690be63bb1ab0002a2157870 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 18 Jul 2014 10:22:49 -0700 Subject: [PATCH 009/126] Add contributors list --- .mailmap | 2 ++ History.md | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.mailmap b/.mailmap index efd15a1605..705934494b 100644 --- a/.mailmap +++ b/.mailmap @@ -27,6 +27,7 @@ GITHUB: codeinthehole GITHUB: dandv GITHUB: davegonzalez GITHUB: ducdigital +GITHUB: duckspeaker GITHUB: emgee3 GITHUB: felixrabe GITHUB: FredericoC @@ -75,3 +76,4 @@ METEOR: sixolet METEOR: Slava METEOR: stubailo METEOR: ekatek +METEOR: mariapacana diff --git a/History.md b/History.md index b9a12f2564..0861c189ec 100644 --- a/History.md +++ b/History.md @@ -68,6 +68,8 @@ * Upgraded dependencies: - less: 1.7.1 (from 1.6.1) +Patches contributed by GitHub users Cangit, cmather, duckspeaker, zol. + ## v0.8.2 From 2c76a9e8868e03930a2abfebf073b70a644beb42 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 18 Jul 2014 16:25:33 -0700 Subject: [PATCH 010/126] Add a sentence about blaze-refactor to history --- History.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index 0861c189ec..00cbe9facb 100644 --- a/History.md +++ b/History.md @@ -3,7 +3,10 @@ ## v0.8.3 -#### XXX blaze-refactor +#### Blaze + +* Refactor Blaze to simplify internals while preserving the public + API. `UI.Component` has been replaced with `Blaze.View.` * Create `` tags as SVG elements when they have `xlink:href` attributes. (Previously, `` tags inside SVGs were never created as From 7ba1d231938e4e981ef5030c08d1cfdbd1a14cb6 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 18 Jul 2014 16:46:16 -0700 Subject: [PATCH 011/126] more History.md --- History.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/History.md b/History.md index 00cbe9facb..170d4843b1 100644 --- a/History.md +++ b/History.md @@ -8,14 +8,16 @@ * Refactor Blaze to simplify internals while preserving the public API. `UI.Component` has been replaced with `Blaze.View.` +* Fix performance issues and memory leaks concerning event handlers. + +* Add `UI.remove` to complement `UI.render`. + * Create `` tags as SVG elements when they have `xlink:href` attributes. (Previously, `` tags inside SVGs were never created as SVG elements.) #2178 * Throw an error in `{{foo bar}}` if `foo` is missing or not a function. -* Check that arguments to `UI.insert` have the right types. - * XXX 3c6c8e5 @@ -57,9 +59,6 @@ * On the server, `Meteor.startup(c)` now calls `c` immediately if the server has already started up, matching the client behavior. #2239 -* Fix `Meteor._inherits` to copy static properties of the parent - function to the child function. - * Make the `tinytest/run` method return immediately, so that `wait` method calls from client tests don't block on server tests completing. From 105c7686280deea8264852fc9bec07d1b9b15e96 Mon Sep 17 00:00:00 2001 From: Slava Kim Date: Fri, 18 Jul 2014 17:10:29 -0700 Subject: [PATCH 012/126] Add a note about Blaze supporting different cursors for #each --- History.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/History.md b/History.md index 170d4843b1..45f1101052 100644 --- a/History.md +++ b/History.md @@ -18,6 +18,10 @@ * Throw an error in `{{foo bar}}` if `foo` is missing or not a function. +* Cursors returned from template helpers for #each should implement + `observeChanges` method and don't have to be Minimongo cursors + (allows new custom data stores for Blaze like Miniredis) + * XXX 3c6c8e5 From c52ca68ea2e73af261034a36ee91322132a542c1 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Sat, 19 Jul 2014 17:04:29 -0700 Subject: [PATCH 013/126] Add note about #2315 and c05ae2 to History --- History.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/History.md b/History.md index 45f1101052..4cb96ecb86 100644 --- a/History.md +++ b/History.md @@ -9,9 +9,9 @@ API. `UI.Component` has been replaced with `Blaze.View.` * Fix performance issues and memory leaks concerning event handlers. - + * Add `UI.remove` to complement `UI.render`. - + * Create `` tags as SVG elements when they have `xlink:href` attributes. (Previously, `` tags inside SVGs were never created as SVG elements.) #2178 @@ -48,6 +48,10 @@ reactive `findOne()` or `fetch` queries that run before a mutator returns. #2275 +* Throw an exception when `observeChanges` is called from within an + observe callback on the same collection. This is a temporary measure + to address #2315. + #### Miscellaneous From d3eae5a2e8166c0108e536595c20aa7d70ad35bc Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Sun, 20 Jul 2014 21:16:33 -0700 Subject: [PATCH 014/126] Make a trivial change to oauth to force new package version. Linux build of rc0's oauth package seems to have disappeared from s3, so forcing a new oauth version to be published. --- packages/oauth/oauth_server.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/oauth/oauth_server.js b/packages/oauth/oauth_server.js index b4f0c3e5c1..d49444e08a 100644 --- a/packages/oauth/oauth_server.js +++ b/packages/oauth/oauth_server.js @@ -11,7 +11,6 @@ var registeredServices = {}; // Internal: Maps from service version to handler function. The // 'oauth1' and 'oauth2' packages manipulate this directly to register // for callbacks. -// OAuth._requestHandlers = {}; From 78d08b5537235f63cb40e538952568dd3e892ab6 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 21 Jul 2014 10:50:32 -0700 Subject: [PATCH 015/126] Revert part of "Test that reverting df2820 fixed #2275." This reverts commit c05ae240af3870de0b743289d174ebcba3ed82ed EXCEPT for the "fetch in observe" test, which I'm leaving in because we still want a test for #2275. This commit made it an error to start an observeChanges from within an observeChanges callback for the same collection, but it turns out that this is actually a somewhat common thing to do (for example, nested 'each'). Instead, we'll leave things the way they were pre-0.8.2: you can start an observeChanges from within an observeChanges callback, but it'll be subtly buggy in that you won't get synchronous 'added' callbacks. This issue is described in #2315, along with the fact that insert/update/remove/resumeObservers won't run their affected observe callbacks if they are called from within a task on the collection's queue. Eventually we'll implement the full fix (which relaxes the requirement that insert/update/remove run all their callbacks before returning) described in #2315. --- packages/minimongo/minimongo.js | 42 ++++++++++++--------------- packages/minimongo/minimongo_tests.js | 25 ---------------- 2 files changed, 18 insertions(+), 49 deletions(-) diff --git a/packages/minimongo/minimongo.js b/packages/minimongo/minimongo.js index d9c13dff08..25fc10f8b6 100644 --- a/packages/minimongo/minimongo.js +++ b/packages/minimongo/minimongo.js @@ -339,6 +339,21 @@ _.extend(LocalCollection.Cursor.prototype, { query.movedBefore = wrapCallback(options.movedBefore); } + if (!options._suppress_initial && !self.collection.paused) { + // XXX unify ordered and unordered interface + var each = ordered + ? _.bind(_.each, null, query.results) + : _.bind(query.results.forEach, query.results); + each(function (doc) { + var fields = EJSON.clone(doc); + + delete fields._id; + if (ordered) + query.addedBefore(doc._id, fields, null); + query.added(doc._id, fields); + }); + } + var handle = new LocalCollection.ObserveHandle; _.extend(handle, { collection: self.collection, @@ -358,30 +373,9 @@ _.extend(LocalCollection.Cursor.prototype, { handle.stop(); }); } - - if (!options._suppress_initial && !self.collection.paused) { - // XXX unify ordered and unordered interface - var each = ordered - ? _.bind(_.each, null, query.results) - : _.bind(query.results.forEach, query.results); - each(function (doc) { - var fields = EJSON.clone(doc); - - delete fields._id; - if (ordered) - query.addedBefore(doc._id, fields, null); - query.added(doc._id, fields); - }); - - // run the observe callbacks resulting from the initial contents - // before we leave the observe. - if (self.collection._observeQueue.safeToRunTask()) { - self.collection._observeQueue.drain(); - } else if (options.added || options.addedBefore) { - // See #2315. - throw Error("observeChanges called from an observe callback on the same collection cannot differentiate between initial and later adds"); - } - } + // run the observe callbacks resulting from the initial contents + // before we leave the observe. + self.collection._observeQueue.drain(); return handle; } diff --git a/packages/minimongo/minimongo_tests.js b/packages/minimongo/minimongo_tests.js index 90b43cb094..c7eaba4acb 100644 --- a/packages/minimongo/minimongo_tests.js +++ b/packages/minimongo/minimongo_tests.js @@ -3109,28 +3109,3 @@ Tinytest.add("minimongo - fetch in observe", function (test) { observe.stop(); computation.stop(); }); - -Tinytest.add("minimongo - observe in observe", function (test) { - var coll = new LocalCollection; - coll.insert({foo: 2}); - - var observe1AddedCalled = false; - var observe1 = coll.find({foo: 1}).observeChanges({ - added: function (id, fields) { - observe1AddedCalled = true; - test.equal(fields, {foo: 1}); - - // It would be even better if this didn't throw; see #2315. - test.throws(function () { - coll.find({foo: 2}).observeChanges({ - added: function () { - } - }); - }); - } - }); - test.isFalse(observe1AddedCalled); - coll.insert({foo: 1}); - test.isTrue(observe1AddedCalled); - observe1.stop(); -}); From 9c37a006539c29d757794b3305c20aedd03103a2 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 21 Jul 2014 11:07:13 -0700 Subject: [PATCH 016/126] Remove history note for c05ae240a, which has been reverted --- History.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/History.md b/History.md index 4cb96ecb86..cb80c76339 100644 --- a/History.md +++ b/History.md @@ -48,10 +48,6 @@ reactive `findOne()` or `fetch` queries that run before a mutator returns. #2275 -* Throw an exception when `observeChanges` is called from within an - observe callback on the same collection. This is a temporary measure - to address #2315. - #### Miscellaneous From be73e042a1bad00cd5218a1910223b772013148f Mon Sep 17 00:00:00 2001 From: Ryan Yeske Date: Thu, 15 May 2014 12:08:31 -0700 Subject: [PATCH 017/126] recognize forceApprovalPrompt option in Accounts.ui.config this option was originally added via #1226 --- docs/client/api.js | 5 +++++ packages/accounts-ui-unstyled/accounts_ui.js | 20 +++++++++++++++++-- .../accounts-ui-unstyled/accounts_ui_tests.js | 6 +++++- .../login_buttons_single.js | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/client/api.js b/docs/client/api.js index a4c9a3a828..d18cdf2f76 100644 --- a/docs/client/api.js +++ b/docs/client/api.js @@ -1186,6 +1186,11 @@ Template.api.accounts_ui_config = { type: "Object", descr: "To ask the user for permission to act on their behalf when offline, map the relevant external service to `true`. Currently only supported with Google. See [Meteor.loginWithExternalService](#meteor_loginwithexternalservice) for more details." }, + { + name: "forceApprovalPrompt", + type: "Boolean", + descr: "If true, forces the user to approve the app's permissions, even if previously approved. Currently only supported with Google." + }, { name: "passwordSignupFields", type: "String", diff --git a/packages/accounts-ui-unstyled/accounts_ui.js b/packages/accounts-ui-unstyled/accounts_ui.js index 5f241ab997..6a0f0851c2 100644 --- a/packages/accounts-ui-unstyled/accounts_ui.js +++ b/packages/accounts-ui-unstyled/accounts_ui.js @@ -2,12 +2,14 @@ Accounts.ui = {}; Accounts.ui._options = { requestPermissions: {}, - requestOfflineToken: {} + requestOfflineToken: {}, + forceApprovalPrompt: {} }; +// XXX refactor duplicated code in this function Accounts.ui.config = function(options) { // validate options keys - var VALID_KEYS = ['passwordSignupFields', 'requestPermissions', 'requestOfflineToken']; + var VALID_KEYS = ['passwordSignupFields', 'requestPermissions', 'requestOfflineToken', 'forceApprovalPrompt']; _.each(_.keys(options), function (key) { if (!_.contains(VALID_KEYS, key)) throw new Error("Accounts.ui.config: Invalid key: " + key); @@ -56,6 +58,20 @@ Accounts.ui.config = function(options) { } }); } + + // deal with `forceApprovalPrompt` + if (options.forceApprovalPrompt) { + _.each(options.forceApprovalPrompt, function (value, service) { + if (service !== 'google') + throw new Error("Accounts.ui.config: `forceApprovalPrompt` only supported for Google login at the moment."); + + if (Accounts.ui._options.forceApprovalPrompt[service]) { + throw new Error("Accounts.ui.config: Can't set `forceApprovalPrompt` more than once for " + service); + } else { + Accounts.ui._options.forceApprovalPrompt[service] = value; + } + }); + } }; passwordSignupFields = function () { diff --git a/packages/accounts-ui-unstyled/accounts_ui_tests.js b/packages/accounts-ui-unstyled/accounts_ui_tests.js index a8f4fc40f5..5e4bef9ea0 100644 --- a/packages/accounts-ui-unstyled/accounts_ui_tests.js +++ b/packages/accounts-ui-unstyled/accounts_ui_tests.js @@ -5,7 +5,7 @@ // XXX it'd be cool to also test that the right thing happens if options -// *are* validated, but Accouns.ui._options is global state which makes this hard +// *are* validated, but Accounts.ui._options is global state which makes this hard // (impossible?) Tinytest.add('accounts-ui - config validates keys', function (test) { test.throws(function () { @@ -19,4 +19,8 @@ Tinytest.add('accounts-ui - config validates keys', function (test) { test.throws(function () { Accounts.ui.config({requestPermissions: {facebook: "not an array"}}); }); + + test.throws(function () { + Accounts.ui.config({forceApprovalPrompt: {facebook: "only google"}}); + }); }); diff --git a/packages/accounts-ui-unstyled/login_buttons_single.js b/packages/accounts-ui-unstyled/login_buttons_single.js index 79d35a4190..96e1e65624 100644 --- a/packages/accounts-ui-unstyled/login_buttons_single.js +++ b/packages/accounts-ui-unstyled/login_buttons_single.js @@ -29,6 +29,8 @@ Template._loginButtonsLoggedOutSingleLoginButton.events({ options.requestPermissions = Accounts.ui._options.requestPermissions[serviceName]; if (Accounts.ui._options.requestOfflineToken[serviceName]) options.requestOfflineToken = Accounts.ui._options.requestOfflineToken[serviceName]; + if (Accounts.ui._options.forceApprovalPrompt[serviceName]) + options.forceApprovalPrompt = Accounts.ui._options.forceApprovalPrompt[serviceName]; loginWithService(options, callback); } From 265e4e6814daa4e4747a26842edf550024139d94 Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 12 Jun 2014 00:21:51 -0700 Subject: [PATCH 018/126] Assure that transform is not changing cached object. --- packages/minimongo/observe.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/minimongo/observe.js b/packages/minimongo/observe.js index 38bcb4514c..cc97ca73d0 100644 --- a/packages/minimongo/observe.js +++ b/packages/minimongo/observe.js @@ -157,6 +157,7 @@ LocalCollection._observeFromObserveChanges = function (cursor, observeCallbacks) var self = this; if (observeCallbacks.changed) { var oldDoc = self.docs.get(id); + oldDoc = EJSON.clone(oldDoc); var doc = EJSON.clone(oldDoc); LocalCollection._applyChanges(doc, fields); observeCallbacks.changed(transform(doc), transform(oldDoc)); From 6b26cb1e8ea2220a67a33bbe9761a5bd355e4a68 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 21 Jul 2014 18:41:27 -0700 Subject: [PATCH 019/126] Make it more clear that the clone is for transform --- packages/minimongo/observe.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/minimongo/observe.js b/packages/minimongo/observe.js index cc97ca73d0..95b92d3d51 100644 --- a/packages/minimongo/observe.js +++ b/packages/minimongo/observe.js @@ -157,10 +157,10 @@ LocalCollection._observeFromObserveChanges = function (cursor, observeCallbacks) var self = this; if (observeCallbacks.changed) { var oldDoc = self.docs.get(id); - oldDoc = EJSON.clone(oldDoc); var doc = EJSON.clone(oldDoc); LocalCollection._applyChanges(doc, fields); - observeCallbacks.changed(transform(doc), transform(oldDoc)); + observeCallbacks.changed(transform(doc), + transform(EJSON.clone(oldDoc))); } }, removed: function (id) { From 3c0f3bac1472cc2048f3bf13fd97b2a1ff432f28 Mon Sep 17 00:00:00 2001 From: Tom Wang Date: Tue, 22 Jul 2014 18:37:01 +0800 Subject: [PATCH 020/126] fix typo --- docs/client/concepts.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 2884d102fc..4466066fa4 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -33,7 +33,7 @@ packages that most any app will use (for example `webapp`, which handles incoming HTTP connections, and `templating`, which lets you make HTML templates that automatically update live as data changes). Then there are optional packages like `email`, which lets your app -send emails, or the Meteor Accounts series (`account-password`, +send emails, or the Meteor Accounts series (`accounts-password`, `accounts-facebook`, `accounts-ui`, and others) which provide a full-featured user account system that you can drop right into your app. And beyond these "official" packages, there are hundreds of From f0fd348afeb771deb5d8801282884bab47499941 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Thu, 24 Jul 2014 18:30:43 -0700 Subject: [PATCH 021/126] =?UTF-8?q?Don=E2=80=99t=20use=20empty=20text=20no?= =?UTF-8?q?de=20placeholders=20in=20IE=208?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s a long story. See comment. --- packages/blaze/domrange.js | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/blaze/domrange.js b/packages/blaze/domrange.js index 055e71ec52..ba71fc20d3 100644 --- a/packages/blaze/domrange.js +++ b/packages/blaze/domrange.js @@ -29,6 +29,28 @@ Blaze.DOMRange = function (nodeAndRangeArray) { }; var DOMRange = Blaze.DOMRange; +// In IE 8, don't use empty text nodes as placeholders +// in empty DOMRanges, use comment nodes instead. Using +// empty text nodes in modern browsers is great because +// it doesn't clutter the web inspector. In IE 8, however, +// it seems to lead in some roundabout way to the OAuth +// pop-up crashing the browser completely. In the past, +// we didn't use empty text nodes on IE 8 because they +// don't accept JS properties, so just use the same logic +// even though we don't need to set properties on the +// placeholder anymore. +DOMRange._USE_COMMENT_PLACEHOLDERS = (function () { + var result = false; + var textNode = document.createTextNode(""); + try { + textNode.someProp = true; + } catch (e) { + // IE 8 + result = true; + } + return result; +})(); + // static methods DOMRange._insert = function (rangeOrNode, parentElement, nextNode, _isMove) { var m = rangeOrNode; @@ -118,7 +140,10 @@ DOMRange.prototype.attach = function (parentElement, nextNode, _isMove) { DOMRange._insert(members[i], parentElement, nextNode, _isMove); } } else { - var placeholder = document.createTextNode(""); + var placeholder = ( + DOMRange._USE_COMMENT_PLACEHOLDERS ? + document.createComment("") : + document.createTextNode("")); this.emptyRangePlaceholder = placeholder; parentElement.insertBefore(placeholder, nextNode || null); } From fa28ac2cbc57a77b0fdd65a5e1f7db207974f406 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Thu, 24 Jul 2014 18:34:34 -0700 Subject: [PATCH 022/126] =?UTF-8?q?Add=20template=20instance=20=E2=80=9Cth?= =?UTF-8?q?is.autorun=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Needs docs --- packages/templating/templating.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/templating/templating.js b/packages/templating/templating.js index b871b9c180..e74c1506ac 100644 --- a/packages/templating/templating.js +++ b/packages/templating/templating.js @@ -37,6 +37,9 @@ Template.__updateTemplateInstance = function (view) { data: null, firstNode: null, lastNode: null, + autorun: function (f) { + return view.autorun(f); + }, __view__: view }; } From 631e9aab73554eb57bc8f84c91d50f05ae41bd1c Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 24 Jul 2014 22:21:46 -0700 Subject: [PATCH 023/126] Update canonicalizeHtml for 0fd348a. Makes tests pass in IE8. Maybe 0fd348a should have been used as the placeholder? --- packages/test-helpers/canonicalize_html.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/test-helpers/canonicalize_html.js b/packages/test-helpers/canonicalize_html.js index 28dd2520f0..1d159b85ae 100644 --- a/packages/test-helpers/canonicalize_html.js +++ b/packages/test-helpers/canonicalize_html.js @@ -2,6 +2,7 @@ canonicalizeHtml = function(html) { var h = html; // kill IE-specific comments inserted by DomRange h = h.replace(//g, ''); + h = h.replace(//g, ''); // ignore exact text of comments h = h.replace(//g, ''); // make all tags lowercase From abbf3c78fac5b188789eb07b55540f57e3a3ab4c Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 08:20:33 -0700 Subject: [PATCH 024/126] shrinkwrap update --- packages/mongo-livedata/.npm/package/npm-shrinkwrap.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json index ea17322d4e..622dfc20e9 100644 --- a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json +++ b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json @@ -4,7 +4,7 @@ "version": "1.4.1", "dependencies": { "bson": { - "version": "0.2.7", + "version": "https://github.com/meteor/js-bson/tarball/574c0ee", "dependencies": { "nan": { "version": "0.8.0" From adeb649bf684d5166b4444883d8b08785c0ffd49 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 25 Jul 2014 13:57:47 -0700 Subject: [PATCH 025/126] Fix Blaze.currentView in event handlers with test --- packages/blaze/view.js | 6 ++++- packages/spacebars-tests/template_tests.html | 4 ++++ packages/spacebars-tests/template_tests.js | 23 ++++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/packages/blaze/view.js b/packages/blaze/view.js index 9b4e703e91..c3412d34e3 100644 --- a/packages/blaze/view.js +++ b/packages/blaze/view.js @@ -529,7 +529,11 @@ Blaze._addEventMap = function (view, eventMap, thisInHandler) { function (evt) { if (! range.containsElement(evt.currentTarget)) return null; - return handler.apply(thisInHandler || this, arguments); + var handlerThis = thisInHandler || this; + var handlerArgs = arguments; + return Blaze.withCurrentView(view, function () { + return handler.apply(handlerThis, handlerArgs); + }); }, range, function (r) { return r.parentRange; diff --git a/packages/spacebars-tests/template_tests.html b/packages/spacebars-tests/template_tests.html index bfd11eef8f..d7d715c09d 100644 --- a/packages/spacebars-tests/template_tests.html +++ b/packages/spacebars-tests/template_tests.html @@ -922,3 +922,7 @@ Hi there! + + diff --git a/packages/spacebars-tests/template_tests.js b/packages/spacebars-tests/template_tests.js index dffe1f750d..655f476f44 100644 --- a/packages/spacebars-tests/template_tests.js +++ b/packages/spacebars-tests/template_tests.js @@ -2604,3 +2604,26 @@ _.each([1, 2, 3], function (n) { } ); }); + +Tinytest.add('spacebars-tests - template_tests - current view in event handler', function (test) { + var tmpl = Template.spacebars_test_current_view_in_event; + + var currentView; + var currentData; + + tmpl.events({ + 'click span': function () { + currentView = Blaze.getCurrentView(); + currentData = Blaze.getCurrentData(); + } + }); + + var div = renderToDiv(tmpl, 'blah'); + test.equal(canonicalizeHtml(div.innerHTML), 'blah'); + document.body.appendChild(div); + clickElement(div.querySelector('span')); + $(div).remove(); + + test.isTrue(currentView); + test.equal(currentData, 'blah'); +}); From c715613e483302421982112f3404b94036d81685 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 14:04:11 -0700 Subject: [PATCH 026/126] Add sketchy fallback for flaky `window.close()` in OAuth popup. Using an onerror event handler looks like the only semi-reliable way to be able to close the popup in iOS Chrome, even though it's almost certainly a bug that this works. We'll replace it soon with redirect-based OAuth. --- packages/oauth/end_of_login_response.html | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/oauth/end_of_login_response.html b/packages/oauth/end_of_login_response.html index 81f24b22f0..53ac2fa80d 100644 --- a/packages/oauth/end_of_login_response.html +++ b/packages/oauth/end_of_login_response.html @@ -18,5 +18,12 @@ window.close(); - + +

+ Login completed. + Click here to close this window. +

+ + From 59d38b28bb81606fee52c323ee52f70fcbe17d5e Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 25 Jul 2014 14:12:21 -0700 Subject: [PATCH 027/126] minor cleanup --- tools/commands.js | 2 +- tools/config.js | 16 +++++++++------- tools/selftest.js | 6 +++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/commands.js b/tools/commands.js index 4e42276acb..5cc58ced94 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1409,7 +1409,7 @@ main.registerCommand({ browserstack: options.browserstack }; - return selftest.runTests({ + return selftest.runTests({ onlyChanged: options.changed, offline: offline, includeSlowTests: options.slow, diff --git a/tools/config.js b/tools/config.js index d34b3a9e06..916a01376b 100644 --- a/tools/config.js +++ b/tools/config.js @@ -161,22 +161,24 @@ _.extend(exports, { if (!serverUrl) serverUrl = self.getPackageServerUrl(); var serLen = serverUrl.length; - // Chop off http:// and https://. - serverUrl = serverUrl.replace(/^\https:/, ''); - serverUrl = serverUrl.replace(/^\http:/, ''); - serverUrl = serverUrl.slice(2, serLen); + // Chop off http:// and https:// and trailing slashes. + serverUrl = serverUrl.replace(/^\https:\/\//, ''); + serverUrl = serverUrl.replace(/^\http:\/\//, ''); + serverUrl = serverUrl.replace(/\/+$/, ''); // Chop off meteor.com. - serverUrl = serverUrl.replace(/meteor\.com/, ''); + serverUrl = serverUrl.replace(/\.meteor\.com$/, ''); + + // Replace other weird stuff with X. + serverUrl = serverUrl.replace(/[^a-zA-Z0-9.:-]/g, 'X'); // Should look like 'packages.data.json' in the default case // (test-packages.data.json before 0.9.0). - return serverUrl + "data.json"; + return serverUrl + ".data.json"; }, getPackageStorage: function() { var self = this; - var serverFile = self.getPackageServerUrl() + ".data.json"; return path.join(tropohouse.default.root, "package-metadata", "v1", self.getLocalPackageCacheFilename()); diff --git a/tools/selftest.js b/tools/selftest.js index 6290edd78f..0e55c8604e 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -578,9 +578,9 @@ _.extend(Sandbox.prototype, { // releases containing that tool only (and no packages). // // packageServerUrl indicates which package server we think we are using. Use - // the default, if we do not pass this in -- which means that if you - // initialize a warehouse w/o specifying the packageServerUrl and *then* set a - // new PACKAGE_SERVER_URL environment variable, you will be sad. + // the default, if we do not pass this in; you should pass it in any case that + // you will be specifying $METEOR_PACKAGE_SERVER_URL in the environment of a + // command you are running in this sandbox. _makeWarehouse: function (releases, packageServerUrl) { var self = this; files.mkdir_p(path.join(self.warehouse, 'packages'), 0755); From 0cc2624c5936bedbb5d6dc055cdd87956ae58512 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 25 Jul 2014 14:20:33 -0700 Subject: [PATCH 028/126] Fix #2339 (dynamic attributes on textareas) --- packages/blaze/materializer.js | 20 ++++-- packages/spacebars-tests/template_tests.html | 12 ++++ packages/spacebars-tests/template_tests.js | 69 ++++++++++++++++++++ 3 files changed, 94 insertions(+), 7 deletions(-) diff --git a/packages/blaze/materializer.js b/packages/blaze/materializer.js index 84b95003dd..fc71f11f66 100644 --- a/packages/blaze/materializer.js +++ b/packages/blaze/materializer.js @@ -49,13 +49,19 @@ Blaze.DOMMaterializer.def({ var rawAttrs = tag.attrs; var children = tag.children; - if (tagName === 'textarea' && ! (rawAttrs && ('value' in rawAttrs))) { - // turn TEXTAREA contents into a value attribute. - // Reactivity in the form of nested Views won't work here - // because the Views have already been instantiated. To - // get Views in a textarea they need to be wrapped in a - // function and provided as the "value" attribute by the - // compiler. + if (tagName === 'textarea' && tag.children.length && + ! (rawAttrs && ('value' in rawAttrs))) { + // Provide very limited support for TEXTAREA tags with children + // rather than a "value" attribute. + // Reactivity in the form of Views nested in the tag's children + // won't work. Compilers should compile textarea contents into + // the "value" attribute of the tag, wrapped in a function if there + // is reactivity. + if (typeof rawAttrs === 'function' || + HTML.isArray(rawAttrs)) { + throw new Error("Can't have reactive children of TEXTAREA node; " + + "use the 'value' attribute instead."); + } rawAttrs = _.extend({}, rawAttrs || null); rawAttrs.value = Blaze._expand(children, self.parentView); children = []; diff --git a/packages/spacebars-tests/template_tests.html b/packages/spacebars-tests/template_tests.html index d7d715c09d..4cd7ba58bf 100644 --- a/packages/spacebars-tests/template_tests.html +++ b/packages/spacebars-tests/template_tests.html @@ -926,3 +926,15 @@ Hi there! + + + + + + diff --git a/packages/spacebars-tests/template_tests.js b/packages/spacebars-tests/template_tests.js index 655f476f44..c24c325152 100644 --- a/packages/spacebars-tests/template_tests.js +++ b/packages/spacebars-tests/template_tests.js @@ -2627,3 +2627,72 @@ Tinytest.add('spacebars-tests - template_tests - current view in event handler', test.isTrue(currentView); test.equal(currentData, 'blah'); }); + + +Tinytest.add( + "spacebars-tests - template_tests - textarea attrs", function (test) { + var tmplNoContents = { + tmpl: Template.spacebars_test_textarea_attrs, + hasTextAreaContents: false + }; + var tmplWithContents = { + tmpl: Template.spacebars_test_textarea_attrs_contents, + hasTextAreaContents: true + }; + var tmplWithContentsAndMoreAttrs = { + tmpl: Template.spacebars_test_textarea_attrs_array_contents, + hasTextAreaContents: true + }; + + _.each( + [tmplNoContents, tmplWithContents, + tmplWithContentsAndMoreAttrs], + function (tmplInfo) { + + var id = new ReactiveVar("textarea-" + Random.id()); + var name = new ReactiveVar("one"); + var attrs = new ReactiveVar({ + id: "textarea-" + Random.id() + }); + + var div = renderToDiv(tmplInfo.tmpl, { + attrs: function () { + return attrs.get(); + }, + name: function () { + return name.get(); + } + }); + + // Check that the id and value attribute are as we expect. + // We can't check div.innerHTML because Chrome at least doesn't + // appear to put textarea value attributes in innerHTML. + var textarea = div.querySelector("textarea"); + test.equal(textarea.id, attrs.get().id); + test.equal( + textarea.value, tmplInfo.hasTextAreaContents ? "Hello one" : ""); + // One of the templates has a separate attribute in addition to + // an attributes dictionary. + if (tmplInfo === tmplWithContentsAndMoreAttrs) { + test.equal($(textarea).attr("class"), "bar"); + } + + // Change the id, check that the attribute updates reactively. + attrs.set({ id: "textarea-" + Random.id() }); + Deps.flush(); + test.equal(textarea.id, attrs.get().id); + + // Change the name variable, check that the textarea value + // updates reactively. + name.set("two"); + Deps.flush(); + test.equal( + textarea.value, tmplInfo.hasTextAreaContents ? "Hello two" : ""); + + if (tmplInfo === tmplWithContentsAndMoreAttrs) { + test.equal($(textarea).attr("class"), "bar"); + } + + }); + + }); From 4a231a4c76a55660b63c6f2db77e0872e93b7c17 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 25 Jul 2014 14:39:30 -0700 Subject: [PATCH 029/126] improved error handling around bad versions --- tools/package-source.js | 49 +++++++++++++++++++++++++++++++++-------- 1 file changed, 40 insertions(+), 9 deletions(-) diff --git a/tools/package-source.js b/tools/package-source.js index 45ed06638b..1ddf15adb0 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -1,6 +1,7 @@ var fs = require('fs'); var path = require('path'); var _ = require('underscore'); +var semver = require('semver'); var sourcemap = require('source-map'); var files = require('./files.js'); @@ -33,10 +34,10 @@ var earliestCompatible = function (version) { // This is not the place to check to see if version parses as // semver. That should have been done when we first received it from // the user. - var m = version.match(/^(\d)+\./); - if (! m) + var parsed = semver.parse(version); + if (! parsed) throw new Error("not a valid version: " + version); - return m[1] + ".0.0"; + return parsed.major + ".0.0"; }; // Returns a sort comparator to order files into load order. @@ -405,6 +406,10 @@ _.extend(PackageSource.prototype, { } } + if (!utils.validPackageName(self.name)) { + buildmessage.error("Package name invalid: " + self.name); + return; + } if (! fs.existsSync(self.sourceRoot)) throw new Error("putative package directory " + dir + " doesn't exist?"); @@ -639,7 +644,7 @@ _.extend(PackageSource.prototype, { npmDependencies = null; } - if (! self.version && options.requireVersion) { + if (self.version === null && options.requireVersion) { if (! buildmessage.jobHasMessages()) { // Only write the error if there have been no errors so // far. (Otherwise if there is a parse error we'll always get @@ -658,13 +663,39 @@ _.extend(PackageSource.prototype, { // not like we didn't already have to think about this case. } - if (self.version && ! self.earliestCompatibleVersion) { - self.earliestCompatibleVersion = - earliestCompatible(self.version); + if (self.version !== null && typeof(self.version) !== "string") { + if (!buildmessage.jobHasMessages()) { + buildmessage.error("The package version (specified with " + + "Package.describe) must be a string."); + } + // Recover by pretending there was no version (see above). + self.version = null; } - if (!utils.validPackageName(self.name)) { - buildmessage.error("Package name invalid: " + self.name); + if (self.version !== null) { + var parsedVersion = semver.parse(self.version); + if (!parsedVersion) { + if (!buildmessage.jobHasMessages()) { + buildmessage.error( + "The package version (specified with Package.describe) must be " + + "valid semver (see http://semver.org/)."); + } + // Recover by pretending there was no version (see above). + self.version = null; + } else if (parsedVersion.build.length) { + if (!buildmessage.jobHasMessages()) { + buildmessage.error( + "The package version (specified with Package.describe) may not " + + "contain a plus-separated build ID."); + } + // Recover by pretending there was no version (see above). + self.version = null; + } + } + + if (self.version !== null && ! self.earliestCompatibleVersion) { + self.earliestCompatibleVersion = + earliestCompatible(self.version); } // source files used From c622fb4072ddf144cadb3ca6e8489a169f44d730 Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 25 Jul 2014 10:54:25 -0700 Subject: [PATCH 030/126] label online tests as online --- tools/tests/package-tests.js | 2 +- tools/tests/publish.js | 6 +++--- tools/tests/report-stats.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/tests/package-tests.js b/tools/tests/package-tests.js index dff674090b..1821b99e1d 100644 --- a/tools/tests/package-tests.js +++ b/tools/tests/package-tests.js @@ -183,7 +183,7 @@ selftest.define("change packages", function () { // Add packages through the command line, and make sure that the correct set of // changes is reflected in .meteor/packages, .meteor/versions and list -selftest.define("add packages", function () { +selftest.define("add packages", ["net"], function () { var s = new Sandbox(); var run; diff --git a/tools/tests/publish.js b/tools/tests/publish.js index 61ed486711..ae254b8019 100644 --- a/tools/tests/publish.js +++ b/tools/tests/publish.js @@ -9,7 +9,7 @@ var Sandbox = selftest.Sandbox; var testPackagesServer = "https://test-packages.meteor.com"; process.env.METEOR_PACKAGE_SERVER_URL = testPackagesServer; -selftest.define("publish-and-search", ["slow", "online"], function () { +selftest.define("publish-and-search", ["slow", "net"], function () { var s = new Sandbox; var username = "test"; @@ -72,7 +72,7 @@ selftest.define("publish-and-search", ["slow", "online"], function () { run.match(githubUrl); }); -selftest.define("publish-one-arch", ["slow", "online"], function () { +selftest.define("publish-one-arch", ["slow", "net"], function () { var s = new Sandbox; var username = "test"; @@ -110,7 +110,7 @@ selftest.define("publish-one-arch", ["slow", "online"], function () { }); -selftest.define("list-with-a-new-version", ["slow", "online"], function () { +selftest.define("list-with-a-new-version", ["slow", "net"], function () { var s = new Sandbox; var username = "test"; diff --git a/tools/tests/report-stats.js b/tools/tests/report-stats.js index 8c82aebbca..54cace62e0 100644 --- a/tools/tests/report-stats.js +++ b/tools/tests/report-stats.js @@ -51,7 +51,7 @@ var checkMeta = function (appPackages, sessionId, useFakeRelease) { // NOTE: This test will fail if your machine's time is skewed by more // than 30 minutes. This is because the `fetchAppPackageUsage` method // works by passing an hour time range. -selftest.define("report-stats", ["slow"], function () { +selftest.define("report-stats", ["slow", "net"], function () { _.each( // If we are currently running from a checkout, then we run this // test twice (once in which the sandbox uses the current checkout, From 416b0170d7ca27814d7f02d3851c4ee40922465a Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 25 Jul 2014 15:01:00 -0700 Subject: [PATCH 031/126] Simplify UI.dynamic --- packages/spacebars/dynamic.html | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/spacebars/dynamic.html b/packages/spacebars/dynamic.html index 3e93b50538..054e208f6a 100644 --- a/packages/spacebars/dynamic.html +++ b/packages/spacebars/dynamic.html @@ -17,10 +17,6 @@ render the template. --> the template to render) and a `data` property, which can be falsey. --> From 1ac501784a4435079df8723260a152871f65e2c2 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 15:04:42 -0700 Subject: [PATCH 032/126] Add test for `this.autorun` --- packages/spacebars-tests/template_tests.html | 10 ++++ packages/spacebars-tests/template_tests.js | 50 ++++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/packages/spacebars-tests/template_tests.html b/packages/spacebars-tests/template_tests.html index 4cd7ba58bf..5268fa64cc 100644 --- a/packages/spacebars-tests/template_tests.html +++ b/packages/spacebars-tests/template_tests.html @@ -938,3 +938,13 @@ Hi there! + + + + diff --git a/packages/spacebars-tests/template_tests.js b/packages/spacebars-tests/template_tests.js index c24c325152..6bcc2aaf86 100644 --- a/packages/spacebars-tests/template_tests.js +++ b/packages/spacebars-tests/template_tests.js @@ -2696,3 +2696,53 @@ Tinytest.add( }); }); + +Tinytest.add( + "spacebars-tests - template_tests - this.autorun", + function (test) { + var tmpl = Template.spacebars_test_autorun; + var tmplInner = Template.spacebars_test_autorun_inner; + + // Keep track of the value of `UI._templateInstance()` inside the + // autorun each time it runs. + var autorunTemplateInstances = []; + var actualTemplateInstance; + + var show = new ReactiveVar(true); + var rv = new ReactiveVar("foo"); + + tmplInner.created = function () { + actualTemplateInstance = this; + this.autorun(function () { + rv.get(); + autorunTemplateInstances.push(UI._templateInstance()); + }); + }; + + tmpl.helpers({ + show: function () { + return show.get(); + } + }); + + var div = renderToDiv(tmpl); + test.equal(autorunTemplateInstances.length, 1); + test.equal(autorunTemplateInstances[0], actualTemplateInstance); + + // Make sure the autorun re-runs when `rv` changes, and that it has + // the correct current view. + rv.set("bar"); + Deps.flush(); + test.equal(autorunTemplateInstances.length, 2); + test.equal(autorunTemplateInstances[1], actualTemplateInstance); + + // If the inner template is destroyed, the autorun should be stopped. + show.set(false); + Deps.flush(); + rv.set("baz"); + Deps.flush(); + + test.equal(autorunTemplateInstances.length, 2); + test.equal(rv.numListeners(), 0); + } +); From a58f2c3bb2a974f9a95b63f65e1f202634703c01 Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 25 Jul 2014 15:34:54 -0700 Subject: [PATCH 033/126] mark some mroe tests as net with comments; first pass on making it ... exit, instead of craash in ensureDeps. Next up, maybe it can do something better than process.exit --- tools/project.js | 21 +++++++++++++++------ tools/tests/releases.js | 16 +++++++++++++--- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/tools/project.js b/tools/project.js index 84eb44c7dd..9ee5a31827 100644 --- a/tools/project.js +++ b/tools/project.js @@ -183,11 +183,18 @@ _.extend(Project.prototype, { // Call the constraint solver, using the previous dependencies as the last // solution. It is useful to set ignoreProjectDeps, but not nessessary, // since self.viableDepSource is false. - var newVersions = catalog.complete.resolveConstraints( - self.combinedConstraints, - { previousSolution: self.dependencies }, - { ignoreProjectDeps: true } - ); + try { + var newVersions = catalog.complete.resolveConstraints( + self.combinedConstraints, + { previousSolution: self.dependencies }, + { ignoreProjectDeps: true } + ); + } catch (err) { + process.stdout.write( + "Could not resolve the specified constraints for this project:\n" + + err +"\n"); + process.exit(1); + } // Download packages to disk, and rewrite .meteor/versions if it has // changed. @@ -198,7 +205,9 @@ _.extend(Project.prototype, { }); if (!setV.success) { - throw new Error ("Could not install all the requested packages."); + process.stdout.write( + "Could not install all the requested packages. \n"); + process.exit(1); } // Finally, initialize the package loader. diff --git a/tools/tests/releases.js b/tools/tests/releases.js index e351205553..8c35f16d4a 100644 --- a/tools/tests/releases.js +++ b/tools/tests/releases.js @@ -4,7 +4,13 @@ var files = require('../files.js'); var catalog = require('../catalog.js'); var packageLoader = require("../package-loader.js"); -selftest.define("springboard", ['checkout'], function () { +// XXX: Why is this an internet using test? Because our warehouse is a +// hackhackhack. If we clean up the hackhackhackhack, then this does not need +// the internets. (Or, to be more specific: our warehouse code tries to fetch +// the packages from the internet. If we could fool it into using local packages +// instead, or think that it alreayd has the packages, it would be ok). (This is +// because it calls 'create' from a warehouse, to be specific). +selftest.define("springboard", ['checkout', 'net'], function () { var s = new Sandbox({ warehouse: { v1: { }, @@ -112,8 +118,12 @@ selftest.define("springboard", ['checkout'], function () { }); }); - -selftest.define("writing versions file", ['checkout'], function () { +// XXX: Why is this an internet using test? Because our warehouse is a +// hackhackhack. If we clean up the hackhackhackhack, then this does not need +// the internets. (Or, to be more specific: our warehouse code tries to fetch +// the packages from the internet. If we could fool it into using local packages +// instead, or think that it alreayd has the packages, it would be ok). +selftest.define("writing versions file", ['checkout', 'net'], function () { var s = new Sandbox({ warehouse: { v1: { recommended: true}, From 6ec8f05e4c7abc45ee37417a3d74ca45cc9dc0fc Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 25 Jul 2014 16:06:45 -0700 Subject: [PATCH 034/126] Docs changes for 0.8.3 --- docs/client/api.html | 23 +++++++++++++++++++++++ docs/client/api.js | 27 +++++++++++++++++++++++++++ docs/client/docs.js | 6 ++++-- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index 5e8a7b30f6..876e0b877f 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -2189,6 +2189,12 @@ This property provides access to the data context at the top level of the template. It is updated each time the template is re-rendered. Access is read-only and non-reactive. +{{> api_box template_autorun}} + +You can use `this.autorun` from a [`created`](#template_created) or +[`rendered`](#template_rendered) callback to reactively update the DOM +or the template instance. The Computation is automatically stopped +when the template is destroyed.

Template utilities

@@ -2205,6 +2211,15 @@ any part of the DOM for finer control than just using template inclusions. You can define helpers and event maps on `UI.body` just like on any `Template.myTemplate` object. +Helpers on `UI.body` are only available in the `` tags of your +app. To register a global helper, use +[UI.registerHelper](#ui_registerhelper). + +Event maps on `UI.body` don't apply to elements added to the body via +`UI.insert`, jQuery, or the DOM API, or to the body element itself. +To handle events on the body, window, or document, use jQuery or the +DOM API. + {{> api_box ui_render}} This returns an "rendered template" object, which can be passed to @@ -2238,7 +2253,15 @@ changes. {{> api_box ui_getelementdata}} +{{> api_box ui_dynamic}} +`UI.dynamic` allows you to include a template by name, where the name +may be calculated by a helper and may change reactively. The `data` +argument is optional, and if it is omitted, the current data context +is used. + +For example, if there is a template named "foo", `{{dstache}}> UI.dynamic +template="foo"}}` is equivalent to `{{dstache}}> foo}}`. {{#api_box eventmaps}} diff --git a/docs/client/api.js b/docs/client/api.js index a4c9a3a828..3d469a7c88 100644 --- a/docs/client/api.js +++ b/docs/client/api.js @@ -1845,6 +1845,17 @@ Template.api.template_data = { descr: ["The data context of this instance's latest invocation."] }; +Template.api.template_autorun = { + id: "template_autorun", + name: "this.autorun(runFunc)", + locus: "Client", + descr: ["A version of [Deps.autorun](#deps_autorun) that is stopped when the template is destroyed."], + args: [ + {name: "runFunc", + type: "Function", + descr: "The function to run. It receives one argument: a Deps.Computation object."} + ] +}; Template.api.ui_registerhelper = { id: "ui_registerhelper", @@ -1862,6 +1873,22 @@ Template.api.ui_registerhelper = { }] }; +Template.api.ui_dynamic = { + id: "ui_dynamic", + name: "{{> UI.dynamic template=templateName [data=dataContext]}}", + locus: "Client", + descr: ["Choose a template to include dynamically, by name."], + args: [ + {name: "templateName", + type: "String", + descr: "The name of the template to include." + }, + {name: "dataContext", + type: "Object", + descr: "Optional. The data context in which to include the template." + }] +}; + Template.api.ui_body = { id: "ui_body", name: "UI.body", diff --git a/docs/client/docs.js b/docs/client/docs.js index bb3fc8eec8..f76df144bf 100644 --- a/docs/client/docs.js +++ b/docs/client/docs.js @@ -253,7 +253,8 @@ var toc = [ {instance: "this", name: "find", id: "template_find"}, {instance: "this", name: "firstNode", id: "template_firstNode"}, {instance: "this", name: "lastNode", id: "template_lastNode"}, - {instance: "this", name: "data", id: "template_data"} + {instance: "this", name: "data", id: "template_data"}, + {instance: "this", name: "autorun", id: "template_autorun"} ], "UI", [ "UI.registerHelper", @@ -262,7 +263,8 @@ var toc = [ "UI.renderWithData", "UI.insert", "UI.remove", - "UI.getElementData" + "UI.getElementData", + {name: "{{> UI.dynamic}}", id: "ui_dynamic"} ], {type: "spacer"}, {name: "Event maps", style: "noncode"} From 6bb4b4de7b974215708f05cea26c07ba6687d20f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 25 Jul 2014 16:10:08 -0700 Subject: [PATCH 035/126] Help for 'meteor publish-for-arch' --- tools/help.txt | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/help.txt b/tools/help.txt index 69be2d1998..fd7c7d44e1 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -407,13 +407,24 @@ Options: >>> publish-for-arch -Publishes a bla bla bla. +Builds an already-published package for a new platform -Usage: fry the eggs, then +Usage: meteor publish-for-arch packageName@version -Publishes a +When you publish a package with 'meteor publish' for a package which has +platform-specific components (eg, npm modules with native code), the package +will only be usable on machines of the same architecture that you are currently +on. To make your package's version usable on other architectures, you can use +the publish-for-arch command. (The architectures currently supported by Meteor +are 32-bit Linux, 64-bit Linux, and 64-bit OS X; the 'meteor deploy' servers use +64-bit Linux.) -XXX: WRITE +On a machine of the appropriate architecture, install Meteor and run + $ meteor publish-for-arch packageName@version + +You don't need to have a copy of your package's source to do this: Meteor will +automatically download your package's source and dependencies from the package +server. >>> rebuild From df6e70320a658db1591b871bc433ca0de6f60964 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 25 Jul 2014 16:14:25 -0700 Subject: [PATCH 036/126] Update History.md (template autorun) --- History.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/History.md b/History.md index cb80c76339..eac8d9e9f9 100644 --- a/History.md +++ b/History.md @@ -10,7 +10,9 @@ * Fix performance issues and memory leaks concerning event handlers. -* Add `UI.remove` to complement `UI.render`. +* Add `UI.remove`. + +* Add `this.autorun` to the template instance. * Create `` tags as SVG elements when they have `xlink:href` attributes. (Previously, `` tags inside SVGs were never created as From 0f1e310a10f63c7422c13c42260cc3c35370dfad Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 15:38:07 -0700 Subject: [PATCH 037/126] Test return value and argument of `this.autorun` --- packages/spacebars-tests/package.js | 1 + packages/spacebars-tests/template_tests.js | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/spacebars-tests/package.js b/packages/spacebars-tests/package.js index a0d3be4e4b..442d356067 100644 --- a/packages/spacebars-tests/package.js +++ b/packages/spacebars-tests/package.js @@ -13,6 +13,7 @@ Package.on_test(function (api) { api.use('test-helpers'); api.use('showdown'); api.use('minimongo'); + api.use('deps'); api.use('templating', 'client'); api.add_files([ diff --git a/packages/spacebars-tests/template_tests.js b/packages/spacebars-tests/template_tests.js index 6bcc2aaf86..5c549601c2 100644 --- a/packages/spacebars-tests/template_tests.js +++ b/packages/spacebars-tests/template_tests.js @@ -2707,13 +2707,16 @@ Tinytest.add( // autorun each time it runs. var autorunTemplateInstances = []; var actualTemplateInstance; + var returnedComputation; + var computationArg; var show = new ReactiveVar(true); var rv = new ReactiveVar("foo"); tmplInner.created = function () { actualTemplateInstance = this; - this.autorun(function () { + returnedComputation = this.autorun(function (c) { + computationArg = c; rv.get(); autorunTemplateInstances.push(UI._templateInstance()); }); @@ -2729,6 +2732,11 @@ Tinytest.add( test.equal(autorunTemplateInstances.length, 1); test.equal(autorunTemplateInstances[0], actualTemplateInstance); + // Test that the autorun returned a computation and received a + // computation as an argument. + test.isTrue(returnedComputation instanceof Deps.Computation); + test.equal(returnedComputation, computationArg); + // Make sure the autorun re-runs when `rv` changes, and that it has // the correct current view. rv.set("bar"); From 189116a65b9d15f11e9073ba414a32055b5ebdb9 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 16:14:34 -0700 Subject: [PATCH 038/126] History tweaks --- History.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/History.md b/History.md index eac8d9e9f9..d552d4f6dd 100644 --- a/History.md +++ b/History.md @@ -24,8 +24,8 @@ `observeChanges` method and don't have to be Minimongo cursors (allows new custom data stores for Blaze like Miniredis) -* XXX 3c6c8e5 - +* Remove warnings when {{#each}} iterates over a list of strings, + numbers, or other items that contains duplicates. #1980 #### Meteor Accounts @@ -68,7 +68,8 @@ * Make the `tinytest/run` method return immediately, so that `wait` method calls from client tests don't block on server tests completing. -* Add support for source maps for server-side code. +* Add support for server-side source maps when debugging with + `node-inspector`. * Log errors from method invocations on the client if there is no callback provided. From 1b866b729d969025b71909d39579fd3d7b4a48ec Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Fri, 25 Jul 2014 17:24:32 -0700 Subject: [PATCH 039/126] Try window.opener first in OAuth popup, then localStorage. We've occasionally seen weird configurations of IE where localStorage isn't shared between same-origin windows, so trying window.opener first is safer. --- packages/oauth/end_of_login_response.html | 20 +++++++++++--------- packages/oauth/oauth_client.js | 15 +++++++-------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/oauth/end_of_login_response.html b/packages/oauth/end_of_login_response.html index 53ac2fa80d..f4480867d3 100644 --- a/packages/oauth/end_of_login_response.html +++ b/packages/oauth/end_of_login_response.html @@ -4,15 +4,17 @@ if (##SET_CREDENTIAL_TOKEN##) { var credentialToken = ##TOKEN##; var credentialSecret = ##SECRET##; - try { - localStorage[##LOCAL_STORAGE_PREFIX## + credentialToken] = credentialSecret; - } catch (err) { - // localStorage didn't work; try window.opener. - window.opener && - window.opener.Package.oauth.OAuth._handleCredentialSecret( - credentialToken, credentialSecret); - // If window.opener isn't set, we can't do much else, but at least - // close the popup instead of having it hang around on a blank page. + if (window.opener && window.opener.Package && + window.opener.Package.oauth) { + window.opener.Package.oauth.OAuth._handleCredentialSecret( + credentialToken, credentialSecret); + } else { + try { + localStorage[##LOCAL_STORAGE_PREFIX## + credentialToken] = credentialSecret; + } catch (err) { + // We can't do much else, but at least close the popup instead + // of having it hang around on a blank page. + } } } window.close(); diff --git a/packages/oauth/oauth_client.js b/packages/oauth/oauth_client.js index 8ba5e48aa0..dd05c2e85d 100644 --- a/packages/oauth/oauth_client.js +++ b/packages/oauth/oauth_client.js @@ -92,17 +92,16 @@ OAuth._handleCredentialSecret = function (credentialToken, secret) { // Used by accounts-oauth, which needs both a credentialToken and the // corresponding to credential secret to call the `login` method over DDP. OAuth._retrieveCredentialSecret = function (credentialToken) { - // Check localStorage first, then check the secrets collected by - // OAuth._handleCredentialSecret. This matches what we do in + // First check the secrets collected by OAuth._handleCredentialSecret, + // then check localStorage. This matches what we do in // end_of_login_response.html. - var localStorageKey = OAuth._localStorageTokenPrefix + - credentialToken; - var secret = Meteor._localStorage.getItem(localStorageKey); - - if (secret) { + var secret = credentialSecrets[credentialToken]; + if (! secret) { + var localStorageKey = OAuth._localStorageTokenPrefix + + credentialToken; + secret = Meteor._localStorage.getItem(localStorageKey); Meteor._localStorage.removeItem(localStorageKey); } else { - secret = credentialSecrets[credentialToken]; delete credentialSecrets[credentialToken]; } return secret; From 9e65038d08e4ee89359cbe246f8723e359d0b058 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 28 Jul 2014 10:46:15 -0700 Subject: [PATCH 040/126] Update "old cli test" for new 'meteor remove' output --- tools/tests/old/cli-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/old/cli-test.sh b/tools/tests/old/cli-test.sh index 4a80986e28..3792275fa9 100755 --- a/tools/tests/old/cli-test.sh +++ b/tools/tests/old/cli-test.sh @@ -87,7 +87,7 @@ $METEOR search backbone | grep "backbone" >> $OUTPUT $METEOR add backbone 2>&1 | grep "backbone:" | grep -v "no such package" | >> $OUTPUT $METEOR list | grep "backbone" >> $OUTPUT grep backbone packages >> $OUTPUT # remember, we are already in .meteor -$METEOR remove backbone 2>&1 | grep "Removed constraint backbone" >> $OUTPUT +$METEOR remove backbone 2>&1 | grep "Removed top-level dependency on backbone" >> $OUTPUT ! $METEOR list 2>&1 | grep "backbone" >> $OUTPUT echo "... bundle" From 75c30ef073b864779762aa804b65a29cd52f0070 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 28 Jul 2014 12:23:54 -0700 Subject: [PATCH 041/126] Fix scope of UI.contentBlock argument MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eric Dobbertin’s bug --- packages/blaze/builtins.js | 1 + packages/spacebars-tests/template_tests.html | 12 +++++++++ packages/spacebars-tests/template_tests.js | 10 ++++++++ packages/spacebars/spacebars-runtime.js | 27 +++++++++++++++++++- 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/packages/blaze/builtins.js b/packages/blaze/builtins.js index 524171af48..d65b33b2a2 100644 --- a/packages/blaze/builtins.js +++ b/packages/blaze/builtins.js @@ -169,6 +169,7 @@ Blaze.InOuterTemplateScope = function (templateView, contentFunc) { parentView = parentView.parentView; view.onCreated(function () { + this.originalParentView = this.parentView; this.parentView = parentView; }); return view; diff --git a/packages/spacebars-tests/template_tests.html b/packages/spacebars-tests/template_tests.html index 5268fa64cc..4d32086cb1 100644 --- a/packages/spacebars-tests/template_tests.html +++ b/packages/spacebars-tests/template_tests.html @@ -948,3 +948,15 @@ Hi there! + + + + diff --git a/packages/spacebars-tests/template_tests.js b/packages/spacebars-tests/template_tests.js index 5c549601c2..c748505f84 100644 --- a/packages/spacebars-tests/template_tests.js +++ b/packages/spacebars-tests/template_tests.js @@ -2754,3 +2754,13 @@ Tinytest.add( test.equal(rv.numListeners(), 0); } ); + +// Test that argument in {{> UI.contentBlock arg}} is evaluated in +// the proper data context. +Tinytest.add( + "spacebars-tests - template_tests - contentBlock argument", + function (test) { + var tmpl = Template.spacebars_test_contentBlock_arg; + var div = renderToDiv(tmpl); + test.equal(canonicalizeHtml(div.innerHTML), 'AAA BBB'); + }); diff --git a/packages/spacebars/spacebars-runtime.js b/packages/spacebars/spacebars-runtime.js index 7d02d7f9b9..33ffd83549 100644 --- a/packages/spacebars/spacebars-runtime.js +++ b/packages/spacebars/spacebars-runtime.js @@ -205,7 +205,32 @@ Spacebars.dot = function (value, id1/*, id2, ...*/) { }; Spacebars.TemplateWith = function (argFunc, contentBlock) { - var w = Blaze.With(argFunc, contentBlock); + var w; + + // This is a little messy. When we compile `{{> UI.contentBlock}}`, we + // wrap it in Blaze.InOuterTemplateScope in order to skip the intermediate + // parent Views in the current template. However, when there's an argument + // (`{{> UI.contentBlock arg}}`), the argument needs to be evaluated + // in the original scope. There's no good order to nest + // Blaze.InOuterTemplateScope and Spacebars.TemplateWith to achieve this, + // so we wrap argFunc to run it in the "original parentView" of the + // Blaze.InOuterTemplateScope. + // + // To make this better, reconsider InOuterTemplateScope as a primitive. + // Longer term, evaluate expressions in the proper lexical scope. + var wrappedArgFunc = function () { + var viewToEvaluateArg = null; + if (w.parentView && w.parentView.kind === 'InOuterTemplateScope') { + viewToEvaluateArg = w.parentView.originalParentView; + } + if (viewToEvaluateArg) { + return Blaze.withCurrentView(viewToEvaluateArg, argFunc); + } else { + return argFunc(); + } + }; + + w = Blaze.With(wrappedArgFunc, contentBlock); w.__isTemplateWith = true; return w; }; From 34bfcffa8296be366911c31aad35ad404e4051fb Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 28 Jul 2014 13:28:02 -0700 Subject: [PATCH 042/126] History tweaks --- History.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/History.md b/History.md index d552d4f6dd..ab2a3105e6 100644 --- a/History.md +++ b/History.md @@ -21,21 +21,21 @@ * Throw an error in `{{foo bar}}` if `foo` is missing or not a function. * Cursors returned from template helpers for #each should implement - `observeChanges` method and don't have to be Minimongo cursors - (allows new custom data stores for Blaze like Miniredis) + the `observeChanges` method and don't have to be Minimongo cursors + (allowing new custom data stores for Blaze like Miniredis). * Remove warnings when {{#each}} iterates over a list of strings, numbers, or other items that contains duplicates. #1980 #### Meteor Accounts -* Fix OAuth popup flow in mobile apps that don't support `window.opener` - (such as iOS Chrome). #2302 - * Fix regression in 0.8.2 where an exception would be thrown if `Meteor.loginWithPassword` didn't have a callback. Callbacks to `Meteor.loginWithPassword` are now optional again. #2255 +* Fix OAuth popup flow in mobile apps that don't support + `window.opener`. #2302 + * Fix "Email already exists" error with MongoDB 2.6. #2238 @@ -59,18 +59,18 @@ if there are errors on the page. Add the `reload-safetybelt` package to your app if you want to include this code. -* Add `WebAppInternals.addStaticJs()` for adding static JavaScript code - to be served in the app, inline if allowed by `browser-policy`. - * On the server, `Meteor.startup(c)` now calls `c` immediately if the server has already started up, matching the client behavior. #2239 -* Make the `tinytest/run` method return immediately, so that `wait` - method calls from client tests don't block on server tests completing. - * Add support for server-side source maps when debugging with `node-inspector`. +* Add `WebAppInternals.addStaticJs()` for adding static JavaScript code + to be served in the app, inline if allowed by `browser-policy`. + +* Make the `tinytest/run` method return immediately, so that `wait` + method calls from client tests don't block on server tests completing. + * Log errors from method invocations on the client if there is no callback provided. From 9bb83c4d4418286e700c17793336d490eadd9e41 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 28 Jul 2014 14:20:23 -0700 Subject: [PATCH 043/126] Moar History words --- History.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/History.md b/History.md index ab2a3105e6..0ce44b0e8c 100644 --- a/History.md +++ b/History.md @@ -10,9 +10,10 @@ * Fix performance issues and memory leaks concerning event handlers. -* Add `UI.remove`. +* Add `UI.remove`, which removes a template after `UI.render`/`UI.insert`. -* Add `this.autorun` to the template instance. +* Add `this.autorun` to the template instance, which is like `Deps.autorun` + but is automatically stopped when the template is destroyed. * Create `` tags as SVG elements when they have `xlink:href` attributes. (Previously, `` tags inside SVGs were never created as From 0e74bdf7691a2ce91ce4260772b10e31af7ff46b Mon Sep 17 00:00:00 2001 From: ekatek Date: Mon, 28 Jul 2014 15:07:38 -0700 Subject: [PATCH 044/126] help text for meteor update --- tools/help.txt | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/tools/help.txt b/tools/help.txt index fd7c7d44e1..5d24e7a365 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -69,14 +69,38 @@ Options: >>> update -Upgrade this project to the latest version of Meteor -Usage: meteor update [--release ] +Upgrade this project's dependencies to their latest versions. +Usage: meteor update + meteor update --patch + meteor update --release + meteor update --packages-only + meteor update [packageName packageName2 ...] -Sets the version of Meteor to use with the current project. If a -release is specified with --release, set the project to use that -version. Otherwise download and use the latest release of Meteor. +Updates the meteor release, and then, if applicable, updates the packages +used by the app to the latest versions that don't cause dependency +conflicts with other packages in the app. + +Passing the --patch argument will update to the latest patch, if such exists. +Patch releases contain very minor changes, usually bug fixes. Updating to +the latest patch is always recommended. This will try to not update non-core +packages unless strictly nessessary. + +Passing the --release argument will force update to a specific release of meteor. +This will not update non-core packages unless strictly nessessary. It is also +possible that some packages cannot be updated to be compatible with the new +release. If that happens, the app will not build until dependencies on those +packages are removed. + +Passing --packages-only will try to update non-core packages to their latest +versions. It will not update the version of meteor. To update individual packages +(for example: 'foo:awesome') pass in their names instead, with no options. ('meteor +update foo:awesome'). + +Options: + --packages-only Update the package versions only. Do not update the release. + --patch Update the release to a patch release only. + --release Update to a specific release of meteor. -XXX: change to new world order >>> run-upgrader Execute a specific upgrader by name. Intended for testing. From 8dbda26c32f83a96dcb912f1d96a9a34bd9f827a Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 28 Jul 2014 15:13:10 -0700 Subject: [PATCH 045/126] Banner and notices --- scripts/admin/banner.txt | 8 +++----- scripts/admin/notices.json | 3 +++ 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/admin/banner.txt b/scripts/admin/banner.txt index c72d6b8ddb..6573d75782 100644 --- a/scripts/admin/banner.txt +++ b/scripts/admin/banner.txt @@ -1,7 +1,5 @@ -=> Meteor 0.8.2: Switch `accounts-password` to use bcrypt on the - server. User accounts will seamlessly transition to bcrypt on the - next login, but this transition is one-way, so you cannot downgrade a - production app once you upgrade to 0.8.2. +=> Meteor 0.8.3: Performance improvements and a big refactoring of the + Blaze internals. This release is being downloaded in the background. Update your - project to Meteor 0.8.2 by running 'meteor update'. + project to Meteor 0.8.3 by running 'meteor update'. diff --git a/scripts/admin/notices.json b/scripts/admin/notices.json index 3b002c98b5..4e93ea80c9 100644 --- a/scripts/admin/notices.json +++ b/scripts/admin/notices.json @@ -149,6 +149,9 @@ ] } }, + { + "release": "0.8.3" + }, { "release": "NEXT" } From b7fa7e24526cd91d565c9d20a8eb3da7db97331f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 28 Jul 2014 15:31:58 -0700 Subject: [PATCH 046/126] don't run constraint-solver test from release --- tools/tests/constraint-solver.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/tests/constraint-solver.js b/tools/tests/constraint-solver.js index 4bf228cd58..f2d5c6b0a9 100644 --- a/tools/tests/constraint-solver.js +++ b/tools/tests/constraint-solver.js @@ -3,9 +3,10 @@ var Sandbox = selftest.Sandbox; var files = require('../files.js'); var _= require('underscore'); -// Add packages to an app. Change the contents of the packages and their -// dependencies, make sure that the app still refreshes. -selftest.define('constraint solver benchmark', ['slow'], function () { +// Runs all of the constraint-solver tests, including ones that tie up the CPU +// for too long to safely run in the normal test-packages run. +// Only run from checkouts, because test-packages only works on local packages. +selftest.define('constraint solver benchmark', ['slow', 'checkout'], function () { var s = new Sandbox(); s.set('CONSTRAINT_SOLVER_BENCHMARK', 't'); var run = s.run("test-packages", From 5b69c7c25b1307016442bb8cebac394750a5c86a Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 28 Jul 2014 15:37:30 -0700 Subject: [PATCH 047/126] fix report-stats test from release --- tools/tests/report-stats.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/tests/report-stats.js b/tools/tests/report-stats.js index 54cace62e0..8ec762a05e 100644 --- a/tools/tests/report-stats.js +++ b/tools/tests/report-stats.js @@ -16,8 +16,6 @@ process.env.METEOR_PACKAGE_STATS_SERVER_URL = testStatsServer; var clientAddress; var checkMeta = function (appPackages, sessionId, useFakeRelease) { - var toolsPackage = selftest.getToolsPackage(); - if (! clientAddress) { clientAddress = getClientAddress(); } @@ -36,12 +34,20 @@ var checkMeta = function (appPackages, sessionId, useFakeRelease) { } if (useFakeRelease) { + var toolsPackage = selftest.getToolsPackage(); expectedUserAgentInfo.meteorReleaseTrack = "METEOR-CORE"; expectedUserAgentInfo.meteorReleaseVersion = "v1"; expectedUserAgentInfo.meteorToolsPackageWithVersion = toolsPackage.name + "@" + toolsPackage.version; + } else { + expectedUserAgentInfo.meteorReleaseTrack = + release.current.getReleaseTrack(); + expectedUserAgentInfo.meteorReleaseVersion = + release.current.getReleaseVersion(); + expectedUserAgentInfo.meteorToolsPackageWithVersion = + release.current.getToolsPackageAtVersion(); } selftest.expectEqual(appPackages.meta, expectedUserAgentInfo); From 3aff2199cc815060e21737074926d181a17f6852 Mon Sep 17 00:00:00 2001 From: ekatek Date: Mon, 28 Jul 2014 16:09:09 -0700 Subject: [PATCH 048/126] fixing old cli test --- tools/tests/old/cli-test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/tests/old/cli-test.sh b/tools/tests/old/cli-test.sh index 3792275fa9..19463850f9 100755 --- a/tools/tests/old/cli-test.sh +++ b/tools/tests/old/cli-test.sh @@ -45,7 +45,7 @@ $METEOR --help | grep "List the packages explicitly used" >> $OUTPUT $METEOR run --help | grep "Port to listen" >> $OUTPUT $METEOR test-packages --help | grep "Port to listen" >> $OUTPUT $METEOR create --help | grep "Make a subdirectory" >> $OUTPUT -$METEOR update --help | grep "Sets the version" >> $OUTPUT +$METEOR update --help | grep "Updates the meteor release" >> $OUTPUT $METEOR add --help | grep "Adds packages" >> $OUTPUT $METEOR remove --help | grep "Removes a package" >> $OUTPUT $METEOR list --help | grep "This will not list transitive dependencies" >> $OUTPUT From 452ae36c683f17e6029a7e9b7aca539de3a0751b Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 28 Jul 2014 16:27:49 -0700 Subject: [PATCH 049/126] Change how Blaze.render,toHTML infer parentView MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If you call UI.renderWithData, say, from an event handler, you may be unpleasantly surprised if it gets Blaze.currentView as its parentView (where Blaze.currentView is based on the template where the event handler is defined). On the other hand, showdown/template-integration.js takes advantage of the fact that Blaze.toHTML in render() is infers the parentView from Blaze.currentView. So only infer currentView as parent while in the view’s render(). This change should only affect apps and packages that use Blaze.render, Blaze.toHTML, UI.render, or UI.renderWithData. --- packages/blaze/view.js | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/blaze/view.js b/packages/blaze/view.js index c3412d34e3..7597f97a63 100644 --- a/packages/blaze/view.js +++ b/packages/blaze/view.js @@ -312,14 +312,21 @@ Blaze.HTMLJSExpander.def({ } }); +// Return Blaze.currentView, but only if it is being rendered +// (i.e. we are in its render() method). +var currentViewIfRendering = function () { + var view = Blaze.currentView; + return (view && view.isInRender) ? view : null; +}; + Blaze._expand = function (htmljs, parentView) { - parentView = parentView || Blaze.currentView; + parentView = parentView || currentViewIfRendering(); return (new Blaze.HTMLJSExpander( {parentView: parentView})).visit(htmljs); }; Blaze._expandAttributes = function (attrs, parentView) { - parentView = parentView || Blaze.currentView; + parentView = parentView || currentViewIfRendering(); return (new Blaze.HTMLJSExpander( {parentView: parentView})).visitAttributes(attrs); }; @@ -383,7 +390,7 @@ Blaze.runTemplate = function (t/*, args*/) { }; Blaze.render = function (content, parentView) { - parentView = parentView || Blaze.currentView; + parentView = parentView || currentViewIfRendering(); var view; if (typeof content === 'function') { @@ -401,7 +408,7 @@ Blaze.render = function (content, parentView) { Blaze.toHTML = function (htmljs, parentView) { if (typeof htmljs === 'function') throw new Error("Blaze.toHTML doesn't take a function, just HTMLjs"); - parentView = parentView || Blaze.currentView; + parentView = parentView || currentViewIfRendering(); return HTML.toHTML(Blaze._expand(htmljs, parentView)); }; @@ -414,7 +421,7 @@ Blaze.toText = function (htmljs, parentView, textMode) { textMode = parentView; parentView = null; } - parentView = parentView || Blaze.currentView; + parentView = parentView || currentViewIfRendering(); if (! textMode) throw new Error("textMode required"); From 55fe8a8aae5c53333edeeefeebe02f11c6af41d3 Mon Sep 17 00:00:00 2001 From: Matthew Arbesfeld Date: Fri, 25 Jul 2014 17:16:37 -0700 Subject: [PATCH 050/126] Fix hot code push client-side reloads. We were overwriting the server directory when a client-side file changed, which made all process calls fail, such as process.cwd() and fs.*. We abstracted out some of the builder code so that only the client targets are "rebuilt" when a client side file changes. --- tools/bundler.js | 123 ++++++++++-------- tools/run-app.js | 13 +- .../hot-code-push-test/hot-code-push-test.js | 2 + 3 files changed, 80 insertions(+), 58 deletions(-) diff --git a/tools/bundler.js b/tools/bundler.js index 8bbb9bb180..4433e62aca 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1452,6 +1452,27 @@ var writeFile = function (file, builder) { builder.write(file.targetPath, { data: file.contents() }); }; +// Writes a target a path in 'programs' +var writeTargetToPath = function (name, target, outputPath, options) { + var builder = new Builder({ + outputPath: path.join(outputPath, 'programs', name), + symlink: options.includeNodeModulesSymlink + }); + + var relControlFilePath = + target.write(builder, { + includeNodeModulesSymlink: options.includeNodeModulesSymlink, + getRelativeTargetPath: options.getRelativeTargetPath }); + + builder.complete(); + + return { + name: name, + arch: target.mostCompatibleArch(), + path: path.join('programs', name, relControlFilePath) + }; +}; + /////////////////////////////////////////////////////////////////////////////// // writeSiteArchive /////////////////////////////////////////////////////////////////////////////// @@ -1495,51 +1516,6 @@ var writeSiteArchive = function (targets, outputPath, options) { meteorRelease: options.releaseName }; - // Pick a path in the bundle for each target - var paths = {}; - _.each(targets, function (target, name) { - var p = path.join('programs', name); - builder.reserve(p, { directory: true }); - paths[name] = p; - }); - - // Hack to let servers find relative paths to clients. Should find - // another solution eventually (probably some kind of mount - // directive that mounts the client bundle in the server at runtime) - var getRelativeTargetPath = function (options) { - var pathForTarget = function (target) { - var name; - _.each(targets, function (t, n) { - if (t === target) - name = n; - }); - if (! name) - throw new Error("missing target?"); - - if (! (name in paths)) - throw new Error("missing target path?"); - - return paths[name]; - }; - - return path.relative(pathForTarget(options.relativeTo), - pathForTarget(options.forTarget)); - }; - - // Write out each target - _.each(targets, function (target, name) { - var relControlFilePath = - target.write(builder.enter(paths[name]), { - includeNodeModulesSymlink: options.includeNodeModulesSymlink, - getRelativeTargetPath: getRelativeTargetPath }); - - json.programs.push({ - name: name, - arch: target.mostCompatibleArch(), - path: path.join(paths[name], relControlFilePath) - }); - }); - // Tell Galaxy what version of the dependency kit we're using, so // it can load the right modules. (Include this even if we copied // or symlinked a node_modules, since that's probably enough for @@ -1589,6 +1565,10 @@ var writeSiteArchive = function (targets, outputPath, options) { } }); + _.each(targets, function (target, name) { + json.programs.push(writeTargetToPath(name, target, builder.buildPath, options)); + }); + // We did it! builder.complete(); @@ -1750,9 +1730,11 @@ exports.bundle = function (options) { targets.client = client; // Server - var server = options.cachedServerTarget || makeServerTarget(app, client); - server.clientTarget = client; - targets.server = server; + if (! options.hasCachedBundle) { + var server = makeServerTarget(app, client); + server.clientTarget = client; + targets.server = server; + } } // Pick up any additional targets in /programs @@ -1902,15 +1884,47 @@ exports.bundle = function (options) { if (! (controlProgram in targets)) controlProgram = undefined; + + // Hack to let servers find relative paths to clients. Should find + // another solution eventually (probably some kind of mount + // directive that mounts the client bundle in the server at runtime) + var getRelativeTargetPath = function (options) { + var pathForTarget = function (target) { + var name; + _.each(targets, function (t, n) { + if (t === target) + name = n; + }); + if (! name) + throw new Error("missing target?"); + return path.join('programs', name); + }; + + return path.relative(pathForTarget(options.relativeTo), + pathForTarget(options.forTarget)); + }; + // Write to disk - starResult = writeSiteArchive(targets, outputPath, { + var writeOptions = { includeNodeModulesSymlink: includeNodeModulesSymlink, builtBy: builtBy, controlProgram: controlProgram, - releaseName: releaseName - }); - serverWatchSet.merge(starResult.serverWatchSet); - clientWatchSet.merge(starResult.clientWatchSet); + releaseName: releaseName, + getRelativeTargetPath: getRelativeTargetPath + }; + + if (options.hasCachedBundle) { + // XXX If we already have a cached bundle, just recreate the new targets. + // This might make the contents of "star.json" out of date. + _.each(targets, function (target, name) { + writeTargetToPath(name, target, outputPath, options); + clientWatchSet.merge(target.getWatchSet()); + }); + } else { + starResult = writeSiteArchive(targets, outputPath, writeOptions); + serverWatchSet.merge(starResult.serverWatchSet); + clientWatchSet.merge(starResult.clientWatchSet); + } success = true; }); @@ -1922,8 +1936,7 @@ exports.bundle = function (options) { errors: success ? false : messages, serverWatchSet: serverWatchSet, clientWatchSet: clientWatchSet, - starManifest: starResult && starResult.starManifest, - serverTarget: targets.server + starManifest: starResult && starResult.starManifest }; }; diff --git a/tools/run-app.js b/tools/run-app.js index 0a52320191..51838dd87e 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -423,7 +423,7 @@ _.extend(AppRunner.prototype, { // Cache the server target because the server will not change inside // a single invocation of _runOnce(). - var cachedServerTarget = null; + var cachedBundle; var bundleApp = function () { if (! self.firstRun) packageCache.packageCache.refresh(true); // pick up changes to packages @@ -432,10 +432,17 @@ _.extend(AppRunner.prototype, { outputPath: bundlePath, includeNodeModulesSymlink: true, buildOptions: self.buildOptions, - cachedServerTarget: cachedServerTarget + hasCachedBundle: !! cachedBundle }); - cachedServerTarget = bundle.serverTarget; + // Overwrite the null elements in the new bundle with the elements from + // the cached bundle. However, we make sure to keep the serverWatchSet + // from the original cached bundle. + if (cachedBundle) { + bundle.serverWatchSet = cachedBundle.serverWatchSet; + } + cachedBundle = _.defaults(bundle, cachedBundle); + return bundle; }; diff --git a/tools/tests/apps/hot-code-push-test/hot-code-push-test.js b/tools/tests/apps/hot-code-push-test/hot-code-push-test.js index 3b37107e9c..c1f89ef574 100644 --- a/tools/tests/apps/hot-code-push-test/hot-code-push-test.js +++ b/tools/tests/apps/hot-code-push-test/hot-code-push-test.js @@ -9,6 +9,8 @@ if (Meteor.isServer) { Meteor.methods({ clientLoad: function (jsVar) { + // Make sure that the process still has the correct working directories. + process.cwd(); console.log("client connected: " + clientConnections++); console.log("jsVar: " + jsVar); } From b8ae210d9b3a1bac323aedcd69165fc0c7ce6832 Mon Sep 17 00:00:00 2001 From: Matthew Arbesfeld Date: Fri, 25 Jul 2014 17:31:05 -0700 Subject: [PATCH 051/126] Add documentation --- tools/bundler.js | 3 +++ tools/tests/apps/hot-code-push-test/hot-code-push-test.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/bundler.js b/tools/bundler.js index 4433e62aca..194f0f7672 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1608,6 +1608,9 @@ var writeSiteArchive = function (targets, outputPath, options) { * - minify: minify the CSS and JS assets (boolean, default false) * - arch: the server architecture to target (defaults to archinfo.host()) * + * - hasCachedBundle: true if we already have a cached bundle stored in + * /build. When true, we only build the new client targets in the bundle. + * * Returns an object with keys: * - errors: A buildmessage.MessageSet, or falsy if bundling succeeded. * - watchSet: Information about files and paths that were diff --git a/tools/tests/apps/hot-code-push-test/hot-code-push-test.js b/tools/tests/apps/hot-code-push-test/hot-code-push-test.js index c1f89ef574..e9e66d2631 100644 --- a/tools/tests/apps/hot-code-push-test/hot-code-push-test.js +++ b/tools/tests/apps/hot-code-push-test/hot-code-push-test.js @@ -9,7 +9,7 @@ if (Meteor.isServer) { Meteor.methods({ clientLoad: function (jsVar) { - // Make sure that the process still has the correct working directories. + // Make sure that the process still has the correct working directory. process.cwd(); console.log("client connected: " + clientConnections++); console.log("jsVar: " + jsVar); From c65a199c3cdd585fff3389b493197db409300f6c Mon Sep 17 00:00:00 2001 From: Matthew Arbesfeld Date: Mon, 28 Jul 2014 15:05:55 -0700 Subject: [PATCH 052/126] Glasser fixes --- tools/bundler.js | 14 ++++++++++---- tools/run-app.js | 16 ++++++++-------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/tools/bundler.js b/tools/bundler.js index 194f0f7672..e9a17c40da 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1566,7 +1566,13 @@ var writeSiteArchive = function (targets, outputPath, options) { }); _.each(targets, function (target, name) { - json.programs.push(writeTargetToPath(name, target, builder.buildPath, options)); + json.programs.push(writeTargetToPath(name, target, builder.buildPath, { + includeNodeModulesSymlink: options.includeNodeModulesSymlink, + builtBy: options.builtBy, + controlProgram: options.controlProgram, + releaseName: options.releaseName, + getRelativeTargetPath: options.getRelativeTargetPath + })); }); // We did it! @@ -1917,10 +1923,10 @@ exports.bundle = function (options) { }; if (options.hasCachedBundle) { - // XXX If we already have a cached bundle, just recreate the new targets. - // This might make the contents of "star.json" out of date. + // If we already have a cached bundle, just recreate the new targets. + // XXX This might make the contents of "star.json" out of date. _.each(targets, function (target, name) { - writeTargetToPath(name, target, outputPath, options); + writeTargetToPath(name, target, outputPath, writeOptions); clientWatchSet.merge(target.getWatchSet()); }); } else { diff --git a/tools/run-app.js b/tools/run-app.js index 51838dd87e..b4c8afb0d6 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -423,7 +423,7 @@ _.extend(AppRunner.prototype, { // Cache the server target because the server will not change inside // a single invocation of _runOnce(). - var cachedBundle; + var cachedServerWatchSet; var bundleApp = function () { if (! self.firstRun) packageCache.packageCache.refresh(true); // pick up changes to packages @@ -432,16 +432,16 @@ _.extend(AppRunner.prototype, { outputPath: bundlePath, includeNodeModulesSymlink: true, buildOptions: self.buildOptions, - hasCachedBundle: !! cachedBundle + hasCachedBundle: !! cachedServerWatchSet }); - // Overwrite the null elements in the new bundle with the elements from - // the cached bundle. However, we make sure to keep the serverWatchSet - // from the original cached bundle. - if (cachedBundle) { - bundle.serverWatchSet = cachedBundle.serverWatchSet; + // Keep the server watch set from the initial bundle, because subsequent + // bundles will not contain a server target. + if (cachedServerWatchSet) { + bundle.serverWatchSet = cachedServerWatchSet; + } else { + cachedServerWatchSet = bundle.serverWatchSet; } - cachedBundle = _.defaults(bundle, cachedBundle); return bundle; }; From 381956d40beb040ac1c35f02790bdb7f933d3e69 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 28 Jul 2014 17:36:30 -0700 Subject: [PATCH 053/126] make a test failure not abort the whole self-test --- tools/tests/registration.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tools/tests/registration.js b/tools/tests/registration.js index 1fd49aa31a..5f041de134 100644 --- a/tools/tests/registration.js +++ b/tools/tests/registration.js @@ -34,10 +34,10 @@ var expectInvalidToken = function (token) { // elapses. var waitForEmail = selftest.markStack(function (inbox, subjectRegExp, bodyRegExp, timeoutSecs) { + var timedOut = false; if (timeoutSecs) { var timeout = setTimeout(function () { - throw new Error('Waiting for email to ' + inbox + - ' timed out.'); + timedOut = true; }, timeoutSecs * 1000); } @@ -87,8 +87,12 @@ var waitForEmail = selftest.markStack(function (inbox, subjectRegExp, } }); - if (! match) + if (! match) { utils.sleepMs(3000); + if (timedOut) { + selftest.fail('Waiting for email to ' + inbox + ' timed out.'); + } + } } clearTimeout(timeout); From cd57ac3535bda4119a62b5bb2cfa4d733a9193c2 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 28 Jul 2014 20:24:07 -0700 Subject: [PATCH 054/126] isolate different servers' packages also, in self-test, only set $METEOR_PACKAGE_SERVER_URL for the specific runs that actually want the test server (using a tag) rather than kinda always by accident --- tools/commands-packages.js | 2 +- tools/config.js | 26 +++++++++++++++++++++++--- tools/selftest.js | 31 +++++++++++++++++++++---------- tools/tests/package-tests.js | 15 +++++++-------- tools/tests/publish.js | 10 ++++------ tools/tropohouse.js | 4 +++- 6 files changed, 59 insertions(+), 29 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 5eb1b6e0df..10e580a98d 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -446,7 +446,7 @@ main.registerCommand({ // directory, though we really shouldn't be, or, if we ever restructure the // way that we store packages in the meteor directory, we should be sure to // reevaluate what this command actually does. - var localPackageDir = path.join(files.getCurrentToolsDir(),"packages"); + var localPackageDir = path.join(files.getCurrentToolsDir(), "packages"); var contents = fs.readdirSync(localPackageDir); var myPackages = {}; var toPublish = {}; diff --git a/tools/config.js b/tools/config.js index 916a01376b..ad84f1c111 100644 --- a/tools/config.js +++ b/tools/config.js @@ -156,10 +156,12 @@ _.extend(exports, { } }, - getLocalPackageCacheFilename: function (serverUrl) { + // Note: this is NOT guaranteed to return a distinct prefix for every + // conceivable URL. But it sure ought to return a distinct prefix for every + // server we actually use. + getPackageServerFilePrefix: function (serverUrl) { var self = this; if (!serverUrl) serverUrl = self.getPackageServerUrl(); - var serLen = serverUrl.length; // Chop off http:// and https:// and trailing slashes. serverUrl = serverUrl.replace(/^\https:\/\//, ''); @@ -172,9 +174,27 @@ _.extend(exports, { // Replace other weird stuff with X. serverUrl = serverUrl.replace(/[^a-zA-Z0-9.:-]/g, 'X'); + return serverUrl; + }, + + getPackagesDirectoryName: function (serverUrl) { + var self = this; + + var prefix = config.getPackageServerFilePrefix(); + if (prefix !== 'packages') { + prefix = path.join('packages-from-server', prefix); + } + + return prefix; + }, + + getLocalPackageCacheFilename: function (serverUrl) { + var self = this; + var prefix = self.getPackageServerFilePrefix(); + // Should look like 'packages.data.json' in the default case // (test-packages.data.json before 0.9.0). - return serverUrl + ".data.json"; + return prefix + ".data.json"; }, getPackageStorage: function() { diff --git a/tools/selftest.js b/tools/selftest.js index 0e55c8604e..35c07bacd1 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -10,6 +10,7 @@ var archinfo = require('./archinfo.js'); var packageLoader = require('./package-loader.js'); var Future = require('fibers/future'); var uniload = require('./uniload.js'); +var config = require('./config.js'); var util = require('util'); var child_process = require('child_process'); var webdriver = require('browserstack-webdriver'); @@ -350,6 +351,14 @@ var Sandbox = function (options) { self.env = {}; self.fakeMongo = options.fakeMongo; + // By default, tests use the package server that this meteor binary is built + // with. If a test is tagged 'test-package-server', it uses the test + // server. Tests that publish packages should have this flag; tests that + // assume that the release's packages can be found on the server should not. + if (runningTest.tags['test-package-server']) { + self.set('METEOR_PACKAGE_SERVER_URL', 'https://test-packages.meteor.com'); + } + if (_.has(options, 'warehouse')) { if (!files.inCheckout()) throw Error("make only use a fake warehouse in a checkout"); @@ -581,9 +590,11 @@ _.extend(Sandbox.prototype, { // the default, if we do not pass this in; you should pass it in any case that // you will be specifying $METEOR_PACKAGE_SERVER_URL in the environment of a // command you are running in this sandbox. - _makeWarehouse: function (releases, packageServerUrl) { + _makeWarehouse: function (releases) { var self = this; - files.mkdir_p(path.join(self.warehouse, 'packages'), 0755); + var serverUrl = self.env.METEOR_PACKAGE_SERVER_URL; + var packagesDirectoryName = config.getPackagesDirectoryName(serverUrl); + files.mkdir_p(path.join(self.warehouse, packagesDirectoryName), 0755); files.mkdir_p(path.join(self.warehouse, 'package-metadata', 'v1'), 0755); var stubCatalog = { @@ -608,11 +619,11 @@ _.extend(Sandbox.prototype, { var toolPackageDirectory = '.' + toolPackage.version + '.XXX++' + toolPackage.buildArchitectures(); - toolPackage.saveToPath(path.join(self.warehouse, 'packages', + toolPackage.saveToPath(path.join(self.warehouse, packagesDirectoryName, toolPackageName, toolPackageDirectory)); fs.symlinkSync(toolPackageDirectory, - path.join(self.warehouse, 'packages', toolPackageName, - toolPackage.version)); + path.join(self.warehouse, packagesDirectoryName, + toolPackageName, toolPackage.version)); stubCatalog.collections.packages.push({ name: toolPackageName, _id: utils.randomToken() @@ -642,14 +653,14 @@ _.extend(Sandbox.prototype, { }); // Now create each requested release. - _.each(releases, function (config, releaseName) { + _.each(releases, function (configuration, releaseName) { // Release info stubCatalog.collections.releaseVersions.push({ track: catalog.complete.DEFAULT_TRACK, version: releaseName, orderKey: releaseName, description: "test release " + releaseName, - recommended: !!config.recommended, + recommended: !!configuration.recommended, // XXX support multiple tools packages for springboard tests tool: toolPackageName + "@" + toolPackage.version, packages: {} @@ -715,14 +726,14 @@ _.extend(Sandbox.prototype, { }); catalog.official.offline = oldOffline; - var config = require("./config.js"); - var dataFile = config. getLocalPackageCacheFilename(packageServerUrl); + var dataFile = config.getLocalPackageCacheFilename(serverUrl); fs.writeFileSync( path.join(self.warehouse, 'package-metadata', 'v1', dataFile), JSON.stringify(stubCatalog, null, 2)); // And a cherry on top - fs.symlinkSync(path.join('packages', toolPackageName, toolPackage.version, + fs.symlinkSync(path.join(packagesDirectoryName, + toolPackageName, toolPackage.version, 'meteor-tool-' + archinfo.host(), 'meteor'), path.join(self.warehouse, 'meteor')); } diff --git a/tools/tests/package-tests.js b/tools/tests/package-tests.js index 1821b99e1d..d67a47c80b 100644 --- a/tools/tests/package-tests.js +++ b/tools/tests/package-tests.js @@ -8,9 +8,6 @@ var fs = require("fs"); var path = require("path"); var packageClient = require("../package-client.js"); -var testPackagesServer = "https://test-packages.meteor.com"; -process.env.METEOR_PACKAGE_SERVER_URL = testPackagesServer; - var username = "test"; var password = "testtest"; @@ -89,7 +86,7 @@ var checkVersions = function(sand, packages) { // Add packages to an app. Change the contents of the packages and their // dependencies, make sure that the app still refreshes. -selftest.define("change packages", function () { +selftest.define("change packages", ['test-package-server'], function () { var s = new Sandbox(); var run; @@ -183,7 +180,7 @@ selftest.define("change packages", function () { // Add packages through the command line, and make sure that the correct set of // changes is reflected in .meteor/packages, .meteor/versions and list -selftest.define("add packages", ["net"], function () { +selftest.define("add packages", ["net", "test-package-server"], function () { var s = new Sandbox(); var run; @@ -307,7 +304,7 @@ var publishReleaseInNewTrack = function (s, releaseTrack, tool, packages) { // Add packages through the command line, and make sure that the correct set of // changes is reflected in .meteor/packages, .meteor/versions and list -selftest.define("sync local catalog", ["slow", "online"], function () { +selftest.define("sync local catalog", ["slow", "net", "test-package-server"], function () { var s = new Sandbox(); var run; @@ -413,7 +410,8 @@ var createAndPublishPackage = function (s, packageName) { s.cd(".."); }; -selftest.define("release track defaults to METEOR-CORE", ["net"], function () { +selftest.define("release track defaults to METEOR-CORE", + ["net", "test-package-server"], function () { var s = new Sandbox(); s.set("METEOR_TEST_TMP", files.mkdtemp()); testUtils.login(s, username, password); @@ -447,7 +445,8 @@ selftest.define("release track defaults to METEOR-CORE", ["net"], function () { }); -selftest.define("update server package data unit test", ["net"], function () { +selftest.define("update server package data unit test", + ["net", "test-package-server"], function () { var packageStorageFileDir = files.mkdtemp("update-server-package-data"); var packageStorageFile = path.join(packageStorageFileDir, "data.json"); var opts = { diff --git a/tools/tests/publish.js b/tools/tests/publish.js index ae254b8019..04bd39cec2 100644 --- a/tools/tests/publish.js +++ b/tools/tests/publish.js @@ -6,10 +6,7 @@ var selftest = require('../selftest.js'); var stats = require('../stats.js'); var Sandbox = selftest.Sandbox; -var testPackagesServer = "https://test-packages.meteor.com"; -process.env.METEOR_PACKAGE_SERVER_URL = testPackagesServer; - -selftest.define("publish-and-search", ["slow", "net"], function () { +selftest.define("publish-and-search", ["slow", "net", "test-package-server"], function () { var s = new Sandbox; var username = "test"; @@ -72,7 +69,7 @@ selftest.define("publish-and-search", ["slow", "net"], function () { run.match(githubUrl); }); -selftest.define("publish-one-arch", ["slow", "net"], function () { +selftest.define("publish-one-arch", ["slow", "net", "test-package-server"], function () { var s = new Sandbox; var username = "test"; @@ -110,7 +107,8 @@ selftest.define("publish-one-arch", ["slow", "net"], function () { }); -selftest.define("list-with-a-new-version", ["slow", "net"], function () { +selftest.define("list-with-a-new-version", + ["slow", "net", "test-package-server"], function () { var s = new Sandbox; var username = "test"; diff --git a/tools/tropohouse.js b/tools/tropohouse.js index 282c877017..47b4ac23b4 100644 --- a/tools/tropohouse.js +++ b/tools/tropohouse.js @@ -12,6 +12,7 @@ var release = require('./release.js'); var archinfo = require('./archinfo.js'); var catalog = require('./catalog.js'); var Unipackage = require('./unipackage.js').Unipackage; +var config = require('./config.js'); exports.Tropohouse = function (root, catalog) { var self = this; @@ -53,7 +54,8 @@ _.extend(exports.Tropohouse.prototype, { return null; } - var relativePath = path.join("packages", packageName, version); + var relativePath = path.join(config.getPackagesDirectoryName(), + packageName, version); return relative ? relativePath : path.join(self.root, relativePath); }, From cd8bd67ff86f42ba2188c719311c1912a71490ae Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 10:18:40 -0700 Subject: [PATCH 055/126] Fix a few exceptions in the oplog observe driver First exception: _runQuery didn't check to see if it was stopped after running the query, which could lead to this harmless error: Exception in defer callback: TypeError: Cannot call method 'clear' of null at _.extend._publishNewResults (packages/mongo-livedata/oplog_observe_driver.js:749) at _.extend._runQuery (packages/mongo-livedata/oplog_observe_driver.js:657) at packages/mongo-livedata/oplog_observe_driver.js:615 at _.extend.withValue (packages/meteor/dynamics_nodejs.js:56) at packages/meteor/timers.js:6 at runWithEnvironment (packages/meteor/dynamics_nodejs.js:108) Second exception: _fetchModifiedDocuments thought that it should only be in FETCHING in a certain case, but QUERYING is also OK. This is also harmless since the correct behavior is to end the deferred routine. Exception in defer callback: Error: phase in fetchModifiedDocuments: QUERYING at packages/mongo-livedata/oplog_observe_driver.js:435 at packages/mongo-livedata/oplog_observe_driver.js:16 at _.extend.withValue (packages/meteor/dynamics_nodejs.js:56) at packages/meteor/timers.js:6 at runWithEnvironment (packages/meteor/dynamics_nodejs.js:108) --- packages/mongo-livedata/oplog_observe_driver.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index ac03d24cb0..d9348a62b1 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -431,6 +431,14 @@ _.extend(OplogObserveDriver.prototype, { // fetch() yields. Meteor.defer(finishIfNeedToPollQuery(function () { while (!self._stopped && !self._needToFetch.empty()) { + if (self._phase === PHASE.QUERYING) { + // While fetching, we decided to go into QUERYING mode, and then we + // saw another oplog entry, so _needToFetch is not empty. But we + // shouldn't fetch these documents until AFTER the query is done. + break; + } + + // Being in steady phase here would be surprising. if (self._phase !== PHASE.FETCHING) throw new Error("phase in fetchModifiedDocuments: " + self._phase); @@ -654,6 +662,9 @@ _.extend(OplogObserveDriver.prototype, { } } + if (self._stopped) + return; + self._publishNewResults(newResults, newBuffer); }, @@ -789,6 +800,9 @@ _.extend(OplogObserveDriver.prototype, { // This stop function is invoked from the onStop of the ObserveMultiplexer, so // it shouldn't actually be possible to call it until the multiplexer is // ready. + // + // It's important to check self._stopped after every call in this file that + // can yield! stop: function () { var self = this; if (self._stopped) From b4e02f345a40db826ceae9da318f5532e8d6714e Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 10:18:40 -0700 Subject: [PATCH 056/126] Fix a few exceptions in the oplog observe driver First exception: _runQuery didn't check to see if it was stopped after running the query, which could lead to this harmless error: Exception in defer callback: TypeError: Cannot call method 'clear' of null at _.extend._publishNewResults (packages/mongo-livedata/oplog_observe_driver.js:749) at _.extend._runQuery (packages/mongo-livedata/oplog_observe_driver.js:657) at packages/mongo-livedata/oplog_observe_driver.js:615 at _.extend.withValue (packages/meteor/dynamics_nodejs.js:56) at packages/meteor/timers.js:6 at runWithEnvironment (packages/meteor/dynamics_nodejs.js:108) Second exception: _fetchModifiedDocuments thought that it should only be in FETCHING in a certain case, but QUERYING is also OK. This is also harmless since the correct behavior is to end the deferred routine. Exception in defer callback: Error: phase in fetchModifiedDocuments: QUERYING at packages/mongo-livedata/oplog_observe_driver.js:435 at packages/mongo-livedata/oplog_observe_driver.js:16 at _.extend.withValue (packages/meteor/dynamics_nodejs.js:56) at packages/meteor/timers.js:6 at runWithEnvironment (packages/meteor/dynamics_nodejs.js:108) --- packages/mongo-livedata/oplog_observe_driver.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index ac03d24cb0..d9348a62b1 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -431,6 +431,14 @@ _.extend(OplogObserveDriver.prototype, { // fetch() yields. Meteor.defer(finishIfNeedToPollQuery(function () { while (!self._stopped && !self._needToFetch.empty()) { + if (self._phase === PHASE.QUERYING) { + // While fetching, we decided to go into QUERYING mode, and then we + // saw another oplog entry, so _needToFetch is not empty. But we + // shouldn't fetch these documents until AFTER the query is done. + break; + } + + // Being in steady phase here would be surprising. if (self._phase !== PHASE.FETCHING) throw new Error("phase in fetchModifiedDocuments: " + self._phase); @@ -654,6 +662,9 @@ _.extend(OplogObserveDriver.prototype, { } } + if (self._stopped) + return; + self._publishNewResults(newResults, newBuffer); }, @@ -789,6 +800,9 @@ _.extend(OplogObserveDriver.prototype, { // This stop function is invoked from the onStop of the ObserveMultiplexer, so // it shouldn't actually be possible to call it until the multiplexer is // ready. + // + // It's important to check self._stopped after every call in this file that + // can yield! stop: function () { var self = this; if (self._stopped) From 1b9bb206cd229fe1639d9c1f94455865000ba5c3 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Tue, 29 Jul 2014 10:54:21 -0700 Subject: [PATCH 057/126] Revert "shrinkwrap update" This reverts commit abbf3c78fac5b188789eb07b55540f57e3a3ab4c. Something probably got weird switching between the 'packaging' branch and 'devel'; mongodb and bson are forked on packaging, but we didn't intend to be using a normal mongodb release with a forked bson. --- packages/mongo-livedata/.npm/package/npm-shrinkwrap.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json index 622dfc20e9..ea17322d4e 100644 --- a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json +++ b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json @@ -4,7 +4,7 @@ "version": "1.4.1", "dependencies": { "bson": { - "version": "https://github.com/meteor/js-bson/tarball/574c0ee", + "version": "0.2.7", "dependencies": { "nan": { "version": "0.8.0" From 0f3b16f9425f3ca3ff136a451dd2f32bdfa28416 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 11:01:42 -0700 Subject: [PATCH 058/126] Add a lot of _noYieldsAllowed to oplog driver Makes it easier to reason about where yields are (and ensure that _stopped is checked after all yielding calls) --- .../mongo-livedata/oplog_observe_driver.js | 905 +++++++++--------- 1 file changed, 475 insertions(+), 430 deletions(-) diff --git a/packages/mongo-livedata/oplog_observe_driver.js b/packages/mongo-livedata/oplog_observe_driver.js index d9348a62b1..1ba2e56d49 100644 --- a/packages/mongo-livedata/oplog_observe_driver.js +++ b/packages/mongo-livedata/oplog_observe_driver.js @@ -168,419 +168,453 @@ OplogObserveDriver = function (options) { _.extend(OplogObserveDriver.prototype, { _addPublished: function (id, doc) { var self = this; - var fields = _.clone(doc); - delete fields._id; - self._published.set(id, self._sharedProjectionFn(doc)); - self._multiplexer.added(id, self._projectionFn(fields)); + Meteor._noYieldsAllowed(function () { + var fields = _.clone(doc); + delete fields._id; + self._published.set(id, self._sharedProjectionFn(doc)); + self._multiplexer.added(id, self._projectionFn(fields)); - // After adding this document, the published set might be overflowed - // (exceeding capacity specified by limit). If so, push the maximum element - // to the buffer, we might want to save it in memory to reduce the amount of - // Mongo lookups in the future. - if (self._limit && self._published.size() > self._limit) { - // XXX in theory the size of published is no more than limit+1 - if (self._published.size() !== self._limit + 1) { - throw new Error("After adding to published, " + - (self._published.size() - self._limit) + - " documents are overflowing the set"); + // After adding this document, the published set might be overflowed + // (exceeding capacity specified by limit). If so, push the maximum + // element to the buffer, we might want to save it in memory to reduce the + // amount of Mongo lookups in the future. + if (self._limit && self._published.size() > self._limit) { + // XXX in theory the size of published is no more than limit+1 + if (self._published.size() !== self._limit + 1) { + throw new Error("After adding to published, " + + (self._published.size() - self._limit) + + " documents are overflowing the set"); + } + + var overflowingDocId = self._published.maxElementId(); + var overflowingDoc = self._published.get(overflowingDocId); + + if (EJSON.equals(overflowingDocId, id)) { + throw new Error("The document just added is overflowing the published set"); + } + + self._published.remove(overflowingDocId); + self._multiplexer.removed(overflowingDocId); + self._addBuffered(overflowingDocId, overflowingDoc); } - - var overflowingDocId = self._published.maxElementId(); - var overflowingDoc = self._published.get(overflowingDocId); - - if (EJSON.equals(overflowingDocId, id)) { - throw new Error("The document just added is overflowing the published set"); - } - - self._published.remove(overflowingDocId); - self._multiplexer.removed(overflowingDocId); - self._addBuffered(overflowingDocId, overflowingDoc); - } + }); }, _removePublished: function (id) { var self = this; - self._published.remove(id); - self._multiplexer.removed(id); - if (! self._limit || self._published.size() === self._limit) - return; + Meteor._noYieldsAllowed(function () { + self._published.remove(id); + self._multiplexer.removed(id); + if (! self._limit || self._published.size() === self._limit) + return; - if (self._published.size() > self._limit) - throw Error("self._published got too big"); + if (self._published.size() > self._limit) + throw Error("self._published got too big"); - // OK, we are publishing less than the limit. Maybe we should look in the - // buffer to find the next element past what we were publishing before. + // OK, we are publishing less than the limit. Maybe we should look in the + // buffer to find the next element past what we were publishing before. - if (!self._unpublishedBuffer.empty()) { - // There's something in the buffer; move the first thing in it to - // _published. - var newDocId = self._unpublishedBuffer.minElementId(); - var newDoc = self._unpublishedBuffer.get(newDocId); - self._removeBuffered(newDocId); - self._addPublished(newDocId, newDoc); - return; - } + if (!self._unpublishedBuffer.empty()) { + // There's something in the buffer; move the first thing in it to + // _published. + var newDocId = self._unpublishedBuffer.minElementId(); + var newDoc = self._unpublishedBuffer.get(newDocId); + self._removeBuffered(newDocId); + self._addPublished(newDocId, newDoc); + return; + } - // There's nothing in the buffer. This could mean one of a few things. + // There's nothing in the buffer. This could mean one of a few things. - // (a) We could be in the middle of re-running the query (specifically, we - // could be in _publishNewResults). In that case, _unpublishedBuffer is - // empty because we clear it at the beginning of _publishNewResults. In this - // case, our caller already knows the entire answer to the query and we - // don't need to do anything fancy here. Just return. - if (self._phase === PHASE.QUERYING) - return; + // (a) We could be in the middle of re-running the query (specifically, we + // could be in _publishNewResults). In that case, _unpublishedBuffer is + // empty because we clear it at the beginning of _publishNewResults. In + // this case, our caller already knows the entire answer to the query and + // we don't need to do anything fancy here. Just return. + if (self._phase === PHASE.QUERYING) + return; - // (b) We're pretty confident that the union of _published and - // _unpublishedBuffer contain all documents that match selector. Because - // _unpublishedBuffer is empty, that means we're confident that _published - // contains all documents that match selector. So we have nothing to do. - if (self._safeAppendToBuffer) - return; + // (b) We're pretty confident that the union of _published and + // _unpublishedBuffer contain all documents that match selector. Because + // _unpublishedBuffer is empty, that means we're confident that _published + // contains all documents that match selector. So we have nothing to do. + if (self._safeAppendToBuffer) + return; - // (c) Maybe there are other documents out there that should be in our - // buffer. But in that case, when we emptied _unpublishedBuffer in - // _removeBuffered, we should have called _needToPollQuery, which will - // either put something in _unpublishedBuffer or set _safeAppendToBuffer (or - // both), and it will put us in QUERYING for that whole time. So in fact, we - // shouldn't be able to get here. + // (c) Maybe there are other documents out there that should be in our + // buffer. But in that case, when we emptied _unpublishedBuffer in + // _removeBuffered, we should have called _needToPollQuery, which will + // either put something in _unpublishedBuffer or set _safeAppendToBuffer + // (or both), and it will put us in QUERYING for that whole time. So in + // fact, we shouldn't be able to get here. - throw new Error("Buffer inexplicably empty"); + throw new Error("Buffer inexplicably empty"); + }); }, _changePublished: function (id, oldDoc, newDoc) { var self = this; - self._published.set(id, self._sharedProjectionFn(newDoc)); - var changed = LocalCollection._makeChangedFields(_.clone(newDoc), oldDoc); - changed = self._projectionFn(changed); - if (!_.isEmpty(changed)) - self._multiplexer.changed(id, changed); + Meteor._noYieldsAllowed(function () { + self._published.set(id, self._sharedProjectionFn(newDoc)); + var changed = LocalCollection._makeChangedFields(_.clone(newDoc), oldDoc); + changed = self._projectionFn(changed); + if (!_.isEmpty(changed)) + self._multiplexer.changed(id, changed); + }); }, _addBuffered: function (id, doc) { var self = this; - self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc)); + Meteor._noYieldsAllowed(function () { + self._unpublishedBuffer.set(id, self._sharedProjectionFn(doc)); - // If something is overflowing the buffer, we just remove it from cache - if (self._unpublishedBuffer.size() > self._limit) { - var maxBufferedId = self._unpublishedBuffer.maxElementId(); + // If something is overflowing the buffer, we just remove it from cache + if (self._unpublishedBuffer.size() > self._limit) { + var maxBufferedId = self._unpublishedBuffer.maxElementId(); - self._unpublishedBuffer.remove(maxBufferedId); + self._unpublishedBuffer.remove(maxBufferedId); - // Since something matching is removed from cache (both published set and - // buffer), set flag to false - self._safeAppendToBuffer = false; - } + // Since something matching is removed from cache (both published set and + // buffer), set flag to false + self._safeAppendToBuffer = false; + } + }); }, // Is called either to remove the doc completely from matching set or to move // it to the published set later. _removeBuffered: function (id) { var self = this; - self._unpublishedBuffer.remove(id); - // To keep the contract "buffer is never empty in STEADY phase unless the - // everything matching fits into published" true, we poll everything as soon - // as we see the buffer becoming empty. - if (! self._unpublishedBuffer.size() && ! self._safeAppendToBuffer) - self._needToPollQuery(); + Meteor._noYieldsAllowed(function () { + self._unpublishedBuffer.remove(id); + // To keep the contract "buffer is never empty in STEADY phase unless the + // everything matching fits into published" true, we poll everything as + // soon as we see the buffer becoming empty. + if (! self._unpublishedBuffer.size() && ! self._safeAppendToBuffer) + self._needToPollQuery(); + }); }, // Called when a document has joined the "Matching" results set. // Takes responsibility of keeping _unpublishedBuffer in sync with _published // and the effect of limit enforced. _addMatching: function (doc) { var self = this; - var id = doc._id; - if (self._published.has(id)) - throw Error("tried to add something already published " + id); - if (self._limit && self._unpublishedBuffer.has(id)) - throw Error("tried to add something already existed in buffer " + id); + Meteor._noYieldsAllowed(function () { + var id = doc._id; + if (self._published.has(id)) + throw Error("tried to add something already published " + id); + if (self._limit && self._unpublishedBuffer.has(id)) + throw Error("tried to add something already existed in buffer " + id); - var limit = self._limit; - var comparator = self._comparator; - var maxPublished = (limit && self._published.size() > 0) ? - self._published.get(self._published.maxElementId()) : null; - var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) ? - self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) : null; - // The query is unlimited or didn't publish enough documents yet or the new - // document would fit into published set pushing the maximum element out, - // then we need to publish the doc. - var toPublish = ! limit || self._published.size() < limit || - comparator(doc, maxPublished) < 0; + var limit = self._limit; + var comparator = self._comparator; + var maxPublished = (limit && self._published.size() > 0) ? + self._published.get(self._published.maxElementId()) : null; + var maxBuffered = (limit && self._unpublishedBuffer.size() > 0) + ? self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()) + : null; + // The query is unlimited or didn't publish enough documents yet or the + // new document would fit into published set pushing the maximum element + // out, then we need to publish the doc. + var toPublish = ! limit || self._published.size() < limit || + comparator(doc, maxPublished) < 0; - // Otherwise we might need to buffer it (only in case of limited query). - // Buffering is allowed if the buffer is not filled up yet and all matching - // docs are either in the published set or in the buffer. - var canAppendToBuffer = !toPublish && self._safeAppendToBuffer && - self._unpublishedBuffer.size() < limit; + // Otherwise we might need to buffer it (only in case of limited query). + // Buffering is allowed if the buffer is not filled up yet and all + // matching docs are either in the published set or in the buffer. + var canAppendToBuffer = !toPublish && self._safeAppendToBuffer && + self._unpublishedBuffer.size() < limit; - // Or if it is small enough to be safely inserted to the middle or the - // beginning of the buffer. - var canInsertIntoBuffer = !toPublish && maxBuffered && - comparator(doc, maxBuffered) <= 0; + // Or if it is small enough to be safely inserted to the middle or the + // beginning of the buffer. + var canInsertIntoBuffer = !toPublish && maxBuffered && + comparator(doc, maxBuffered) <= 0; - var toBuffer = canAppendToBuffer || canInsertIntoBuffer; + var toBuffer = canAppendToBuffer || canInsertIntoBuffer; - if (toPublish) { - self._addPublished(id, doc); - } else if (toBuffer) { - self._addBuffered(id, doc); - } else { - // dropping it and not saving to the cache - self._safeAppendToBuffer = false; - } + if (toPublish) { + self._addPublished(id, doc); + } else if (toBuffer) { + self._addBuffered(id, doc); + } else { + // dropping it and not saving to the cache + self._safeAppendToBuffer = false; + } + }); }, // Called when a document leaves the "Matching" results set. // Takes responsibility of keeping _unpublishedBuffer in sync with _published // and the effect of limit enforced. _removeMatching: function (id) { var self = this; - if (! self._published.has(id) && ! self._limit) - throw Error("tried to remove something matching but not cached " + id); + Meteor._noYieldsAllowed(function () { + if (! self._published.has(id) && ! self._limit) + throw Error("tried to remove something matching but not cached " + id); - if (self._published.has(id)) { - self._removePublished(id); - } else if (self._unpublishedBuffer.has(id)) { - self._removeBuffered(id); - } + if (self._published.has(id)) { + self._removePublished(id); + } else if (self._unpublishedBuffer.has(id)) { + self._removeBuffered(id); + } + }); }, _handleDoc: function (id, newDoc) { var self = this; - var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result; + Meteor._noYieldsAllowed(function () { + var matchesNow = newDoc && self._matcher.documentMatches(newDoc).result; - var publishedBefore = self._published.has(id); - var bufferedBefore = self._limit && self._unpublishedBuffer.has(id); - var cachedBefore = publishedBefore || bufferedBefore; + var publishedBefore = self._published.has(id); + var bufferedBefore = self._limit && self._unpublishedBuffer.has(id); + var cachedBefore = publishedBefore || bufferedBefore; - if (matchesNow && !cachedBefore) { - self._addMatching(newDoc); - } else if (cachedBefore && !matchesNow) { - self._removeMatching(id); - } else if (cachedBefore && matchesNow) { - var oldDoc = self._published.get(id); - var comparator = self._comparator; - var minBuffered = self._limit && self._unpublishedBuffer.size() && - self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId()); + if (matchesNow && !cachedBefore) { + self._addMatching(newDoc); + } else if (cachedBefore && !matchesNow) { + self._removeMatching(id); + } else if (cachedBefore && matchesNow) { + var oldDoc = self._published.get(id); + var comparator = self._comparator; + var minBuffered = self._limit && self._unpublishedBuffer.size() && + self._unpublishedBuffer.get(self._unpublishedBuffer.minElementId()); - if (publishedBefore) { - // Unlimited case where the document stays in published once it matches - // or the case when we don't have enough matching docs to publish or the - // changed but matching doc will stay in published anyways. - // XXX: We rely on the emptiness of buffer. Be sure to maintain the fact - // that buffer can't be empty if there are matching documents not - // published. Notably, we don't want to schedule repoll and continue - // relying on this property. - var staysInPublished = ! self._limit || - self._unpublishedBuffer.size() === 0 || - comparator(newDoc, minBuffered) <= 0; + if (publishedBefore) { + // Unlimited case where the document stays in published once it + // matches or the case when we don't have enough matching docs to + // publish or the changed but matching doc will stay in published + // anyways. + // + // XXX: We rely on the emptiness of buffer. Be sure to maintain the + // fact that buffer can't be empty if there are matching documents not + // published. Notably, we don't want to schedule repoll and continue + // relying on this property. + var staysInPublished = ! self._limit || + self._unpublishedBuffer.size() === 0 || + comparator(newDoc, minBuffered) <= 0; - if (staysInPublished) { - self._changePublished(id, oldDoc, newDoc); - } else { - // after the change doc doesn't stay in the published, remove it - self._removePublished(id); - // but it can move into buffered now, check it - var maxBuffered = self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()); + if (staysInPublished) { + self._changePublished(id, oldDoc, newDoc); + } else { + // after the change doc doesn't stay in the published, remove it + self._removePublished(id); + // but it can move into buffered now, check it + var maxBuffered = self._unpublishedBuffer.get( + self._unpublishedBuffer.maxElementId()); - var toBuffer = self._safeAppendToBuffer || - (maxBuffered && comparator(newDoc, maxBuffered) <= 0); + var toBuffer = self._safeAppendToBuffer || + (maxBuffered && comparator(newDoc, maxBuffered) <= 0); - if (toBuffer) { - self._addBuffered(id, newDoc); + if (toBuffer) { + self._addBuffered(id, newDoc); + } else { + // Throw away from both published set and buffer + self._safeAppendToBuffer = false; + } + } + } else if (bufferedBefore) { + oldDoc = self._unpublishedBuffer.get(id); + // remove the old version manually instead of using _removeBuffered so + // we don't trigger the querying immediately. if we end this block + // with the buffer empty, we will need to trigger the query poll + // manually too. + self._unpublishedBuffer.remove(id); + + var maxPublished = self._published.get( + self._published.maxElementId()); + var maxBuffered = self._unpublishedBuffer.size() && + self._unpublishedBuffer.get( + self._unpublishedBuffer.maxElementId()); + + // the buffered doc was updated, it could move to published + var toPublish = comparator(newDoc, maxPublished) < 0; + + // or stays in buffer even after the change + var staysInBuffer = (! toPublish && self._safeAppendToBuffer) || + (!toPublish && maxBuffered && + comparator(newDoc, maxBuffered) <= 0); + + if (toPublish) { + self._addPublished(id, newDoc); + } else if (staysInBuffer) { + // stays in buffer but changes + self._unpublishedBuffer.set(id, newDoc); } else { // Throw away from both published set and buffer self._safeAppendToBuffer = false; + // Normally this check would have been done in _removeBuffered but + // we didn't use it, so we need to do it ourself now. + if (! self._unpublishedBuffer.size()) { + self._needToPollQuery(); + } } - } - } else if (bufferedBefore) { - oldDoc = self._unpublishedBuffer.get(id); - // remove the old version manually instead of using _removeBuffered so - // we don't trigger the querying immediately. if we end this block with - // the buffer empty, we will need to trigger the query poll manually - // too. - self._unpublishedBuffer.remove(id); - - var maxPublished = self._published.get(self._published.maxElementId()); - var maxBuffered = self._unpublishedBuffer.size() && self._unpublishedBuffer.get(self._unpublishedBuffer.maxElementId()); - - // the buffered doc was updated, it could move to published - var toPublish = comparator(newDoc, maxPublished) < 0; - - // or stays in buffer even after the change - var staysInBuffer = (! toPublish && self._safeAppendToBuffer) || - (!toPublish && maxBuffered && comparator(newDoc, maxBuffered) <= 0); - - if (toPublish) { - self._addPublished(id, newDoc); - } else if (staysInBuffer) { - // stays in buffer but changes - self._unpublishedBuffer.set(id, newDoc); } else { - // Throw away from both published set and buffer - self._safeAppendToBuffer = false; - // Normally this check would have been done in _removeBuffered but we - // didn't use it, so we need to do it ourself now. - if (! self._unpublishedBuffer.size()) { - self._needToPollQuery(); - } + throw new Error("cachedBefore implies either of publishedBefore or bufferedBefore is true."); } - } else { - throw new Error("cachedBefore implies either of publishedBefore or bufferedBefore is true."); } - } + }); }, _fetchModifiedDocuments: function () { var self = this; - self._registerPhaseChange(PHASE.FETCHING); - // Defer, because nothing called from the oplog entry handler may yield, but - // fetch() yields. - Meteor.defer(finishIfNeedToPollQuery(function () { - while (!self._stopped && !self._needToFetch.empty()) { - if (self._phase === PHASE.QUERYING) { - // While fetching, we decided to go into QUERYING mode, and then we - // saw another oplog entry, so _needToFetch is not empty. But we - // shouldn't fetch these documents until AFTER the query is done. - break; - } + Meteor._noYieldsAllowed(function () { + self._registerPhaseChange(PHASE.FETCHING); + // Defer, because nothing called from the oplog entry handler may yield, + // but fetch() yields. + Meteor.defer(finishIfNeedToPollQuery(function () { + while (!self._stopped && !self._needToFetch.empty()) { + if (self._phase === PHASE.QUERYING) { + // While fetching, we decided to go into QUERYING mode, and then we + // saw another oplog entry, so _needToFetch is not empty. But we + // shouldn't fetch these documents until AFTER the query is done. + break; + } - // Being in steady phase here would be surprising. - if (self._phase !== PHASE.FETCHING) - throw new Error("phase in fetchModifiedDocuments: " + self._phase); + // Being in steady phase here would be surprising. + if (self._phase !== PHASE.FETCHING) + throw new Error("phase in fetchModifiedDocuments: " + self._phase); - self._currentlyFetching = self._needToFetch; - var thisGeneration = ++self._fetchGeneration; - self._needToFetch = new LocalCollection._IdMap; - var waiting = 0; - var fut = new Future; - // This loop is safe, because _currentlyFetching will not be updated - // during this loop (in fact, it is never mutated). - self._currentlyFetching.forEach(function (cacheKey, id) { - waiting++; - self._mongoHandle._docFetcher.fetch( - self._cursorDescription.collectionName, id, cacheKey, - finishIfNeedToPollQuery(function (err, doc) { - try { - if (err) { - Meteor._debug("Got exception while fetching documents: " + - err); - // If we get an error from the fetcher (eg, trouble connecting - // to Mongo), let's just abandon the fetch phase altogether - // and fall back to polling. It's not like we're getting live - // updates anyway. - if (self._phase !== PHASE.QUERYING) { - self._needToPollQuery(); + self._currentlyFetching = self._needToFetch; + var thisGeneration = ++self._fetchGeneration; + self._needToFetch = new LocalCollection._IdMap; + var waiting = 0; + var fut = new Future; + // This loop is safe, because _currentlyFetching will not be updated + // during this loop (in fact, it is never mutated). + self._currentlyFetching.forEach(function (cacheKey, id) { + waiting++; + self._mongoHandle._docFetcher.fetch( + self._cursorDescription.collectionName, id, cacheKey, + finishIfNeedToPollQuery(function (err, doc) { + try { + if (err) { + Meteor._debug("Got exception while fetching documents: " + + err); + // If we get an error from the fetcher (eg, trouble + // connecting to Mongo), let's just abandon the fetch phase + // altogether and fall back to polling. It's not like we're + // getting live updates anyway. + if (self._phase !== PHASE.QUERYING) { + self._needToPollQuery(); + } + } else if (!self._stopped && self._phase === PHASE.FETCHING + && self._fetchGeneration === thisGeneration) { + // We re-check the generation in case we've had an explicit + // _pollQuery call (eg, in another fiber) which should + // effectively cancel this round of fetches. (_pollQuery + // increments the generation.) + self._handleDoc(id, doc); } - } else if (!self._stopped && self._phase === PHASE.FETCHING - && self._fetchGeneration === thisGeneration) { - // We re-check the generation in case we've had an explicit - // _pollQuery call (eg, in another fiber) which should - // effectively cancel this round of fetches. (_pollQuery - // increments the generation.) - self._handleDoc(id, doc); + } finally { + waiting--; + // Because fetch() never calls its callback synchronously, + // this is safe (ie, we won't call fut.return() before the + // forEach is done). + if (waiting === 0) + fut.return(); } - } finally { - waiting--; - // Because fetch() never calls its callback synchronously, this - // is safe (ie, we won't call fut.return() before the forEach is - // done). - if (waiting === 0) - fut.return(); - } - })); - }); - fut.wait(); - // Exit now if we've had a _pollQuery call (here or in another fiber). - if (self._phase === PHASE.QUERYING) - return; - self._currentlyFetching = null; - } - // We're done fetching, so we can be steady, unless we've had a _pollQuery - // call (here or in another fiber). - if (self._phase !== PHASE.QUERYING) - self._beSteady(); - })); + })); + }); + fut.wait(); + // Exit now if we've had a _pollQuery call (here or in another fiber). + if (self._phase === PHASE.QUERYING) + return; + self._currentlyFetching = null; + } + // We're done fetching, so we can be steady, unless we've had a + // _pollQuery call (here or in another fiber). + if (self._phase !== PHASE.QUERYING) + self._beSteady(); + })); + }); }, _beSteady: function () { var self = this; - self._registerPhaseChange(PHASE.STEADY); - var writes = self._writesToCommitWhenWeReachSteady; - self._writesToCommitWhenWeReachSteady = []; - self._multiplexer.onFlush(function () { - _.each(writes, function (w) { - w.committed(); + Meteor._noYieldsAllowed(function () { + self._registerPhaseChange(PHASE.STEADY); + var writes = self._writesToCommitWhenWeReachSteady; + self._writesToCommitWhenWeReachSteady = []; + self._multiplexer.onFlush(function () { + _.each(writes, function (w) { + w.committed(); + }); }); }); }, _handleOplogEntryQuerying: function (op) { var self = this; - self._needToFetch.set(idForOp(op), op.ts.toString()); + Meteor._noYieldsAllowed(function () { + self._needToFetch.set(idForOp(op), op.ts.toString()); + }); }, _handleOplogEntrySteadyOrFetching: function (op) { var self = this; - var id = idForOp(op); - // If we're already fetching this one, or about to, we can't optimize; make - // sure that we fetch it again if necessary. - if (self._phase === PHASE.FETCHING && - ((self._currentlyFetching && self._currentlyFetching.has(id)) || - self._needToFetch.has(id))) { - self._needToFetch.set(id, op.ts.toString()); - return; - } - - if (op.op === 'd') { - if (self._published.has(id) || (self._limit && self._unpublishedBuffer.has(id))) - self._removeMatching(id); - } else if (op.op === 'i') { - if (self._published.has(id)) - throw new Error("insert found for already-existing ID in published"); - if (self._unpublishedBuffer && self._unpublishedBuffer.has(id)) - throw new Error("insert found for already-existing ID in buffer"); - - // XXX what if selector yields? for now it can't but later it could have - // $where - if (self._matcher.documentMatches(op.o).result) - self._addMatching(op.o); - } else if (op.op === 'u') { - // Is this a modifier ($set/$unset, which may require us to poll the - // database to figure out if the whole document matches the selector) or a - // replacement (in which case we can just directly re-evaluate the - // selector)? - var isReplace = !_.has(op.o, '$set') && !_.has(op.o, '$unset'); - // If this modifier modifies something inside an EJSON custom type (ie, - // anything with EJSON$), then we can't try to use - // LocalCollection._modify, since that just mutates the EJSON encoding, - // not the actual object. - var canDirectlyModifyDoc = - !isReplace && modifierCanBeDirectlyApplied(op.o); - - var publishedBefore = self._published.has(id); - var bufferedBefore = self._limit && self._unpublishedBuffer.has(id); - - if (isReplace) { - self._handleDoc(id, _.extend({_id: id}, op.o)); - } else if ((publishedBefore || bufferedBefore) && canDirectlyModifyDoc) { - // Oh great, we actually know what the document is, so we can apply - // this directly. - var newDoc = self._published.has(id) ? - self._published.get(id) : - self._unpublishedBuffer.get(id); - newDoc = EJSON.clone(newDoc); - - newDoc._id = id; - LocalCollection._modify(newDoc, op.o); - self._handleDoc(id, self._sharedProjectionFn(newDoc)); - } else if (!canDirectlyModifyDoc || - self._matcher.canBecomeTrueByModifier(op.o) || - (self._sorter && self._sorter.affectedByModifier(op.o))) { + Meteor._noYieldsAllowed(function () { + var id = idForOp(op); + // If we're already fetching this one, or about to, we can't optimize; + // make sure that we fetch it again if necessary. + if (self._phase === PHASE.FETCHING && + ((self._currentlyFetching && self._currentlyFetching.has(id)) || + self._needToFetch.has(id))) { self._needToFetch.set(id, op.ts.toString()); - if (self._phase === PHASE.STEADY) - self._fetchModifiedDocuments(); + return; } - } else { - throw Error("XXX SURPRISING OPERATION: " + op); - } + + if (op.op === 'd') { + if (self._published.has(id) || + (self._limit && self._unpublishedBuffer.has(id))) + self._removeMatching(id); + } else if (op.op === 'i') { + if (self._published.has(id)) + throw new Error("insert found for already-existing ID in published"); + if (self._unpublishedBuffer && self._unpublishedBuffer.has(id)) + throw new Error("insert found for already-existing ID in buffer"); + + // XXX what if selector yields? for now it can't but later it could + // have $where + if (self._matcher.documentMatches(op.o).result) + self._addMatching(op.o); + } else if (op.op === 'u') { + // Is this a modifier ($set/$unset, which may require us to poll the + // database to figure out if the whole document matches the selector) or + // a replacement (in which case we can just directly re-evaluate the + // selector)? + var isReplace = !_.has(op.o, '$set') && !_.has(op.o, '$unset'); + // If this modifier modifies something inside an EJSON custom type (ie, + // anything with EJSON$), then we can't try to use + // LocalCollection._modify, since that just mutates the EJSON encoding, + // not the actual object. + var canDirectlyModifyDoc = + !isReplace && modifierCanBeDirectlyApplied(op.o); + + var publishedBefore = self._published.has(id); + var bufferedBefore = self._limit && self._unpublishedBuffer.has(id); + + if (isReplace) { + self._handleDoc(id, _.extend({_id: id}, op.o)); + } else if ((publishedBefore || bufferedBefore) && + canDirectlyModifyDoc) { + // Oh great, we actually know what the document is, so we can apply + // this directly. + var newDoc = self._published.has(id) + ? self._published.get(id) : self._unpublishedBuffer.get(id); + newDoc = EJSON.clone(newDoc); + + newDoc._id = id; + LocalCollection._modify(newDoc, op.o); + self._handleDoc(id, self._sharedProjectionFn(newDoc)); + } else if (!canDirectlyModifyDoc || + self._matcher.canBecomeTrueByModifier(op.o) || + (self._sorter && self._sorter.affectedByModifier(op.o))) { + self._needToFetch.set(id, op.ts.toString()); + if (self._phase === PHASE.STEADY) + self._fetchModifiedDocuments(); + } + } else { + throw Error("XXX SURPRISING OPERATION: " + op); + } + }); }, + // Yields! _runInitialQuery: function () { var self = this; if (self._stopped) throw new Error("oplog stopped surprisingly early"); - self._runQuery(); + self._runQuery(); // yields if (self._stopped) throw new Error("oplog stopped quite early"); @@ -588,7 +622,7 @@ _.extend(OplogObserveDriver.prototype, { // stop() to be called.) self._multiplexer.ready(); - self._doneQuerying(); + self._doneQuerying(); // yields }, // In various circumstances, we may just want to stop processing the oplog and @@ -607,24 +641,26 @@ _.extend(OplogObserveDriver.prototype, { // changes. Will put off implementing this until driver 1.4 is out. _pollQuery: function () { var self = this; + Meteor._noYieldsAllowed(function () { + if (self._stopped) + return; - if (self._stopped) - return; + // Yay, we get to forget about all the things we thought we had to fetch. + self._needToFetch = new LocalCollection._IdMap; + self._currentlyFetching = null; + ++self._fetchGeneration; // ignore any in-flight fetches + self._registerPhaseChange(PHASE.QUERYING); - // Yay, we get to forget about all the things we thought we had to fetch. - self._needToFetch = new LocalCollection._IdMap; - self._currentlyFetching = null; - ++self._fetchGeneration; // ignore any in-flight fetches - self._registerPhaseChange(PHASE.QUERYING); - - // Defer so that we don't block. We don't need finishIfNeedToPollQuery here - // because SwitchedToQuery is not called in QUERYING mode. - Meteor.defer(function () { - self._runQuery(); - self._doneQuerying(); + // Defer so that we don't yield. We don't need finishIfNeedToPollQuery + // here because SwitchedToQuery is not thrown in QUERYING mode. + Meteor.defer(function () { + self._runQuery(); + self._doneQuerying(); + }); }); }, + // Yields! _runQuery: function () { var self = this; var newResults, newBuffer; @@ -647,7 +683,7 @@ _.extend(OplogObserveDriver.prototype, { // buffer if such is needed. var cursor = self._cursorForQuery({ limit: self._limit * 2 }); try { - cursor.forEach(function (doc, i) { + cursor.forEach(function (doc, i) { // yields if (!self._limit || i < self._limit) newResults.set(doc._id, doc); else @@ -682,65 +718,70 @@ _.extend(OplogObserveDriver.prototype, { // _fetchModifiedDocuments does this.) _needToPollQuery: function () { var self = this; - if (self._stopped) - return; + Meteor._noYieldsAllowed(function () { + if (self._stopped) + return; - // If we're not already in the middle of a query, we can query now (possibly - // pausing FETCHING). - if (self._phase !== PHASE.QUERYING) { - self._pollQuery(); - throw new SwitchedToQuery; - } + // If we're not already in the middle of a query, we can query now + // (possibly pausing FETCHING). + if (self._phase !== PHASE.QUERYING) { + self._pollQuery(); + throw new SwitchedToQuery; + } - // We're currently in QUERYING. Set a flag to ensure that we run another - // query when we're done. - self._requeryWhenDoneThisQuery = true; + // We're currently in QUERYING. Set a flag to ensure that we run another + // query when we're done. + self._requeryWhenDoneThisQuery = true; + }); }, + // Yields! _doneQuerying: function () { var self = this; if (self._stopped) return; - self._mongoHandle._oplogHandle.waitUntilCaughtUp(); - + self._mongoHandle._oplogHandle.waitUntilCaughtUp(); // yields if (self._stopped) return; if (self._phase !== PHASE.QUERYING) throw Error("Phase unexpectedly " + self._phase); - if (self._requeryWhenDoneThisQuery) { - self._requeryWhenDoneThisQuery = false; - self._pollQuery(); - } else if (self._needToFetch.empty()) { - self._beSteady(); - } else { - self._fetchModifiedDocuments(); - } + Meteor._noYieldsAllowed(function () { + if (self._requeryWhenDoneThisQuery) { + self._requeryWhenDoneThisQuery = false; + self._pollQuery(); + } else if (self._needToFetch.empty()) { + self._beSteady(); + } else { + self._fetchModifiedDocuments(); + } + }); }, _cursorForQuery: function (optionsOverwrite) { var self = this; + return Meteor._noYieldsAllowed(function () { + // The query we run is almost the same as the cursor we are observing, + // with a few changes. We need to read all the fields that are relevant to + // the selector, not just the fields we are going to publish (that's the + // "shared" projection). And we don't want to apply any transform in the + // cursor, because observeChanges shouldn't use the transform. + var options = _.clone(self._cursorDescription.options); - // The query we run is almost the same as the cursor we are observing, with - // a few changes. We need to read all the fields that are relevant to the - // selector, not just the fields we are going to publish (that's the - // "shared" projection). And we don't want to apply any transform in the - // cursor, because observeChanges shouldn't use the transform. - var options = _.clone(self._cursorDescription.options); + // Allow the caller to modify the options. Useful to specify different + // skip and limit values. + _.extend(options, optionsOverwrite); - // Allow the caller to modify the options. Useful to specify different skip - // and limit values. - _.extend(options, optionsOverwrite); - - options.fields = self._sharedProjection; - delete options.transform; - // We are NOT deep cloning fields or selector here, which should be OK. - var description = new CursorDescription( - self._cursorDescription.collectionName, - self._cursorDescription.selector, - options); - return new Cursor(self._mongoHandle, description); + options.fields = self._sharedProjection; + delete options.transform; + // We are NOT deep cloning fields or selector here, which should be OK. + var description = new CursorDescription( + self._cursorDescription.collectionName, + self._cursorDescription.selector, + options); + return new Cursor(self._mongoHandle, description); + }); }, @@ -749,52 +790,54 @@ _.extend(OplogObserveDriver.prototype, { // Replace self._unpublishedBuffer with newBuffer. // // XXX This is very similar to LocalCollection._diffQueryUnorderedChanges. We - // should really: (a) Unify IdMap and OrderedDict into Unordered/OrderedDict (b) - // Rewrite diff.js to use these classes instead of arrays and objects. + // should really: (a) Unify IdMap and OrderedDict into Unordered/OrderedDict + // (b) Rewrite diff.js to use these classes instead of arrays and objects. _publishNewResults: function (newResults, newBuffer) { var self = this; + Meteor._noYieldsAllowed(function () { - // If the query is limited and there is a buffer, shut down so it doesn't - // stay in a way. - if (self._limit) { - self._unpublishedBuffer.clear(); - } + // If the query is limited and there is a buffer, shut down so it doesn't + // stay in a way. + if (self._limit) { + self._unpublishedBuffer.clear(); + } - // First remove anything that's gone. Be careful not to modify - // self._published while iterating over it. - var idsToRemove = []; - self._published.forEach(function (doc, id) { - if (!newResults.has(id)) - idsToRemove.push(id); + // First remove anything that's gone. Be careful not to modify + // self._published while iterating over it. + var idsToRemove = []; + self._published.forEach(function (doc, id) { + if (!newResults.has(id)) + idsToRemove.push(id); + }); + _.each(idsToRemove, function (id) { + self._removePublished(id); + }); + + // Now do adds and changes. + // If self has a buffer and limit, the new fetched result will be + // limited correctly as the query has sort specifier. + newResults.forEach(function (doc, id) { + self._handleDoc(id, doc); + }); + + // Sanity-check that everything we tried to put into _published ended up + // there. + // XXX if this is slow, remove it later + if (self._published.size() !== newResults.size()) { + throw Error("failed to copy newResults into _published!"); + } + self._published.forEach(function (doc, id) { + if (!newResults.has(id)) + throw Error("_published has a doc that newResults doesn't; " + id); + }); + + // Finally, replace the buffer + newBuffer.forEach(function (doc, id) { + self._addBuffered(id, doc); + }); + + self._safeAppendToBuffer = newBuffer.size() < self._limit; }); - _.each(idsToRemove, function (id) { - self._removePublished(id); - }); - - // Now do adds and changes. - // If self has a buffer and limit, the new fetched result will be - // limited correctly as the query has sort specifier. - newResults.forEach(function (doc, id) { - self._handleDoc(id, doc); - }); - - // Sanity-check that everything we tried to put into _published ended up - // there. - // XXX if this is slow, remove it later - if (self._published.size() !== newResults.size()) { - throw Error("failed to copy newResults into _published!"); - } - self._published.forEach(function (doc, id) { - if (!newResults.has(id)) - throw Error("_published has a doc that newResults doesn't; " + id); - }); - - // Finally, replace the buffer - newBuffer.forEach(function (doc, id) { - self._addBuffered(id, doc); - }); - - self._safeAppendToBuffer = newBuffer.size() < self._limit; }, // This stop function is invoked from the onStop of the ObserveMultiplexer, so @@ -818,7 +861,7 @@ _.extend(OplogObserveDriver.prototype, { // to get flushed (and it's probably not valid to call methods on the // dying multiplexer). _.each(self._writesToCommitWhenWeReachSteady, function (w) { - w.committed(); + w.committed(); // maybe yields? }); self._writesToCommitWhenWeReachSteady = null; @@ -836,16 +879,18 @@ _.extend(OplogObserveDriver.prototype, { _registerPhaseChange: function (phase) { var self = this; - var now = new Date; + Meteor._noYieldsAllowed(function () { + var now = new Date; - if (self._phase) { - var timeDiff = now - self._phaseStartTime; - Package.facts && Package.facts.Facts.incrementServerFact( - "mongo-livedata", "time-spent-in-" + self._phase + "-phase", timeDiff); - } + if (self._phase) { + var timeDiff = now - self._phaseStartTime; + Package.facts && Package.facts.Facts.incrementServerFact( + "mongo-livedata", "time-spent-in-" + self._phase + "-phase", timeDiff); + } - self._phase = phase; - self._phaseStartTime = now; + self._phase = phase; + self._phaseStartTime = now; + }); } }); From 6efb7c85ccff82ddd54a9b67eacef74540906937 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Tue, 29 Jul 2014 14:34:19 -0700 Subject: [PATCH 059/126] Update docs and examples to 0.8.3 --- docs/.meteor/release | 2 +- docs/lib/release-override.js | 2 +- examples/clock/.meteor/release | 2 +- examples/leaderboard/.meteor/release | 2 +- examples/parties/.meteor/release | 2 +- examples/todos/.meteor/release | 2 +- examples/wordplay/.meteor/release | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/.meteor/release b/docs/.meteor/release index 100435be13..ee94dd834b 100644 --- a/docs/.meteor/release +++ b/docs/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 diff --git a/docs/lib/release-override.js b/docs/lib/release-override.js index d0fc420710..6bfbd0e614 100644 --- a/docs/lib/release-override.js +++ b/docs/lib/release-override.js @@ -1,5 +1,5 @@ // While galaxy apps are on their own special meteor releases, override // Meteor.release here. if (Meteor.isClient) { - Meteor.release = Meteor.release ? "0.8.2" : undefined; + Meteor.release = Meteor.release ? "0.8.3" : undefined; } diff --git a/examples/clock/.meteor/release b/examples/clock/.meteor/release index 100435be13..ee94dd834b 100644 --- a/examples/clock/.meteor/release +++ b/examples/clock/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 diff --git a/examples/leaderboard/.meteor/release b/examples/leaderboard/.meteor/release index 100435be13..ee94dd834b 100644 --- a/examples/leaderboard/.meteor/release +++ b/examples/leaderboard/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 diff --git a/examples/parties/.meteor/release b/examples/parties/.meteor/release index 100435be13..ee94dd834b 100644 --- a/examples/parties/.meteor/release +++ b/examples/parties/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 diff --git a/examples/todos/.meteor/release b/examples/todos/.meteor/release index 100435be13..ee94dd834b 100644 --- a/examples/todos/.meteor/release +++ b/examples/todos/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 diff --git a/examples/wordplay/.meteor/release b/examples/wordplay/.meteor/release index 100435be13..ee94dd834b 100644 --- a/examples/wordplay/.meteor/release +++ b/examples/wordplay/.meteor/release @@ -1 +1 @@ -0.8.2 +0.8.3 From 30a397cde5383c1bd9b7c259cf4de2695cafd8f6 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 17:19:31 -0700 Subject: [PATCH 060/126] Rename ServerCatalog -> OfficialCatalog Since it it used for catalog.official. --- tools/catalog.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tools/catalog.js b/tools/catalog.js index 6236549bd9..c2cab9ae25 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -22,13 +22,13 @@ var Future = require('fibers/future'); var catalog = exports; ///////////////////////////////////////////////////////////////////////////////////// -// Server Catalog +// Official Catalog ///////////////////////////////////////////////////////////////////////////////////// -// The serverlog syncs up with the server. It doesn't care about local -// packages. When the user wants information about the state of the package -// world (ex: search), we should use this catalog first. -var ServerCatalog = function () { +// The official catalog syncs up with the package server. It doesn't care about +// local packages. When the user wants information about the state of the +// package world (ex: search), we should use this catalog first. +var OfficialCatalog = function () { var self = this; // Set this to true if we are not going to connect to the remote package @@ -41,9 +41,9 @@ var ServerCatalog = function () { BaseCatalog.call(self); }; -util.inherits(ServerCatalog, BaseCatalog); +util.inherits(OfficialCatalog, BaseCatalog); -_.extend(ServerCatalog.prototype, { +_.extend(OfficialCatalog.prototype, { initialize : function (options) { var self = this; options = options || {}; @@ -869,7 +869,7 @@ _.extend(CompleteCatalog.prototype, { // This is the catalog that's used to answer the specific question of "so what's // on the server?". It does not contain any local catalogs. Typically, we call // catalog.official.refresh() to update data.json. -catalog.official = new ServerCatalog(); +catalog.official = new OfficialCatalog(); // This is the catalog that's used to actually drive the constraint solver: it // contains local packages, and since local packages always beat server From c6ad06fe7781b5dc4e0f405a60c0cf5ac91c3ac5 Mon Sep 17 00:00:00 2001 From: Matthew Arbesfeld Date: Tue, 29 Jul 2014 18:42:25 -0700 Subject: [PATCH 061/126] Fix for run --once test. Actually include programs in star.json --- tools/bundler.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/bundler.js b/tools/bundler.js index e9a17c40da..952622d16d 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1550,9 +1550,6 @@ var writeSiteArchive = function (targets, outputPath, options) { 'utf8')}); } - // Control file - builder.writeJson('star.json', json); - // Merge the WatchSet of everything that went into the bundle. var clientWatchSet = new watch.WatchSet(); var serverWatchSet = new watch.WatchSet(); @@ -1575,6 +1572,9 @@ var writeSiteArchive = function (targets, outputPath, options) { })); }); + // Control file + builder.writeJson('star.json', json); + // We did it! builder.complete(); From 950a3d9c1b3b43d4b2eae9021a3454ac23d1bb78 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 17:26:07 -0700 Subject: [PATCH 062/126] Use buildmessage in publish-for-arch Also, drop a unipkg.saveToPath that we think is redundant (since 5e72f55eb2d changed bundleBuildResult to also call saveToPath) --- tools/commands-packages.js | 50 +++++++++++++++++++++++++++----------- tools/package-client.js | 2 ++ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 10e580a98d..235ee50040 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -246,21 +246,34 @@ main.registerCommand({ return 1; } - var packageSource = new PackageSource; + var unipkg; + var messages = buildmessage.capture({ + title: "building package " + name + }, function () { + var packageSource = new PackageSource; - // This package source, although it is initialized from a directory is - // immutable. It should be built exactly as is. If we need to modify anything, - // such as the version lock file, something has gone terribly wrong and we - // should throw. - packageSource.initFromPackageDir(name, packageDir, { - requireVersion: true, - immutable: true + // This package source, although it is initialized from a directory is + // immutable. It should be built exactly as is. If we need to modify + // anything, such as the version lock file, something has gone terribly + // wrong and we should throw. + packageSource.initFromPackageDir(name, packageDir, { + requireVersion: true, + immutable: true + }); + if (buildmessage.jobHasMessages()) + return; + + unipkg = compiler.compile(packageSource, { + officialBuild: true + }).unipackage; + if (buildmessage.jobHasMessages()) + return; }); - var unipkg = compiler.compile(packageSource, { - officialBuild: true - }).unipackage; - unipkg.saveToPath(path.join(packageDir, '.build.' + packageSource.name)); + if (messages.hasMessages()) { + process.stderr.write("\n" + messages.formatMessages()); + return 1; + } var conn; try { @@ -269,10 +282,19 @@ main.registerCommand({ packageClient.handlePackageServerConnectionError(err); return 1; } - packageClient.createAndPublishBuiltPackage(conn, unipkg); + messages = buildmessage.capture({ + title: "publishing package " + name + }, function () { + packageClient.createAndPublishBuiltPackage(conn, unipkg); + }); - catalog.official.refresh(); + if (messages.hasMessages()) { + process.stderr.write("\n" + messages.formatMessages()); + return 1; + } + + catalog.official.refresh(); // XXX buildmessage.capture? return 0; }); diff --git a/tools/package-client.js b/tools/package-client.js index 86b82cf8b3..4f7e27ba0d 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -370,6 +370,8 @@ var createAndPublishBuiltPackage = function (conn, unipackage) { }); var bundleResult = bundleBuild(unipackage); + if (buildmessage.jobHasMessages()) + return; process.stdout.write('Uploading build...\n'); uploadTarball(uploadInfo.uploadUrl, From 81e4b51cf69113695d4f2b602340cd257eb34f92 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 18:27:27 -0700 Subject: [PATCH 063/126] Put --get-ready's PackageLoader.getPackage in job --- tools/commands.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tools/commands.js b/tools/commands.js index 5cc58ced94..8210f8b82d 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -171,8 +171,9 @@ main.registerCommand({ }); }; - - var messages = buildmessage.capture(function () { + var messages = buildmessage.capture({ + title: 'getting packages ready' + }, function () { // First, build all accessible *local* packages, whether or not this app // uses them. Use the "all packages are local" loader. loadPackages(catalog.complete.getLocalPackageNames(), From 70c0ff2c98f6b1b37201104aa9dcb735a327d423 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 18:52:46 -0700 Subject: [PATCH 064/126] put calls to publishPackage in buildmessage --- tools/commands-packages.js | 50 ++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 16 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 235ee50040..2e5986df3f 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -161,11 +161,20 @@ main.registerCommand({ } // We have initialized everything, so perform the publish oepration. - var ec = packageClient.publishPackage( - packageSource, compileResult, conn, { - new: options.create, - existingVersion: options['existing-version'] - }); + var ec; // XXX maybe combine with messages? + messages = buildmessage.capture({ + title: "publishing the package" + }, function () { + ec = packageClient.publishPackage( + packageSource, compileResult, conn, { + new: options.create, + existingVersion: options['existing-version'] + }); + }); + if (messages.hasMessages()) { + process.stdout.write(messages.formatMessages()); + return ec || 1; + } // Warn the user if their package is not good for all architectures. if (compileResult.unipackage.buildArchitectures() !== "browser+os") { @@ -638,20 +647,29 @@ main.registerCommand({ }; process.stdout.write("Publishing package: " + name + "\n"); - // If we are creating a new package, dsPS will document this for us, so we - // don't need to do this here. Though, in the future, once we are done - // bootstrapping package servers, we should consider having some extra - // checks around this. - var pub = packageClient.publishPackage( - prebuilt.source, - prebuilt.compileResult, - conn, - opts); + var pubEC; // XXX merge with messages? + messages = buildmessage.capture({ + title: "publishing package " + name + }, function () { + // If we are creating a new package, dsPS will document this for us, so + // we don't need to do this here. Though, in the future, once we are + // done bootstrapping package servers, we should consider having some + // extra checks around this. + pubEC = packageClient.publishPackage( + prebuilt.source, + prebuilt.compileResult, + conn, + opts); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return pubEC || 1; + } // If we fail to publish, just exit outright, something has gone wrong. - if (pub > 0) { + if (pubEC > 0) { process.stderr.write("Failed to publish: " + name + "\n"); - return 1; + return pubEC; } } From 43b04a12b2c614297aaafe13912ad3cd967b799d Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 29 Jul 2014 18:57:17 -0700 Subject: [PATCH 065/126] add some buildmessage.assertInJob to PackageClient --- tools/buildmessage.js | 14 +++++++++++++- tools/package-client.js | 6 ++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/tools/buildmessage.js b/tools/buildmessage.js index f7840c4dbd..26eeb99ead 100644 --- a/tools/buildmessage.js +++ b/tools/buildmessage.js @@ -365,6 +365,16 @@ var exception = function (error) { } }; +var assertInJob = function () { + if (! currentJob) + throw new Error("Expected to be in a buildmessage job"); +}; + +var assertInCapture = function () { + if (! currentMessageSet) + throw new Error("Expected to be in a buildmessage capture"); +}; + var buildmessage = exports; _.extend(exports, { capture: capture, @@ -372,5 +382,7 @@ _.extend(exports, { markBoundary: markBoundary, error: error, exception: exception, - jobHasMessages: jobHasMessages + jobHasMessages: jobHasMessages, + assertInJob: assertInJob, + assertInCapture: assertInCapture }); diff --git a/tools/package-client.js b/tools/package-client.js index 4f7e27ba0d..db17759ba5 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -334,6 +334,8 @@ var uploadTarball = function (putUrl, tarball) { exports.uploadTarball = uploadTarball; var bundleBuild = function (unipackage) { + buildmessage.assertInJob(); + var tempDir = files.mkdtemp('build-package-'); var packageTarName = unipackage.tarballName(); var tarInputDir = path.join(tempDir, packageTarName); @@ -362,6 +364,8 @@ var bundleBuild = function (unipackage) { exports.bundleBuild = bundleBuild; var createAndPublishBuiltPackage = function (conn, unipackage) { + buildmessage.assertInJob(); + process.stdout.write('Creating package build...\n'); var uploadInfo = conn.call('createPackageBuild', { packageName: unipackage.name, @@ -427,6 +431,8 @@ exports.handlePackageServerConnectionError = function (error) { // // Return true on success and an error code otherwise. exports.publishPackage = function (packageSource, compileResult, conn, options) { + buildmessage.assertInJob(); + options = options || {}; if (options.new && options.existingVersion) From d316eacab2c1925b753fadc0164cfe2c95963394 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 29 Jul 2014 21:19:43 -0700 Subject: [PATCH 066/126] check if the package already exists and return a different error code --- tools/commands-packages.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 2e5986df3f..49973b7873 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -113,7 +113,17 @@ main.registerCommand({ // Refresh the catalog, caching the remote package data on the server. We can // optimize the workflow by using this data to weed out obviously incorrect // submissions before they ever hit the wire. - catalog.official.refresh(true); + catalog.official.refresh(); + var packageName = path.basename(options.packageDir); + + // Fail early if the package already exists. + if (options.create) { + if (catalog.official.getPackage(packageName)) { + process.stderr.write("Package already exists. To create a new version of an existing "+ + "package, do not use the --create flag! \n"); + return 2; + } + }; try { var conn = packageClient.loggedInPackagesConnection(); @@ -134,7 +144,6 @@ main.registerCommand({ var messages = buildmessage.capture( { title: "building the package" }, function () { - var packageName = path.basename(options.packageDir); if (! utils.validPackageName(packageName)) { buildmessage.error("Invalid package name:", packageName); From 0902f7e39a137027c5ad744331e8acbed04d94dd Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 30 Jul 2014 08:07:12 -0700 Subject: [PATCH 067/126] make sure errors go to stderr: cursory pass --- tools/commands-packages.js | 8 ++++---- tools/commands.js | 12 ++++++------ tools/package-client.js | 21 +++++++++++---------- tools/uniload.js | 4 ++-- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 49973b7873..80fcff3080 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -165,7 +165,7 @@ main.registerCommand({ }); if (messages.hasMessages()) { - process.stdout.write(messages.formatMessages()); + process.stderr.write(messages.formatMessages()); return 1; } @@ -181,7 +181,7 @@ main.registerCommand({ }); }); if (messages.hasMessages()) { - process.stdout.write(messages.formatMessages()); + process.stderr.write(messages.formatMessages()); return ec || 1; } @@ -640,7 +640,7 @@ main.registerCommand({ }); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; }; @@ -939,7 +939,7 @@ main.registerCommand({ }); }); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; } diff --git a/tools/commands.js b/tools/commands.js index 8210f8b82d..d05b10cd9e 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -200,7 +200,7 @@ main.registerCommand({ }); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; }; @@ -546,8 +546,8 @@ main.registerCommand({ } }); if (bundleResult.errors) { - process.stdout.write("Errors prevented bundling:\n"); - process.stdout.write(bundleResult.errors.formatMessages()); + process.stderr.write("Errors prevented bundling:\n"); + process.stderr.write(bundleResult.errors.formatMessages()); return 1; } @@ -1015,7 +1015,7 @@ main.registerCommand({ }); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; } } @@ -1134,7 +1134,7 @@ main.registerCommand({ if (count) console.log("Built " + count + " packages."); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; } }); @@ -1260,7 +1260,7 @@ main.registerCommand({ }); if (messages.hasMessages()) { - process.stdout.write("\n" + messages.formatMessages()); + process.stderr.write("\n" + messages.formatMessages()); return 1; }; diff --git a/tools/package-client.js b/tools/package-client.js index db17759ba5..5dd061dbf8 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -62,12 +62,11 @@ exports.loadCachedServerData = function (packageStorageFile) { var data = fs.readFileSync(config.getPackageStorage(), 'utf8'); } catch (e) { if (e.code == 'ENOENT') { -// process.stderr.write("No cached server data found on disk.\n"); return noDataToken; } // XXX we should probably return an error to the caller here to // figure out how to handle it - console.log(e.message); + process.stderr.write("ERROR " + e.message + "\n"); process.exit(1); } var ret = noDataToken; @@ -75,7 +74,10 @@ exports.loadCachedServerData = function (packageStorageFile) { ret = JSON.parse(data); } catch (err) { // XXX error handling - console.log("Could not parse JSON in data.json."); + process.stderr.write( + "ERROR: Could not parse JSON for local package-metadata cache. \n"); + // This should only happen if you decided to manually edit this or + // whatever. Regardless, go on and treat this as an empty file. } return ret; }; @@ -195,7 +197,7 @@ exports.updateServerPackageData = function (cachedServerData, _optionsForTest) { useShortPages: _optionsForTest.useShortPages }); } catch (err) { - console.log(err); + process.stderr.write("ERROR " + err.message + "\n"); if (err instanceof ServiceConnection.ConnectionTimeoutError) { return null; } else { @@ -388,7 +390,7 @@ var createAndPublishBuiltPackage = function (conn, unipackage) { bundleResult.tarballHash, bundleResult.treeHash); } catch (err) { - console.log(err); + process.stderr.write("ERROR " + err.message + "\n"); return; } @@ -542,7 +544,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) }); if (messages.hasMessages()) { - process.stdout.write(messages.formatMessages()); + process.stderr.write(messages.formatMessages()); return 1; } @@ -572,7 +574,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) name: packageSource.name }); } catch (err) { - console.log(err.message); + process.stderr.write(err.message + "\n"); return 1; } @@ -606,8 +608,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) try { var uploadInfo = conn.call('createPackageVersion', uploadRec); } catch (err) { - console.log("ERROR:", err.message); - console.log("Package could not be published."); + process.stderr.write("ERROR " + err.message + "\n"); return 1; } @@ -625,7 +626,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) { tarballHash: sourceBundleResult.tarballHash, treeHash: sourceBundleResult.treeHash }); } catch (err) { - console.log(err.message); + process.stderr.write("ERROR " + err.message + "\n"); return 1; } diff --git a/tools/uniload.js b/tools/uniload.js index 0974d6cda3..c960bff54c 100644 --- a/tools/uniload.js +++ b/tools/uniload.js @@ -107,8 +107,8 @@ var load = function (options) { // happen in a built release. In the future, the command line // tool will be a normal Meteor app and will be built ahead of // time like any other app and this case will disappear. - process.stdout.write("Errors prevented unipackage load:\n"); - process.stdout.write(messages.formatMessages()); + process.stderr.write("Errors prevented unipackage load:\n"); + process.stderr.write(messages.formatMessages()); throw new Error("unipackage load failed?"); } From 33eee2c25acc353d5ca6922e56c782d4cee2a9cc Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Wed, 30 Jul 2014 09:37:48 -0700 Subject: [PATCH 068/126] history tweak --- History.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/History.md b/History.md index 6b4ee782f4..7769472360 100644 --- a/History.md +++ b/History.md @@ -1,4 +1,4 @@ -## v.NEXT.NEXT +## v.NEXT * The `appcache` package now defaults to functioning on all browsers that support the AppCache API, rather than a whitelist of browsers. You can still @@ -7,10 +7,6 @@ Firefox no longer makes a confusing popup. #2241 - -## v.NEXT - - ## v0.8.3 #### Blaze From dd82a8bcd55af2f1fda8b6b9c67f757ff180ad4f Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Wed, 30 Jul 2014 09:38:30 -0700 Subject: [PATCH 069/126] Add node upgrade to History --- History.md | 1 + 1 file changed, 1 insertion(+) diff --git a/History.md b/History.md index 7769472360..6811c731ed 100644 --- a/History.md +++ b/History.md @@ -82,6 +82,7 @@ callback provided. * Upgraded dependencies: + - node: 0.10.29 (from 0.10.28) - less: 1.7.1 (from 1.6.1) Patches contributed by GitHub users Cangit, cmather, duckspeaker, zol. From 47831e8744d2a5cc9d3def613d72bc5b9ed42822 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 12:00:51 -0700 Subject: [PATCH 070/126] Expose match failures in server logs --- packages/livedata/livedata_server.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index 0ff5dd81f3..11d1114bba 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1456,6 +1456,11 @@ var wrapInternalException = function (exception, context) { if (!exception || exception instanceof Meteor.Error) return exception; + // tests can set the 'expected' flag on an exception so it won't go to the + // server log + if (!exception.expected) + Meteor._debug("Exception " + context, exception.stack); + // Did the error contain more details that could have been useful if caught in // server code (or if thrown from non-client-originated code), but also // provided a "sanitized" version with more context than 500 Internal server @@ -1467,11 +1472,6 @@ var wrapInternalException = function (exception, context) { "is not a Meteor.Error; ignoring"); } - // tests can set the 'expected' flag on an exception so it won't go to the - // server log - if (!exception.expected) - Meteor._debug("Exception " + context, exception.stack); - return new Meteor.Error(500, "Internal server error"); }; From c3508acc07a60c3fb07770cb608792d0766c53d0 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 29 Jul 2014 09:44:12 -0700 Subject: [PATCH 071/126] Tweak benchmark to log each operation --- examples/unfinished/benchmark/benchmark.js | 25 +++++++++++++++---- .../benchmark/scenarios/scale10.json | 11 ++++++++ .../benchmark/scenarios/scale100.json | 11 ++++++++ .../benchmark/scenarios/scale50.json | 11 ++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) create mode 100644 examples/unfinished/benchmark/scenarios/scale10.json create mode 100644 examples/unfinished/benchmark/scenarios/scale100.json create mode 100644 examples/unfinished/benchmark/scenarios/scale50.json diff --git a/examples/unfinished/benchmark/benchmark.js b/examples/unfinished/benchmark/benchmark.js index abbbde6ad8..1a206ea094 100644 --- a/examples/unfinished/benchmark/benchmark.js +++ b/examples/unfinished/benchmark/benchmark.js @@ -35,6 +35,16 @@ var randomString = function (length) { return ret; }; +var preCall = function (name) { + console.log('> ' + name); +}; + +var postCall = function (name) { + return function (err, callback) { + console.log('< ' + name + ' ' + (err ? 'ERR' : 'OK')); + }; +}; + var pickCollection = function () { return Random.choice(Collections); }; @@ -96,7 +106,8 @@ if (Meteor.isServer) { Meteor.setInterval(function () { var when = +(new Date) - PARAMS.maxAgeSeconds*1000; _.each(Collections, function (C) { - C.remove({when: {$lt: when}}); + preCall('removeMaxAge'); + C.remove({when: {$lt: when}}, postCall('removeMaxAge')); }); // Clear out 5% of the DB each time, steady state. XXX parameterize? }, 1000*PARAMS.maxAgeSeconds / 20); @@ -121,7 +132,8 @@ if (Meteor.isServer) { doc.when = +(new Date); var C = pickCollection(); - C.insert(doc); + preCall('insert'); + C.insert(doc, postCall('insert')); }, update: function (processId, field, value) { check([processId, field, value], [String]); @@ -130,15 +142,18 @@ if (Meteor.isServer) { var C = pickCollection(); // update one message. - C.update({fromProcess: processId}, {$set: modifer}, {multi: false}); + preCall('update'); + C.update({fromProcess: processId}, {$set: modifer}, {multi: false}, postCall('update')); }, remove: function (processId) { check(processId, String); var C = pickCollection(); // remove one message. var obj = C.findOne({fromProcess: processId}); - if (obj) - C.remove(obj._id); + if (obj) { + preCall('remove'); + C.remove(obj._id, postCall('remove')); + } } }); diff --git a/examples/unfinished/benchmark/scenarios/scale10.json b/examples/unfinished/benchmark/scenarios/scale10.json new file mode 100644 index 0000000000..5962540e04 --- /dev/null +++ b/examples/unfinished/benchmark/scenarios/scale10.json @@ -0,0 +1,11 @@ + { + "params": { + "numCollections": 1, + "maxAgeSeconds": 60, + "insertsPerSecond": 10, + "updatesPerSecond": 10, + "removesPerSecond": 1, + "documentSize": 128, + "documentNumFields": 2 + } +} diff --git a/examples/unfinished/benchmark/scenarios/scale100.json b/examples/unfinished/benchmark/scenarios/scale100.json new file mode 100644 index 0000000000..ec0f4853e9 --- /dev/null +++ b/examples/unfinished/benchmark/scenarios/scale100.json @@ -0,0 +1,11 @@ + { + "params": { + "numCollections": 1, + "maxAgeSeconds": 60, + "insertsPerSecond": 100, + "updatesPerSecond": 100, + "removesPerSecond": 10, + "documentSize": 128, + "documentNumFields": 2 + } +} diff --git a/examples/unfinished/benchmark/scenarios/scale50.json b/examples/unfinished/benchmark/scenarios/scale50.json new file mode 100644 index 0000000000..0e4df53c12 --- /dev/null +++ b/examples/unfinished/benchmark/scenarios/scale50.json @@ -0,0 +1,11 @@ + { + "params": { + "numCollections": 1, + "maxAgeSeconds": 60, + "insertsPerSecond": 50, + "updatesPerSecond": 50, + "removesPerSecond": 5, + "documentSize": 128, + "documentNumFields": 2 + } +} From 4e6efe91a5fa390da56708e11af34b54f7ce784d Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 29 Jul 2014 09:44:50 -0700 Subject: [PATCH 072/126] Output test results in xunit format when the path is 'xunit' We output via console.log; to differentiate from normal output, we prefix xunit output with 'XUNIT '. --- packages/test-in-console/driver.js | 72 +++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/packages/test-in-console/driver.js b/packages/test-in-console/driver.js index 50e03156f3..35208a0b74 100644 --- a/packages/test-in-console/driver.js +++ b/packages/test-in-console/driver.js @@ -8,18 +8,43 @@ TEST_STATUS = { FAILURES: null }; +// xUnit format uses XML output +var XML_CHAR_MAP = { + '<': '<', + '>': '>', + '&': '&', + '"': '"', + "'": ''' +}; +// Escapes a string for insertion into XML +var escapeXml = function (s) { + return s.replace(/[<>&"']/g, function (c) { + return XML_CHAR_MAP[c]; + }); +} + +// Returns a human name for a test var getName = function (result) { return (result.server ? "S: " : "C: ") + result.groupPath.join(" - ") + " - " + result.test; }; +// Calls console.log, but returns silently if console.log is not available var log = function (/*arguments*/) { if (typeof console !== 'undefined') { console.log.apply(console, arguments); } }; +// Logs xUnit output, if xunit output is enabled +// Output is sent to console.log, prefixed with a magic string 'XUNIT ' +// By grepping for that prefix, the xUnit output can be extracted +var xunit = function (s) { + if (xunitEnabled) { + log('XUNIT ' + s); + } +}; var passed = 0; var failed = 0; @@ -31,6 +56,10 @@ var hrefPath = document.location.href.split("/"); var platform = decodeURIComponent(hrefPath.length && hrefPath[hrefPath.length - 1]); if (!platform) platform = "local"; + +// We enable xUnit output when platform is xunit +var xunitEnabled = (platform == 'xunit'); + var doReport = Meteor && Meteor.settings && Meteor.settings.public && @@ -82,10 +111,13 @@ Meteor.startup(function () { status: "PENDING", events: [], server: !!results.server, - testPath: testPath + testPath: testPath, + test: results.test }; report(name, false); } + // Loop through events, and record status for each test + // Also log result if test has finished _.each(results.events, function (event) { resultSet[name].events.push(event); switch (event.type) { @@ -136,6 +168,7 @@ Meteor.startup(function () { }); }, + // After test completion, log a quick summary function () { if (failed > 0) { log("~~~~~~~ THERE ARE FAILURES ~~~~~~~"); @@ -153,6 +186,43 @@ Meteor.startup(function () { TEST_STATUS.DONE = DONE = true; } }); + + // Also log xUnit output + xunit(''); + _.each(resultSet, function (result, name) { + var classname = result.testPath.join('.').replace(/ /g, '-') + (result.server ? "-server" : "-client"); + var name = result.test.replace(/ /g, '-') + (result.server ? "-server" : "-client"); + var time = ""; + var error = ""; + _.each(result.events, function (event) { + switch (event.type) { + case "finish": + var timeMs = event.timeMs; + if (timeMs !== undefined) { + time = (timeMs / 1000) + ""; + } + break; + case "fail": + var details = event.details || {}; + error = (details.message || '?') + " filename=" + (details.filename || '?') + " line=" + (details.line || '?'); + } + }); + switch (event.status) { + case "FAIL": + error = error || '?'; + break; + case "EXPECTED": + error = "Expected failure"; + break; + } + + xunit(''); + if (error) { + xunit(' ' + escapeXml(error) + ''); + } + xunit(''); + }); + xunit(''); }, ["tinytest"]); }); From 3163112235b3d294bca151b2477ceab71a41f402 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 29 Jul 2014 09:47:11 -0700 Subject: [PATCH 073/126] Get S3 credentials from env variables, if set, in publish-release.js --- .../admin/publish-release/server/publish-release.js | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/scripts/admin/publish-release/server/publish-release.js b/scripts/admin/publish-release/server/publish-release.js index 75caa7c2cf..d33885113d 100644 --- a/scripts/admin/publish-release/server/publish-release.js +++ b/scripts/admin/publish-release/server/publish-release.js @@ -59,7 +59,15 @@ var configureS3 = function () { return {accessKey: accessKey, secretKey: secretKey}; }; - var s3Credentials = getS3Credentials(); + var s3Credentials; + if (process.env.AWS_ACCESS_KEY_ID) { + s3Credentials = {}; + s3Credentials.accessKey = process.env.AWS_ACCESS_KEY_ID; + s3Credentials.secretKey = process.env.AWS_SECRET_ACCESS_KEY; + } else { + s3Credentials = getS3Credentials(); + } + var s3 = new S3({ accessKeyId: s3Credentials.accessKey, secretAccessKey: s3Credentials.secretKey, From 706fc7ebe77afc60a5b5967f753c4f16761b1a39 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 29 Jul 2014 09:49:03 -0700 Subject: [PATCH 074/126] Allow TIMEOUT_SCALE_FACTOR env variable to scale up timeouts for self-test Particularly for automated tests, where we may run on a slow machine, we need an escape valve to let us boost the timeouts. This also allows for a 'tolerant' test (scale factor 2+?) and a 'strict' test (scale factor 0.5?) etc --- tools/selftest.js | 4 ++++ tools/test-utils.js | 2 +- tools/utils.js | 7 +++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tools/selftest.js b/tools/selftest.js index 0d84ac73e8..52099a8633 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -698,6 +698,7 @@ _.extend(Run.prototype, { self._ensureStarted(); var timeout = self.baseTimeout + self.extraTime; + timeout *= utils.timeoutScaleFactor; self.extraTime = 0; return self.stdoutMatcher.match(pattern, timeout, _strict); }), @@ -708,6 +709,7 @@ _.extend(Run.prototype, { self._ensureStarted(); var timeout = self.baseTimeout + self.extraTime; + timeout *= utils.timeoutScaleFactor; self.extraTime = 0; return self.stderrMatcher.match(pattern, timeout, _strict); }), @@ -753,6 +755,7 @@ _.extend(Run.prototype, { self._ensureStarted(); var timeout = self.baseTimeout + self.extraTime; + timeout *= utils.timeoutScaleFactor; self.extraTime = 0; self.expectExit(); @@ -770,6 +773,7 @@ _.extend(Run.prototype, { if (self.exitStatus === undefined) { var timeout = self.baseTimeout + self.extraTime; + timeout *= utils.timeoutScaleFactor; self.extraTime = 0; var fut = new Future; diff --git a/tools/test-utils.js b/tools/test-utils.js index c9840ff65b..80858e2520 100644 --- a/tools/test-utils.js +++ b/tools/test-utils.js @@ -12,7 +12,7 @@ var randomString = function (charsCount) { return str; }; -exports.accountsCommandTimeoutSecs = 15; +exports.accountsCommandTimeoutSecs = 15 * exports.timeoutScaleFactor; exports.randomString = randomString; diff --git a/tools/utils.js b/tools/utils.js index 80ae7ab6e2..1fe6f54569 100644 --- a/tools/utils.js +++ b/tools/utils.js @@ -151,3 +151,10 @@ exports.validEmail = function (address) { exports.quotemeta = function (str) { return String(str).replace(/(\W)/g, '\\$1'); }; + +// Allow a simple way to scale up all timeouts from the command line +var timeoutScaleFactor = 1.0; +if (process.env.TIMEOUT_SCALE_FACTOR) { + timeoutScaleFactor = parseFloat(process.env.TIMEOUT_SCALE_FACTOR); +} +exports.timeoutScaleFactor = timeoutScaleFactor; From 812a3af2cd0b44cb5f4e9192ead1b7dd0c252208 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Tue, 29 Jul 2014 09:50:15 -0700 Subject: [PATCH 075/126] By default, don't run server & client tests in parallel --- packages/tinytest/tinytest_client.js | 31 +++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/packages/tinytest/tinytest_client.js b/packages/tinytest/tinytest_client.js index f1c1f677d2..6749c172e2 100644 --- a/packages/tinytest/tinytest_client.js +++ b/packages/tinytest/tinytest_client.js @@ -2,23 +2,40 @@ // the server. Sets a 'server' flag on test results that came from the // server. // -Tinytest._runTestsEverywhere = function (onReport, onComplete, pathPrefix) { +// Options: +// serial if true, will not run tests in parallel. Currently this means +// running the server tests before running the client tests. +// Default is currently true (serial operation), but we will likely +// change this to false in future. +Tinytest._runTestsEverywhere = function (onReport, onComplete, pathPrefix, options) { var runId = Random.id(); var localComplete = false; + var localStarted = false; var remoteComplete = false; var done = false; + options = _.extend({ + serial: true + }, options); + var serial = !!options.serial; + var maybeDone = function () { if (!done && localComplete && remoteComplete) { done = true; onComplete && onComplete(); } + if (serial && remoteComplete && !localStarted) { + startLocalTests(); + } }; - Tinytest._runTests(onReport, function () { - localComplete = true; - maybeDone(); - }, pathPrefix); + var startLocalTests = function() { + localStarted = true; + Tinytest._runTests(onReport, function () { + localComplete = true; + maybeDone(); + }, pathPrefix); + }; var handle; @@ -59,4 +76,8 @@ Tinytest._runTestsEverywhere = function (onReport, onComplete, pathPrefix) { // XXX better report error throw new Error("Test server returned an error"); }); + + if (!serial) { + startLocalTests(); + } }; From f2184d3dbe47a3be0e7edc09df73ddaeef05da81 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Wed, 30 Jul 2014 11:55:10 -0700 Subject: [PATCH 076/126] Tweak examples/unfinished/benchmark script --- examples/unfinished/benchmark/run-local.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/unfinished/benchmark/run-local.sh b/examples/unfinished/benchmark/run-local.sh index 3c533ad54a..9fecfcaba9 100755 --- a/examples/unfinished/benchmark/run-local.sh +++ b/examples/unfinished/benchmark/run-local.sh @@ -1,8 +1,12 @@ #!/bin/bash PORT=9000 -NUM_CLIENTS=10 -DURATION=120 +if [ -z "$NUM_CLIENTS" ]; then + NUM_CLIENTS=10 +fi +if [ -z "$DURATION" ]; then + DURATION=120 +fi REPORT_INTERVAL=10 set -e @@ -20,7 +24,7 @@ pkill -f "$PROJDIR/.meteor/local/db" || true ../../../meteor reset || true # start the benchmark app -../../../meteor --production --settings "scenarios/${SCENARIO}.json" --port 9000 & +../../../meteor --production --settings "scenarios/${SCENARIO}.json" --port ${PORT} & OUTER_PID=$! echo "Waiting for server to come up" @@ -30,12 +34,13 @@ function wait_for_port { sleep 1 N=$(($N+1)) if [ $N -ge $2 ] ; then + curl -v "$1" || true echo "Timed out waiting for port $1" exit 2 fi done } -wait_for_port "http://localhost:9001" 60 +wait_for_port "http://localhost:${PORT}" 60 echo "Starting phantoms" From 26f9b283a51269857bb1113b1179387faf29f021 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 30 Jul 2014 15:44:12 -0700 Subject: [PATCH 077/126] Tree hashes of builds now ignore package.json We were finding that npm was inconsistently including various fields (eg "readme") in package.json, leading to spurious "must update the version number" errors in publish-release --from-checkout. This should be good enough, as any actual package change should also have some other file changed! --- tools/files.js | 13 ++++++++++++- tools/package-client.js | 13 ++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tools/files.js b/tools/files.js index f43360a73b..075358273e 100644 --- a/tools/files.js +++ b/tools/files.js @@ -283,7 +283,13 @@ files.fileHash = function (filename) { // Returns a base64 SHA256 hash representing a tree on disk. It is not sensitive // to modtime, uid/gid, or any permissions bits other than the current-user-exec // bit on normal files. -files.treeHash = function (root) { +files.treeHash = function (root, options) { + options = _.extend({ + ignore: function (relativePath) { + return false; + } + }, options); + var crypto = require('crypto'); var hash = crypto.createHash('sha256'); @@ -296,6 +302,11 @@ files.treeHash = function (root) { }; var traverse = function (relativePath) { + if (options.ignore(relativePath)) { + hashLog && hashLog.push('SKIP ' + JSON.stringify(relativePath) + '\n'); + return; + } + var absPath = path.join(root, relativePath); var stat = fs.lstatSync(absPath); diff --git a/tools/package-client.js b/tools/package-client.js index 5dd061dbf8..07ffeabe06 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -354,7 +354,18 @@ var bundleBuild = function (unipackage) { files.createTarball(tarInputDir, buildTarball); var tarballHash = files.fileHash(buildTarball); - var treeHash = files.treeHash(tarInputDir); + var treeHash = files.treeHash(tarInputDir, { + // We don't include any package.json from an npm module in the tree hash, + // because npm isn't super consistent about what it puts in there (eg, does + // it include the "readme" field)? This ends up leading to spurious + // differences. The tree hash will still notice any actual CODE changes in + // the npm packages. + ignore: function (relativePath) { + var pieces = relativePath.split(path.sep); + return pieces.length && _.last(pieces) === 'package.json' + && _.contains(pieces, 'npm'); + } + }); return { buildTarball: buildTarball, From f2ebbb1238396f075e09b06502c17aa9cd451b49 Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 30 Jul 2014 08:32:59 -0700 Subject: [PATCH 078/126] more piping errors to stderr --- tools/commands-packages.js | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 80fcff3080..1a018ecad5 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -698,16 +698,22 @@ main.registerCommand({ } process.stdout.write("Creating a new release version...\n"); - // Send it over! - var record = { - track: relConf.track, - version: relConf.version, - orderKey: relConf.orderKey, - description: relConf.description, - recommended: !!relConf.recommended, - tool: relConf.tool, - packages: relConf.packages - }; + try { + // Send it over! + var record = { + track: relConf.track, + version: relConf.version, + orderKey: relConf.orderKey, + description: relConf.description, + recommended: !!relConf.recommended, + tool: relConf.tool, + packages: relConf.packages + }; + } catch (err) { + process.stderr.write("ERROR: " + err + "\n"); + return 1; + } + var uploadInfo; if (!relConf.patchFrom) { uploadInfo = conn.call('createReleaseVersion', record); @@ -1554,7 +1560,7 @@ main.registerCommand({ process.stdout.write(" Done!\n"); } } catch (err) { - process.stdout.write("\n" + err + "\n"); + process.stderr.write("\n" + err + "\n"); } conn.close(); catalog.official.refresh(); @@ -1618,7 +1624,7 @@ main.registerCommand({ " is now a recommended release\n"); } } catch (err) { - process.stdout.write("\n" + err + "\n"); + process.stderr.write("\n" + err + "\n"); } conn.close(); catalog.official.refresh(); @@ -1669,7 +1675,7 @@ main.registerCommand({ conn.call('_setEarliestCompatibleVersion', versionInfo, ecv); process.stdout.write("Done!\n"); } catch (err) { - process.stdout.write("\n" + err + "\n"); + process.stderr.write("\n" + err + "\n"); } conn.close(); catalog.official.refresh(); @@ -1712,7 +1718,7 @@ main.registerCommand({ conn.call('_changePackageHomepage', name, url); process.stdout.write("Done!\n"); } catch (err) { - process.stdout.write("\n" + err + "\n"); + process.stderr.write("\n" + err + "\n"); } conn.close(); catalog.official.refresh(); From a91ff48da5806c3844e67ad6c0f061d65143a335 Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 30 Jul 2014 15:47:42 -0700 Subject: [PATCH 079/126] Allow MDG members to publish unprefixed releases in client --- tools/package-client.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/package-client.js b/tools/package-client.js index 07ffeabe06..c0a364b3ab 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -500,7 +500,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) } var authorized = _.indexOf( _.pluck(packRecord.maintainers, 'username'), auth.loggedInUsername()); - if (authorized == -1) { + if (authorized == -1 && name.indexOf(":") !== -1) { process.stderr.write('You are not an authorized maintainer of ' + name + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions. \n'); return 1; From 0460c5015f88ce9d8357cc79108cd0b8c753d4a7 Mon Sep 17 00:00:00 2001 From: ekatek Date: Wed, 30 Jul 2014 16:01:32 -0700 Subject: [PATCH 080/126] oh right wrap try around the block that can actually fail --- tools/commands-packages.js | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 1a018ecad5..368638c360 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -698,8 +698,6 @@ main.registerCommand({ } process.stdout.write("Creating a new release version...\n"); - try { - // Send it over! var record = { track: relConf.track, version: relConf.version, @@ -709,18 +707,19 @@ main.registerCommand({ tool: relConf.tool, packages: relConf.packages }; + + var uploadInfo; + try { + if (!relConf.patchFrom) { + uploadInfo = conn.call('createReleaseVersion', record); + } else { + uploadInfo = conn.call('createPatchReleaseVersion', record, relConf.patchFrom); + } } catch (err) { process.stderr.write("ERROR: " + err + "\n"); return 1; } - var uploadInfo; - if (!relConf.patchFrom) { - uploadInfo = conn.call('createReleaseVersion', record); - } else { - uploadInfo = conn.call('createPatchReleaseVersion', record, relConf.patchFrom); - } - // Get it back. catalog.official.refresh(); process.stdout.write("Done creating " + relConf.track + "@" + From 693b78001c83d48ef31ea0fe432327f68ee16023 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 16:21:13 -0700 Subject: [PATCH 081/126] Also log sanitized error in server logs (which gets sent to client) on match errors --- packages/livedata/livedata_server.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index 11d1114bba..d845232113 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1458,8 +1458,11 @@ var wrapInternalException = function (exception, context) { // tests can set the 'expected' flag on an exception so it won't go to the // server log - if (!exception.expected) + if (!exception.expected) { Meteor._debug("Exception " + context, exception.stack); + Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); + Meteor._debug(); + } // Did the error contain more details that could have been useful if caught in // server code (or if thrown from non-client-originated code), but also From b85e3e1c5efeff31048ae3e3bc0fc9bf7fd5ecbf Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 16:22:58 -0700 Subject: [PATCH 082/126] Update History.md: log match errors on server --- History.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/History.md b/History.md index 6811c731ed..0c02250709 100644 --- a/History.md +++ b/History.md @@ -6,6 +6,9 @@ change is that `appcache` is now enabled by default on Firefox, because Firefox no longer makes a confusing popup. #2241 +* When a call to `match` fails in a method or subscription, log the + failure on the server. (This matches the behavior described in our docs) + ## v0.8.3 From 2658a18a010a7cdb8c5b6e7d075a9f4de1168e7e Mon Sep 17 00:00:00 2001 From: Ryan Yeske Date: Thu, 15 May 2014 12:08:31 -0700 Subject: [PATCH 083/126] recognize forceApprovalPrompt option in Accounts.ui.config this option was originally added via #1226 --- docs/client/api.js | 5 +++++ packages/accounts-ui-unstyled/accounts_ui.js | 20 +++++++++++++++++-- .../accounts-ui-unstyled/accounts_ui_tests.js | 6 +++++- .../login_buttons_single.js | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/client/api.js b/docs/client/api.js index a4c9a3a828..d18cdf2f76 100644 --- a/docs/client/api.js +++ b/docs/client/api.js @@ -1186,6 +1186,11 @@ Template.api.accounts_ui_config = { type: "Object", descr: "To ask the user for permission to act on their behalf when offline, map the relevant external service to `true`. Currently only supported with Google. See [Meteor.loginWithExternalService](#meteor_loginwithexternalservice) for more details." }, + { + name: "forceApprovalPrompt", + type: "Boolean", + descr: "If true, forces the user to approve the app's permissions, even if previously approved. Currently only supported with Google." + }, { name: "passwordSignupFields", type: "String", diff --git a/packages/accounts-ui-unstyled/accounts_ui.js b/packages/accounts-ui-unstyled/accounts_ui.js index 5f241ab997..6a0f0851c2 100644 --- a/packages/accounts-ui-unstyled/accounts_ui.js +++ b/packages/accounts-ui-unstyled/accounts_ui.js @@ -2,12 +2,14 @@ Accounts.ui = {}; Accounts.ui._options = { requestPermissions: {}, - requestOfflineToken: {} + requestOfflineToken: {}, + forceApprovalPrompt: {} }; +// XXX refactor duplicated code in this function Accounts.ui.config = function(options) { // validate options keys - var VALID_KEYS = ['passwordSignupFields', 'requestPermissions', 'requestOfflineToken']; + var VALID_KEYS = ['passwordSignupFields', 'requestPermissions', 'requestOfflineToken', 'forceApprovalPrompt']; _.each(_.keys(options), function (key) { if (!_.contains(VALID_KEYS, key)) throw new Error("Accounts.ui.config: Invalid key: " + key); @@ -56,6 +58,20 @@ Accounts.ui.config = function(options) { } }); } + + // deal with `forceApprovalPrompt` + if (options.forceApprovalPrompt) { + _.each(options.forceApprovalPrompt, function (value, service) { + if (service !== 'google') + throw new Error("Accounts.ui.config: `forceApprovalPrompt` only supported for Google login at the moment."); + + if (Accounts.ui._options.forceApprovalPrompt[service]) { + throw new Error("Accounts.ui.config: Can't set `forceApprovalPrompt` more than once for " + service); + } else { + Accounts.ui._options.forceApprovalPrompt[service] = value; + } + }); + } }; passwordSignupFields = function () { diff --git a/packages/accounts-ui-unstyled/accounts_ui_tests.js b/packages/accounts-ui-unstyled/accounts_ui_tests.js index a8f4fc40f5..5e4bef9ea0 100644 --- a/packages/accounts-ui-unstyled/accounts_ui_tests.js +++ b/packages/accounts-ui-unstyled/accounts_ui_tests.js @@ -5,7 +5,7 @@ // XXX it'd be cool to also test that the right thing happens if options -// *are* validated, but Accouns.ui._options is global state which makes this hard +// *are* validated, but Accounts.ui._options is global state which makes this hard // (impossible?) Tinytest.add('accounts-ui - config validates keys', function (test) { test.throws(function () { @@ -19,4 +19,8 @@ Tinytest.add('accounts-ui - config validates keys', function (test) { test.throws(function () { Accounts.ui.config({requestPermissions: {facebook: "not an array"}}); }); + + test.throws(function () { + Accounts.ui.config({forceApprovalPrompt: {facebook: "only google"}}); + }); }); diff --git a/packages/accounts-ui-unstyled/login_buttons_single.js b/packages/accounts-ui-unstyled/login_buttons_single.js index 79d35a4190..96e1e65624 100644 --- a/packages/accounts-ui-unstyled/login_buttons_single.js +++ b/packages/accounts-ui-unstyled/login_buttons_single.js @@ -29,6 +29,8 @@ Template._loginButtonsLoggedOutSingleLoginButton.events({ options.requestPermissions = Accounts.ui._options.requestPermissions[serviceName]; if (Accounts.ui._options.requestOfflineToken[serviceName]) options.requestOfflineToken = Accounts.ui._options.requestOfflineToken[serviceName]; + if (Accounts.ui._options.forceApprovalPrompt[serviceName]) + options.forceApprovalPrompt = Accounts.ui._options.forceApprovalPrompt[serviceName]; loginWithService(options, callback); } From 9fd2214d89b31c229b3faa371f16e27a405a914e Mon Sep 17 00:00:00 2001 From: Mitar Date: Thu, 12 Jun 2014 00:21:51 -0700 Subject: [PATCH 084/126] Assure that transform is not changing cached object. --- packages/minimongo/observe.js | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/minimongo/observe.js b/packages/minimongo/observe.js index 38bcb4514c..cc97ca73d0 100644 --- a/packages/minimongo/observe.js +++ b/packages/minimongo/observe.js @@ -157,6 +157,7 @@ LocalCollection._observeFromObserveChanges = function (cursor, observeCallbacks) var self = this; if (observeCallbacks.changed) { var oldDoc = self.docs.get(id); + oldDoc = EJSON.clone(oldDoc); var doc = EJSON.clone(oldDoc); LocalCollection._applyChanges(doc, fields); observeCallbacks.changed(transform(doc), transform(oldDoc)); From e76332b10274560e473081d38c26fa033c2fd4ef Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 21 Jul 2014 18:41:27 -0700 Subject: [PATCH 085/126] Make it more clear that the clone is for transform --- packages/minimongo/observe.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/minimongo/observe.js b/packages/minimongo/observe.js index cc97ca73d0..95b92d3d51 100644 --- a/packages/minimongo/observe.js +++ b/packages/minimongo/observe.js @@ -157,10 +157,10 @@ LocalCollection._observeFromObserveChanges = function (cursor, observeCallbacks) var self = this; if (observeCallbacks.changed) { var oldDoc = self.docs.get(id); - oldDoc = EJSON.clone(oldDoc); var doc = EJSON.clone(oldDoc); LocalCollection._applyChanges(doc, fields); - observeCallbacks.changed(transform(doc), transform(oldDoc)); + observeCallbacks.changed(transform(doc), + transform(EJSON.clone(oldDoc))); } }, removed: function (id) { From 114b36bbcbfc012f88bc0f7db1dd893ad07f9859 Mon Sep 17 00:00:00 2001 From: Tom Wang Date: Tue, 22 Jul 2014 18:37:01 +0800 Subject: [PATCH 086/126] fix typo --- docs/client/concepts.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/client/concepts.html b/docs/client/concepts.html index 1ef9bbfd4b..ea2f5809e6 100644 --- a/docs/client/concepts.html +++ b/docs/client/concepts.html @@ -33,7 +33,7 @@ packages that most any app will use (for example `webapp`, which handles incoming HTTP connections, and `templating`, which lets you make HTML templates that automatically update live as data changes). Then there are optional packages like `email`, which lets your app -send emails, or the Meteor Accounts series (`account-password`, +send emails, or the Meteor Accounts series (`accounts-password`, `accounts-facebook`, `accounts-ui`, and others) which provide a full-featured user account system that you can drop right into your app. And beyond these "official" packages, there are hundreds of From 135dd942aa6c7026529e32937862c378e518b716 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 12:00:51 -0700 Subject: [PATCH 087/126] Expose match failures in server logs --- packages/livedata/livedata_server.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index 0ff5dd81f3..11d1114bba 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1456,6 +1456,11 @@ var wrapInternalException = function (exception, context) { if (!exception || exception instanceof Meteor.Error) return exception; + // tests can set the 'expected' flag on an exception so it won't go to the + // server log + if (!exception.expected) + Meteor._debug("Exception " + context, exception.stack); + // Did the error contain more details that could have been useful if caught in // server code (or if thrown from non-client-originated code), but also // provided a "sanitized" version with more context than 500 Internal server @@ -1467,11 +1472,6 @@ var wrapInternalException = function (exception, context) { "is not a Meteor.Error; ignoring"); } - // tests can set the 'expected' flag on an exception so it won't go to the - // server log - if (!exception.expected) - Meteor._debug("Exception " + context, exception.stack); - return new Meteor.Error(500, "Internal server error"); }; From d8e28eb74bd6ada4306b66dd256ff24a89786b1e Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 16:21:13 -0700 Subject: [PATCH 088/126] Also log sanitized error in server logs (which gets sent to client) on match errors --- packages/livedata/livedata_server.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index 11d1114bba..d845232113 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1458,8 +1458,11 @@ var wrapInternalException = function (exception, context) { // tests can set the 'expected' flag on an exception so it won't go to the // server log - if (!exception.expected) + if (!exception.expected) { Meteor._debug("Exception " + context, exception.stack); + Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); + Meteor._debug(); + } // Did the error contain more details that could have been useful if caught in // server code (or if thrown from non-client-originated code), but also From 7c4af8d12fd627995f3a4aeb789b0d0247a5031d Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 16:51:12 -0700 Subject: [PATCH 089/126] Add guard I forgot to add in 693b78001c83d48ef31ea0fe432327f68ee16023 (Thanks @justinsb for noticing this!) --- packages/livedata/livedata_server.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index d845232113..e7b00b6fde 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1460,8 +1460,10 @@ var wrapInternalException = function (exception, context) { // server log if (!exception.expected) { Meteor._debug("Exception " + context, exception.stack); - Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); - Meteor._debug(); + if (exception.sanitizedError) { + Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); + Meteor._debug(); + } } // Did the error contain more details that could have been useful if caught in From 30e02600b490ecd2976e72554cb0a3ffd7dcde3f Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 30 Jul 2014 16:51:12 -0700 Subject: [PATCH 090/126] Add guard I forgot to add in 693b78001c83d48ef31ea0fe432327f68ee16023 (Thanks @justinsb for noticing this!) --- packages/livedata/livedata_server.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/livedata/livedata_server.js b/packages/livedata/livedata_server.js index d845232113..e7b00b6fde 100644 --- a/packages/livedata/livedata_server.js +++ b/packages/livedata/livedata_server.js @@ -1460,8 +1460,10 @@ var wrapInternalException = function (exception, context) { // server log if (!exception.expected) { Meteor._debug("Exception " + context, exception.stack); - Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); - Meteor._debug(); + if (exception.sanitizedError) { + Meteor._debug("Sanitized and reported to the client as:", exception.sanitizedError.message); + Meteor._debug(); + } } // Did the error contain more details that could have been useful if caught in From c5acfdf8c12b9c791bd4fb004e12df4b438a6f32 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 30 Jul 2014 19:24:47 -0700 Subject: [PATCH 091/126] add missing underscore dep --- packages/autoupdate/package.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/autoupdate/package.js b/packages/autoupdate/package.js index db0eac9014..10c74e4480 100644 --- a/packages/autoupdate/package.js +++ b/packages/autoupdate/package.js @@ -6,7 +6,7 @@ Package.describe({ Package.on_use(function (api) { api.use('webapp', 'server'); api.use(['deps', 'retry'], 'client'); - api.use(['livedata', 'mongo-livedata'], ['client', 'server']); + api.use(['livedata', 'mongo-livedata', 'underscore'], ['client', 'server']); api.use('deps', 'client'); api.use('reload', 'client', {weak: true}); From e59fb948b5fe2ef36af1fdfd4585f7413cd860ea Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 30 Jul 2014 19:24:55 -0700 Subject: [PATCH 092/126] fix tests broken by 6850b679e9 --- .../package-version-parser/package-version-parser-tests.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/package-version-parser/package-version-parser-tests.js b/packages/package-version-parser/package-version-parser-tests.js index cd48477b92..e247958980 100644 --- a/packages/package-version-parser/package-version-parser-tests.js +++ b/packages/package-version-parser/package-version-parser-tests.js @@ -1,7 +1,11 @@ var currentTest = null; var t = function (versionString, expected, descr) { - currentTest.equal(PackageVersion.parseConstraint(versionString), expected, descr); + currentTest.equal( + _.omit(PackageVersion.parseConstraint(versionString), + 'constraintString'), + expected, + descr); }; var FAIL = function (versionString) { From b11a623193456f98cd1d48141b4b2f58648e7c90 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 30 Jul 2014 20:38:27 -0700 Subject: [PATCH 093/126] assert rebuildLocalPackages in capture --- tools/catalog.js | 1 + tools/selftest.js | 12 +++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/catalog.js b/tools/catalog.js index c2cab9ae25..23995fa782 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -774,6 +774,7 @@ _.extend(CompleteCatalog.prototype, { rebuildLocalPackages: function (namedPackages) { var self = this; self._requireInitialized(); + buildmessage.assertInCapture(); // Clear any cached builds in the package cache. packageCache.packageCache.refresh(); diff --git a/tools/selftest.js b/tools/selftest.js index 5074ffd3b9..f30a68cf1e 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -11,6 +11,7 @@ var packageLoader = require('./package-loader.js'); var Future = require('fibers/future'); var uniload = require('./uniload.js'); var config = require('./config.js'); +var buildmessage = require('./buildmessage.js'); var util = require('util'); var child_process = require('child_process'); var webdriver = require('browserstack-webdriver'); @@ -67,9 +68,14 @@ var expectThrows = markStack(function (f) { var getToolsPackage = function () { // Rebuild the tool package --- necessary because we don't actually // rebuild the tool in the cached version every time. - if (catalog.complete.rebuildLocalPackages([toolPackageName]) !== 1) { - throw Error("didn't rebuild meteor-tool?"); - } + var messages = buildmessage.capture(function () { + if (catalog.complete.rebuildLocalPackages([toolPackageName]) !== 1) { + throw Error("didn't rebuild meteor-tool?"); + } + }); + if (messages.hasMessages()) { + throw Error(messages.formatMessages()); + }; var loader = new packageLoader.PackageLoader({versions: null}); var toolPackage = loader.getPackage(toolPackageName); return toolPackage; From 06a7dc90b54e2f22c575a9c17b850b99bad11966 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Wed, 30 Jul 2014 21:00:33 -0700 Subject: [PATCH 094/126] Add many buildmessage.capture/assertInCapture Moving towards a world where all things that might invoke buildmessage.error are encouraged to be in a buildmessage.capture. This commit is the answer to the question "how many small changes need to be made to add buildmessage.assertInCapture to PackageCache.loadPackageAtPath?" Next steps include: - Making catalog.resolveConstraints ALWAYS buildmessage.assertInCapture (not just when ignoreProjectDeps isn't passed) - Then changing resolveConstraints to complain using buildmessage - Removing the process.exit(1) in _ensureDepsUpToDate - Adding a more structured way to ensure that most commands call _ensureDepsUpToDate at an unsurprising location --- tools/bundler.js | 32 +++-- tools/catalog-base.js | 17 --- tools/catalog.js | 8 ++ tools/commands-packages.js | 166 ++++++++++++++++-------- tools/commands.js | 71 +++++++--- tools/compiler.js | 14 +- tools/deploy-galaxy.js | 9 +- tools/deploy.js | 12 +- tools/main.js | 22 +++- tools/package-cache.js | 2 + tools/package-loader.js | 3 + tools/project.js | 8 ++ tools/run-app.js | 12 +- tools/selftest.js | 33 ++--- tools/stats.js | 3 + tools/tests/old/test-bundler-options.js | 9 +- tools/tests/report-stats.js | 18 ++- tools/tropohouse.js | 2 + tools/unipackage.js | 7 +- tools/updater.js | 41 +++--- 20 files changed, 331 insertions(+), 158 deletions(-) diff --git a/tools/bundler.js b/tools/bundler.js index 952622d16d..0a561ee442 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -431,6 +431,7 @@ _.extend(Target.prototype, { // on server targets. make: function (options) { var self = this; + buildmessage.assertInCapture(); // Populate the list of unibuilds to load self._determineLoadOrder({ @@ -478,6 +479,8 @@ _.extend(Target.prototype, { // package name as a string _determineLoadOrder: function (options) { var self = this; + buildmessage.assertInCapture(); + var packageLoader = self.packageLoader; // Find the roots @@ -1641,23 +1644,12 @@ exports.bundle = function (options) { var includeNodeModulesSymlink = !!options.includeNodeModulesSymlink; var buildOptions = options.buildOptions || {}; + var appDir = project.project.rootDir; if (! release.usingRightReleaseForApp(appDir)) throw new Error("running wrong release for app?"); var arch = buildOptions.arch || archinfo.host(); - var appDir = project.project.rootDir; - var packageLoader = project.project.getPackageLoader(); - var downloaded = project.project._ensurePackagesExistOnDisk( - project.project.dependencies, { arch: arch, verbose: true }); - - if (_.keys(downloaded).length !== - _.keys(project.project.dependencies).length) { - buildmessage.error("Unable to download package builds for this architecture."); - // Recover by returning. - return; - } - var releaseName = release.current.isCheckout() ? "none" : release.current.name; var builtBy = "Meteor" + (release.current.name ? @@ -1672,6 +1664,17 @@ exports.bundle = function (options) { var messages = buildmessage.capture({ title: "building the application" }, function () { + var packageLoader = project.project.getPackageLoader(); + var downloaded = project.project._ensurePackagesExistOnDisk( + project.project.dependencies, { arch: arch, verbose: true }); + + if (_.keys(downloaded).length !== + _.keys(project.project.dependencies).length) { + buildmessage.error("Unable to download package builds for this architecture."); + // Recover by returning. + return; + } + var controlProgram = null; var makeClientTarget = function (app) { @@ -1843,8 +1846,7 @@ exports.bundle = function (options) { _.each(programs, function (p) { // Read this directory as a package and create a target from // it - var pkg = packageCache.packageCache. - loadPackageAtPath(p.name, p.path); + var pkg = packageCache.packageCache.loadPackageAtPath(p.name, p.path); var target; switch (p.type) { case "server": @@ -1985,6 +1987,7 @@ exports.bundle = function (options) { // without also saying "make its namespace the same as the global // namespace." It should be an easy refactor, exports.buildJsImage = function (options) { + buildmessage.assertInCapture(); if (options.npmDependencies && ! options.npmDir) throw new Error("Must indicate .npm directory to use"); if (! options.name) @@ -2036,6 +2039,7 @@ exports.readJsImage = function (controlFilePath) { // with a topological sort. exports.iterateOverAllUsedUnipackages = function (loader, arch, packageNames, callback) { + buildmessage.assertInCapture(); var target = new Target({packageLoader: loader, arch: arch}); target._determineLoadOrder({packages: packageNames}); diff --git a/tools/catalog-base.js b/tools/catalog-base.js index a83b2d1260..3e787c08a4 100644 --- a/tools/catalog-base.js +++ b/tools/catalog-base.js @@ -339,23 +339,6 @@ _.extend(baseCatalog.BaseCatalog.prototype, { return self._recordOrRefresh(getDef); }, - // Given a name and a version of a package, return a path on disk - // from which we can load it. If we don't have it on disk (we - // haven't downloaded it, or it just plain doesn't exist in the - // catalog) return null. - // - // Doesn't download packages. Downloading should be done at the time - // that .meteor/versions is updated. - getLoadPathForPackage: function (name, version) { - var self = this; - - var packageDir = tropohouse.default.packagePath(name, version); - if (fs.existsSync(packageDir)) { - return packageDir; - } - return null; - }, - // Reload catalog data to account for new information if needed. refresh: function () { throw new Error("no such thing as a base refresh"); diff --git a/tools/catalog.js b/tools/catalog.js index 23995fa782..bb75999aa6 100644 --- a/tools/catalog.js +++ b/tools/catalog.js @@ -245,6 +245,9 @@ _.extend(CompleteCatalog.prototype, { var self = this; opts = opts || {}; self._requireInitialized(); + // XXX for now, just doing the assertion if we have to call project + // stuff. but oh, this will be improved. + opts.ignoreProjectDeps || buildmessage.assertInCapture(); // Kind of a hack, as per specification. We don't have a constraint solver // initialized yet. We are probably trying to build the constraint solver @@ -549,6 +552,8 @@ _.extend(CompleteCatalog.prototype, { // compiled and built in the directory, and null otherwise. _maybeGetUpToDateBuild : function (name, constraintSolverOpts) { var self = this; + buildmessage.assertInCapture(); + var sourcePath = self.effectiveLocalPackages[name]; var buildDir = path.join(sourcePath, '.build.' + name); if (fs.existsSync(buildDir)) { @@ -588,6 +593,8 @@ _.extend(CompleteCatalog.prototype, { // that we use is in the catalog already, we build it here. _build : function (name, onStack, constraintSolverOpts) { var self = this; + buildmessage.assertInCapture(); + var unip = null; if (! _.has(self.unbuilt, name)) { @@ -840,6 +847,7 @@ _.extend(CompleteCatalog.prototype, { getLoadPathForPackage: function (name, version, constraintSolverOpts) { var self = this; self._requireInitialized(); + buildmessage.assertInCapture(); constraintSolverOpts = constraintSolverOpts || {}; // Check local packages first. diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 368638c360..51249e1437 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -905,14 +905,15 @@ main.registerCommand({ }, function (options) { var items = []; - // Packages that are used by this app - var packages = project.getConstraints(); - // Versions of the packages. We need this to get the right description for the - // user, in case it changed between versions. - var versions = project.getVersions(); var newVersionsAvailable = false; var messages = buildmessage.capture(function () { + // Packages that are used by this app + var packages = project.getConstraints(); + // Versions of the packages. We need this to get the right description for + // the user, in case it changed between versions. + var versions = project.getVersions(); + _.each(packages, function (version, name) { if (!version) { version = versions[name]; @@ -1152,7 +1153,14 @@ main.registerCommand({ var solutionPackageVersions = null; var directDependencies = project.getConstraints(); - var previousVersions = project.getVersions(); + var previousVersions; + var messages = buildmessage.capture(function () { + previousVersions = project.getVersions(); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } var solutionReleaseVersion = _.find(releaseVersionsToTry, function (versionToTry) { var releaseRecord = catalog.complete.getReleaseVersion(releaseTrack, versionToTry); if (!releaseRecord) @@ -1193,8 +1201,16 @@ main.registerCommand({ var upgradersToRun = upgraders.upgradersToRun(); // Write the new versions to .meteor/packages and .meteor/versions. - var setV = project.setVersions(solutionPackageVersions, - { alwaysRecord : true }); + var setV; + messages = buildmessage.capture(function () { + setV = project.setVersions(solutionPackageVersions, + { alwaysRecord : true }); + }); + if (messages.hasMessages()) { + process.stderr.write("Error while setting versions:\n" + + messages.formatMessages()); + return 1; + } project.showPackageChanges(previousVersions, solutionPackageVersions, { onDiskPackages: setV.downloaded }); @@ -1227,13 +1243,20 @@ main.registerCommand({ // We can't update packages when we are not in a release. if (!options.appDir) return 0; - var versions = project.getVersions(); - var allPackages = project.getCurrentCombinedConstraints(); - var upgradePackages; + var versions, allPackages; + messages = buildmessage.capture(function () { + versions = project.getVersions(); + allPackages = project.getCurrentCombinedConstraints(); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } // If no packages have been specified, then we will send in a request to // update all direct dependencies. If a specific list of packages has been // specified, then only upgrade those. + var upgradePackages; if (options.args.length === 0) { upgradePackages = _.pluck(allPackages, 'packageName'); } else { @@ -1257,12 +1280,22 @@ main.registerCommand({ } // Set our versions and download the new packages. - setV = project.setVersions(newVersions, { - alwaysRecord : true }); - - // Display changes: what we have added/removed/upgraded. - return project.showPackageChanges(versions, newVersions, { - onDiskPackages: setV.downloaded}); + messages = buildmessage.capture(function () { + setV = project.setVersions(newVersions, { alwaysRecord : true }); + }); + // XXX cleanup this madness of error handling + if (messages.hasMessages()) { + process.stderr.write("Error while setting package versions:\n" + + messages.formatMessages()); + return 1; + } + var showExitCode = project.showPackageChanges( + versions, newVersions, { onDiskPackages: setV.downloaded }); + if (!setV.success) { + process.stderr.write("Could not install all the requested packages.\n"); + return 1; + } + return showExitCode; } return 0; }); @@ -1293,9 +1326,16 @@ main.registerCommand({ // Read in existing package dependencies. var packages = project.getConstraints(); - // Combine into one object mapping package name to list of - // constraints, to pass in to the constraint solver. - var allPackages = project.getCurrentCombinedConstraints(); + var allPackages; + var messages = buildmessage.capture(function () { + // Combine into one object mapping package name to list of constraints, to + // pass in to the constraint solver. + allPackages = project.getCurrentCombinedConstraints(); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } // For every package name specified, add it to our list of package // constraints. Don't run the constraint solver until you have added all of @@ -1377,36 +1417,43 @@ main.registerCommand({ return 1; } - // Get the contents of our versions file. We need to pass them to the - // constraint solver, because our contract with the user says that we will - // never downgrade a dependency. - var versions = project.getVersions(); + var downloaded, versions, newVersions; + var messages = buildmessage.capture(function () { + // Get the contents of our versions file. We need to pass them to the + // constraint solver, because our contract with the user says that we will + // never downgrade a dependency. + versions = project.getVersions(); + // Call the constraint solver. + var resolverOpts = { + previousSolution: versions, + breaking: !!options.force + }; + newVersions = catalog.complete.resolveConstraints( + allPackages, + resolverOpts, + { ignoreProjectDeps: true }); + if ( ! newVersions) { + // XXX: Better error handling. + process.stderr.write("Cannot resolve package dependencies.\n"); + return; + } - // Call the constraint solver. - var resolverOpts = { - previousSolution: versions, - breaking: !!options.force - }; - var newVersions = catalog.complete.resolveConstraints(allPackages, - resolverOpts, - { ignoreProjectDeps: true }); - if ( ! newVersions) { - // XXX: Better error handling. - process.stderr.write("Cannot resolve package dependencies.\n"); + // Don't tell the user what all the operations were until we finish -- we + // don't want to give a false sense of completeness until everything is + // written to disk. + var messageLog = []; + + // Install the new versions. If all new versions were installed + // successfully, then change the .meteor/packages and .meteor/versions to + // match expected reality. + downloaded = project.addPackages(constraints, newVersions); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; } - // Don't tell the user what all the operations were until we finish -- we - // don't want to give a false sense of completeness until everything is - // written to disk. - var messageLog = []; - - - // Install the new versions. If all new versions were installed successfully, - // then change the .meteor/packages and .meteor/versions to match expected - // reality. - var downloaded = project.addPackages(constraints, newVersions); - var ret = project.showPackageChanges(versions, newVersions, { onDiskPackages: downloaded}); if (ret !== 0) return ret; @@ -1466,19 +1513,24 @@ main.registerCommand({ } }); + var messages = buildmessage.capture(function () { + // Get the contents of our versions file, we will want them in order to + // remove to the user what we removed. + var versions = project.getVersions(); - // Get the contents of our versions file, we will want them in order to remove - // to the user what we removed. - var versions = project.getVersions(); + // Remove the packages from the project! There is really no way for this to + // fail, unless something has gone horribly wrong, so we don't need to check + // for it. + project.removePackages(packagesToRemove); - // Remove the packages from the project! There is really no way for this to - // fail, unless something has gone horribly wrong, so we don't need to check - // for it. - project.removePackages(packagesToRemove); - - // Retrieve the new dependency versions that we have chosen for this project - // and do some pretty output reporting. - var newVersions = project.getVersions(); + // Retrieve the new dependency versions that we have chosen for this project + // and do some pretty output reporting. + var newVersions = project.getVersions(); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } // Log that we removed the constraints. It is possible that there are // constraints that we officially removed that the project still 'depends' on, diff --git a/tools/commands.js b/tools/commands.js index d05b10cd9e..ea5e91f620 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -163,6 +163,7 @@ main.registerCommand({ catalog.official.refresh(); var loadPackages = function (packagesToLoad, loader) { + buildmessage.assertInCapture(); _.each(packagesToLoad, function (name) { // Calling getPackage on the loader will return a unipackage object, which // means that the package will be compiled/downloaded. That we throw the @@ -184,7 +185,7 @@ main.registerCommand({ // of everything we need from versions have been downloaded. (Calling // buildPackages may be redundant, but can't hurt.) if (options.appDir) { - loadPackages(_.keys(project.getVersions), project.getPackageLoader()); + loadPackages(_.keys(project.getVersions()), project.getPackageLoader()); } // Using a release? Get all the packages in the release. @@ -237,7 +238,13 @@ main.registerCommand({ // those issues are concurrency-related, or possibly some weird // order-of-execution of interaction that we are failing to understand. This // seems to fix it in a clear and understandable fashion.) - project.getVersions(); + var messages = buildmessage.capture(function () { + project.getVersions(); // #StructuredProjectInitialization + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } // XXX factor this out into a {type: host/port}? var portMatch = options.port.match(/^(?:(.+):)?([0-9]+)$/); @@ -452,7 +459,13 @@ main.registerCommand({ project.setMuted(true); project.writeMeteorReleaseVersion( release.current.isCheckout() ? "none" : release.current.name); - project._ensureDepsUpToDate(); + var messages = buildmessage.capture(function () { + project._ensureDepsUpToDate(); + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } process.stdout.write(appPath + ": created"); if (options.example && options.example !== appPath) @@ -530,9 +543,17 @@ main.registerCommand({ var bundlePath = options['directory'] ? outputPath : path.join(buildDir, 'bundle'); - var bundler = require(path.join(__dirname, 'bundler.js')); - var loader = project.getPackageLoader(); + var loader; + var messages = buildmessage.capture(function () { + loader = project.getPackageLoader(); + }); + if (messages.hasMessages()) { + process.stderr.write("Errors prevented bundling your app:\n"); + process.stderr.write(messages.formatMessages()); + return 1; + } + var bundler = require(path.join(__dirname, 'bundler.js')); var bundleResult = bundler.bundle({ outputPath: bundlePath, buildOptions: { @@ -728,7 +749,13 @@ main.registerCommand({ // issues are concurrency-related, or possibly some weird order-of-execution // of interaction that we are failing to understand. This seems to fix it in a // clear and understandable fashion.) - project.getVersions(); + var messages = buildmessage.capture(function () { + project.getVersions(); // #StructuredProjectInitialization + }); + if (messages.hasMessages()) { + process.stderr.write(messages.formatMessages()); + return 1; + } if (options.password) { if (useGalaxy) { @@ -1248,7 +1275,7 @@ main.registerCommand({ _.each(osArches, function (osArch) { _.each(release.packages, function (pkgVersion, pkgName) { buildmessage.enterJob({ - title: "Looking up " + pkgName + "@" + pkgVersion + " on " + osArch + title: "looking up " + pkgName + "@" + pkgVersion + " on " + osArch }, function () { if (!catalog.official.getBuildsForArches(pkgName, pkgVersion, [osArch])) { buildmessage.error("missing build of " + pkgName + "@" + pkgVersion + @@ -1274,20 +1301,30 @@ main.registerCommand({ // XXX update to '.meteor' when we combine houses var tmpTropo = new tropohouse.Tropohouse( path.join(tmpdir, '.meteor0'), catalog.official); - try { - tmpTropo.maybeDownloadPackageForArchitectures( - {packageName: toolPkg.package, version: toolPkg.constraint}, - [osArch], // XXX 'browser' too? - true); - _.each(release.packages, function (pkgVersion, pkgName) { + var messages = buildmessage.capture(function () { + buildmessage.enterJob({ + title: "downloading tool package " + toolPkg.package + "@" + + toolPkg.constraint + }, function () { tmpTropo.maybeDownloadPackageForArchitectures( - {packageName: pkgName, version: pkgVersion}, + {packageName: toolPkg.package, version: toolPkg.constraint}, [osArch], // XXX 'browser' too? true); }); - } catch (err) { - console.log(err); - process.exit(1); + _.each(release.packages, function (pkgVersion, pkgName) { + buildmessage.enterJob({ + title: "downloading package " + pkgName + "@" + pkgVersion + }, function () { + tmpTropo.maybeDownloadPackageForArchitectures( + {packageName: pkgName, version: pkgVersion}, + [osArch], // XXX 'browser' too? + true); + }); + }); + }); + if (messages.hasMessage()) { + process.stderr.write("\n" + messages.formatMessages()); + return 1; } // XXX should we include some sort of preliminary package-metadata as well? diff --git a/tools/compiler.js b/tools/compiler.js index efd0d6207a..2e25c9acd0 100644 --- a/tools/compiler.js +++ b/tools/compiler.js @@ -133,10 +133,11 @@ compiler.eachUsedUnibuild = function ( // PackgeSource), or we could have some kind of cache (the ideal place // for such a cache might be inside the constraint solver, since it // will know how/when to invalidate it). -var determineBuildTimeDependencies = function - (packageSource, constraintSolverOpts) { +var determineBuildTimeDependencies = function (packageSource, + constraintSolverOpts) { var ret = {}; - constraintSolverOpts = constraintSolverOpts || {}; + constraintSolverOpts = constraintSolverOpts || {}; + constraintSolverOpts.ignoreProjectDeps || buildmessage.assertInCapture(); // There are some special cases where we know that the package has no source // files, which means it can't have any interesting build-time @@ -745,6 +746,7 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // disk) that were used by the compilation (the source files you'd have to // ship to a different machine to replicate the build there) compiler.compile = function (packageSource, options) { + buildmessage.assertInCapture(); var sources = []; var pluginWatchSet = packageSource.pluginWatchSet.clone(); var plugins = {}; @@ -910,7 +912,10 @@ var getPluginProviders = function (versions) { // string). Yes, it is possible that multiple versions of some other // package might be build-time dependencies (because of plugins). compiler.getBuildOrderConstraints = function ( - packageSource, constraintSolverOpts) { + packageSource, constraintSolverOpts) { + constraintSolverOpts = constraintSolverOpts || {}; + constraintSolverOpts.ignoreProjectDeps || buildmessage.assertInCapture(); + var versions = {}; // map from package name to version to true var addVersion = function (version, name) { if (name !== packageSource.name) { @@ -949,6 +954,7 @@ compiler.getBuildOrderConstraints = function ( // build-time dependency has changed. compiler.checkUpToDate = function ( packageSource, unipackage, constraintSolverOpts) { + buildmessage.assertInCapture(); if (unipackage.forceNotUpToDate) { return false; diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js index 38cac24a10..1340a0785b 100644 --- a/tools/deploy-galaxy.js +++ b/tools/deploy-galaxy.js @@ -206,7 +206,14 @@ exports.deploy = function (options) { if (! options.starball && ! messages.hasMessages()) { process.stdout.write('Deploying ' + options.app + '. Bundling...\n'); - stats.recordPackages(); + var statsMessages = buildmessage.capture(function () { + stats.recordPackages(); + }); + if (statsMessages.hasMessages()) { + process.stdout.write("Error talking to stats server:\n" + + statsMessages.formatMessages()); + // ... but continue; + } var bundleResult = bundler.bundle({ outputPath: bundlePath, buildOptions: options.buildOptions diff --git a/tools/deploy.js b/tools/deploy.js index d43a40b019..191af29615 100644 --- a/tools/deploy.js +++ b/tools/deploy.js @@ -397,8 +397,16 @@ var bundleAndDeploy = function (options) { if (! messages.hasMessages()) { var bundler = require('./bundler.js'); - if (options.recordPackageUsage) - stats.recordPackages(options.appDir); + if (options.recordPackageUsage) { + var statsMessages = buildmessage.capture(function () { + stats.recordPackages(options.appDir); + }); + if (statsMessages.hasMessages()) { + process.stdout.write("Error talking to stats server:\n" + + statsMessages.formatMessages()); + // ... but continue; + } + } var bundleResult = bundler.bundle({ outputPath: bundlePath, diff --git a/tools/main.js b/tools/main.js index 255bfc5cb6..b09c0e5a24 100644 --- a/tools/main.js +++ b/tools/main.js @@ -313,11 +313,15 @@ var springboard = function (rel, releaseOverride) { // XXX split better try { - tropohouse.default.maybeDownloadPackageForArchitectures( - {packageName: toolsPkg, version: toolsVersion}, - [archinfo.host()], - true /* print downloading message */ - ); + var messages = buildmessage.capture({ + title: "downloading tools package " + toolsPkg + "@" + toolsVersion + }, function () { + tropohouse.default.maybeDownloadPackageForArchitectures( + {packageName: toolsPkg, version: toolsVersion}, + [archinfo.host()], + true /* print downloading message */ + ); + }); } catch (err) { // We have failed to download the tool that we are supposed to springboard // to! That's bad. Let's exit. @@ -327,8 +331,12 @@ var springboard = function (rel, releaseOverride) { rel.getToolsPackageAtVersion() + "\n"); process.exit(1); } - - // XXX support warehouse too + if (messages.hasMessages()) { + process.stderr.write( + "Could not springboard to release: " + rel.name + ".\n" + + messages.formatMessages()); + process.exit(1); + } var packagePath = tropohouse.default.packagePath(toolsPkg, toolsVersion); var toolUnipackage = new unipackage.Unipackage; diff --git a/tools/package-cache.js b/tools/package-cache.js index 6248390c4c..0e48428d9c 100644 --- a/tools/package-cache.js +++ b/tools/package-cache.js @@ -71,6 +71,7 @@ _.extend(PackageCache.prototype, { // loadPackageAtPath() is called for them, see refresh(). loadPackageAtPath: function (name, loadPath) { var self = this; + buildmessage.assertInCapture(); // We need to build and load both the test and normal package, which, // frequently means 2 packages per directory/loadPath. Rather than @@ -206,6 +207,7 @@ _.extend(PackageCache.prototype, { // ignore when scanning for source files.) loadAppAtPath: function (appDir, ignoreFiles) { var self = this; + buildmessage.assertInCapture(); var packageSource = new PackageSource; packageSource.initFromAppDir(appDir, ignoreFiles); diff --git a/tools/package-loader.js b/tools/package-loader.js index 497a1c348b..1d20b4ae6d 100644 --- a/tools/package-loader.js +++ b/tools/package-loader.js @@ -36,6 +36,7 @@ _.extend(exports.PackageLoader.prototype, { // XXX rename to throwOnNotFound getPackage: function (name, options) { var self = this; + buildmessage.assertInCapture(); options = options || {}; if (options.throwOnError === undefined) { @@ -87,6 +88,7 @@ _.extend(exports.PackageLoader.prototype, { // getPackageLoadPath / getPackageVersionLoadPath? getLoadPathForPackage: function (name) { var self = this; + buildmessage.assertInCapture(); if (self.uniloadDir) { var packagePath = path.join(self.uniloadDir, name); @@ -116,6 +118,7 @@ _.extend(exports.PackageLoader.prototype, { // that package at the right architecture. getUnibuild: function (packageName, arch) { var self = this; + buildmessage.assertInCapture(); var pkg = self.getPackage(packageName, { throwOnError: true }); return pkg.getUnibuildAtArch(arch); diff --git a/tools/project.js b/tools/project.js index 9ee5a31827..7cabd810d1 100644 --- a/tools/project.js +++ b/tools/project.js @@ -158,6 +158,7 @@ _.extend(Project.prototype, { // the package loader for this project. This WILL REWRITE THE VERSIONS FILE. _ensureDepsUpToDate : function () { var self = this; + buildmessage.assertInCapture(); // To calculate project dependencies, we need to know what release we are // on, but to do that, we need to have a rootDirectory. So, we initialize @@ -415,6 +416,7 @@ _.extend(Project.prototype, { // null if the package is unconstrained. getCurrentCombinedConstraints : function () { var self = this; + buildmessage.assertInCapture(); self._ensureDepsUpToDate(); return self.combinedConstraints; }, @@ -433,6 +435,7 @@ _.extend(Project.prototype, { // Returns an object mapping package name to its string version. getVersions : function () { var self = this; + buildmessage.assertInCapture(); self._ensureDepsUpToDate(); return self.dependencies; }, @@ -450,6 +453,7 @@ _.extend(Project.prototype, { // transitive dependencies. getPackageLoader : function () { var self = this; + buildmessage.assertInCapture(); self._ensureDepsUpToDate(); return self.packageLoader; }, @@ -565,6 +569,7 @@ _.extend(Project.prototype, { // here because this really shouldn't fail (we are just removing things). removePackages : function (names) { var self = this; + buildmessage.assertInCapture(); self._removePackageRecords(names); // Force a recalculation of all the dependencies, and record them to disk. @@ -585,6 +590,7 @@ _.extend(Project.prototype, { setVersions: function (newVersions, options) { var self = this; options = options || {}; + buildmessage.assertInCapture(); var downloaded = self._ensurePackagesExistOnDisk(newVersions); var ret = { @@ -641,6 +647,7 @@ _.extend(Project.prototype, { // that could lead to changes in the versions file. _ensurePackagesExistOnDisk : function (versions, options) { var self = this; + buildmessage.assertInCapture(); options = options || {}; var arch = options.arch || archinfo.host(); var verbose = options.verbose || !self.muted; @@ -681,6 +688,7 @@ _.extend(Project.prototype, { // disk and the operation has failed. addPackages : function (moreDeps, newVersions) { var self = this; + buildmessage.assertInCapture(); // First, we need to make sure that we have downloaded all the packages that // we are going to use. So, go through the versions and call tropohouse to diff --git a/tools/run-app.js b/tools/run-app.js index b4c8afb0d6..c01285e6f4 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -418,8 +418,16 @@ _.extend(AppRunner.prototype, { // Bundle up the app var bundlePath = path.join(self.appDir, '.meteor', 'local', 'build'); - if (self.recordPackageUsage) - stats.recordPackages(self.appDir); + if (self.recordPackageUsage) { + var statsMessages = buildmessage.capture(function () { + stats.recordPackages(self.appDir); + }); + if (statsMessages.hasMessages()) { + process.stdout.write("Error talking to stats server:\n" + + statsMessages.formatMessages()); + // ... but continue; + } + } // Cache the server target because the server will not change inside // a single invocation of _runOnce(). diff --git a/tools/selftest.js b/tools/selftest.js index f30a68cf1e..48336d3d7c 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -66,19 +66,14 @@ var expectThrows = markStack(function (f) { }); var getToolsPackage = function () { + buildmessage.assertInCapture(); // Rebuild the tool package --- necessary because we don't actually // rebuild the tool in the cached version every time. - var messages = buildmessage.capture(function () { - if (catalog.complete.rebuildLocalPackages([toolPackageName]) !== 1) { - throw Error("didn't rebuild meteor-tool?"); - } - }); - if (messages.hasMessages()) { - throw Error(messages.formatMessages()); - }; + if (catalog.complete.rebuildLocalPackages([toolPackageName]) !== 1) { + throw Error("didn't rebuild meteor-tool?"); + } var loader = new packageLoader.PackageLoader({versions: null}); - var toolPackage = loader.getPackage(toolPackageName); - return toolPackage; + return loader.getPackage(toolPackageName); }; // Execute a command synchronously, discarding stderr. @@ -621,12 +616,18 @@ _.extend(Sandbox.prototype, { // be building some packages besides meteor-tool (so that we can // build apps that contain core packages). - var toolPackage = getToolsPackage(); - var toolPackageDirectory = - '.' + toolPackage.version + '.XXX++' - + toolPackage.buildArchitectures(); - toolPackage.saveToPath(path.join(self.warehouse, packagesDirectoryName, - toolPackageName, toolPackageDirectory)); + var toolPackage, toolPackageDirectory; + var messages = buildmessage.capture(function () { + toolPackage = getToolsPackage(); + toolPackageDirectory = '.' + toolPackage.version + '.XXX++' + + toolPackage.buildArchitectures(); + toolPackage.saveToPath(path.join(self.warehouse, packagesDirectoryName, + toolPackageName, toolPackageDirectory)); + }); + if (messages.hasMessages()) { + throw Error(messages.formatMessages()); + } + fs.symlinkSync(toolPackageDirectory, path.join(self.warehouse, packagesDirectoryName, toolPackageName, toolPackage.version)); diff --git a/tools/stats.js b/tools/stats.js index 00519b939e..3dca61b1a2 100644 --- a/tools/stats.js +++ b/tools/stats.js @@ -9,6 +9,7 @@ var project = require("./project.js"); var auth = require("./auth.js"); var ServiceConnection = require("./service-connection.js"); var release = require("./release.js"); +var buildmessage = require("./buildmessage.js"); // The name of the package that you add to your app to opt out of // sending stats. @@ -33,6 +34,7 @@ var optOutPackageName = "package-stats-opt-out"; // that it is pointing to a root directory with an existing // .meteor/versions file. var packageList = function (_currentProjectForTest) { + buildmessage.assertInCapture(); var directDeps = (_currentProjectForTest || project.project).getConstraints(); var versions; @@ -55,6 +57,7 @@ var packageList = function (_currentProjectForTest) { }; var recordPackages = function () { + buildmessage.assertInCapture(); // Before doing anything, look at the app's dependencies to see if the // opt-out package is there; if present, we don't record any stats. var packages = packageList(); diff --git a/tools/tests/old/test-bundler-options.js b/tools/tests/old/test-bundler-options.js index 597603b4fa..1c954296a6 100644 --- a/tools/tests/old/test-bundler-options.js +++ b/tools/tests/old/test-bundler-options.js @@ -8,6 +8,7 @@ var files = require('../../files.js'); var catalog = require('../../catalog.js'); var project = require('../../project.js'); var compiler = require('../../compiler.js'); +var buildmessage = require('../../buildmessage.js'); // an empty app. notably this app has no .meteor/release file. var emptyAppDir = path.join(__dirname, 'empty-app'); @@ -41,7 +42,13 @@ var runTest = function () { }; setAppDir(emptyAppDir); - var loader = project.project.getPackageLoader(); + var loader; + var messages = buildmessage.capture(function () { + loader = project.project.getPackageLoader(); + }); + if (messages.hasMessages()) { + throw Error("failed to get package loader: " + messages.formatMessages()); + } console.log("nodeModules: 'skip'"); assert.doesNotThrow(function () { diff --git a/tools/tests/report-stats.js b/tools/tests/report-stats.js index 8ec762a05e..0e437812bb 100644 --- a/tools/tests/report-stats.js +++ b/tools/tests/report-stats.js @@ -9,6 +9,7 @@ var testUtils = require('../test-utils.js'); var stats = require('../stats.js'); var Sandbox = selftest.Sandbox; var project = require('../project.js'); +var buildmessage = require('../buildmessage.js'); var testStatsServer = "https://test-package-stats.meteor.com"; process.env.METEOR_PACKAGE_STATS_SERVER_URL = testStatsServer; @@ -116,7 +117,7 @@ selftest.define("report-stats", ["slow", "net"], function () { // package usage stats var usage = fetchPackageUsageForApp(identifier); selftest.expectEqual(_.sortBy(usage.packages, "name"), - _.sortBy(stats.packageList(sandboxProject), "name")); + _.sortBy(packageList(sandboxProject), "name")); // Check that the direct dependency was recorded as such. _.each(usage.packages, function (package) { @@ -131,7 +132,7 @@ selftest.define("report-stats", ["slow", "net"], function () { selftest.expectEqual(appPackages.appId, identifier); selftest.expectEqual(appPackages.userId, null); selftest.expectEqual(_.sortBy(appPackages.packages, "name"), - _.sortBy(stats.packageList(sandboxProject), "name")); + _.sortBy(packageList(sandboxProject), "name")); checkMeta(appPackages, sessionId, useFakeRelease); // now bundle again while logged in. verify that the stats server @@ -183,7 +184,7 @@ selftest.define("report-stats", ["slow", "net"], function () { appPackages = stats.getPackagesForAppIdInTest(sandboxProject); selftest.expectEqual(appPackages.userId, testUtils.getUserId(s)); selftest.expectEqual(_.sortBy(appPackages.packages, "name"), - _.sortBy(stats.packageList(sandboxProject), "name")); + _.sortBy(packageList(sandboxProject), "name")); } ); }); @@ -257,3 +258,14 @@ var getClientAddress = function () { var stats = testUtils.ddpConnect(testStatsServer); return stats.call("getClientAddress"); }; + +var packageList = function (proj) { + var ret; + var messages = buildmessage.capture(function () { + ret = stats.packageList(proj); + }); + if (messages.hasMessages()) { + selftest.fail(messages.formatMessages()); + } + return ret; +}; diff --git a/tools/tropohouse.js b/tools/tropohouse.js index 47b4ac23b4..937e9edc66 100644 --- a/tools/tropohouse.js +++ b/tools/tropohouse.js @@ -13,6 +13,7 @@ var archinfo = require('./archinfo.js'); var catalog = require('./catalog.js'); var Unipackage = require('./unipackage.js').Unipackage; var config = require('./config.js'); +var buildmessage = require('./buildmessage.js'); exports.Tropohouse = function (root, catalog) { var self = this; @@ -87,6 +88,7 @@ _.extend(exports.Tropohouse.prototype, { // get around to implement cross-linking (which is the point) maybeDownloadPackageForArchitectures: function (versionInfo, requiredArches, verbose) { var self = this; + buildmessage.assertInCapture(); var packageName = versionInfo.packageName; var version = versionInfo.version; diff --git a/tools/unipackage.js b/tools/unipackage.js index cb5958b3a9..3ddafa6d4f 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -186,6 +186,8 @@ _.extend(Unibuild.prototype, { // If the optional `filter` function is provided, then we will only load // packages for which `filter(packageName, version)` returns truthy. var getLoadedPackageVersions = function (versions, filter) { + buildmessage.assertInCapture(); + var result = {}; var loader = new packageLoader.PackageLoader({ versions: versions }); @@ -655,6 +657,7 @@ _.extend(Unipackage.prototype, { var self = this; var outputPath = outputDir; options = options || {}; + buildmessage.assertInCapture(); var builder = new Builder({ outputPath: outputPath }); @@ -906,6 +909,7 @@ _.extend(Unipackage.prototype, { _writeTool: function (builder) { var self = this; + buildmessage.assertInCapture(); var pathsToCopy = files.runGitInCheckout( 'ls-tree', @@ -970,8 +974,8 @@ _.extend(Unipackage.prototype, { // Returns the build id as a hex string. getBuildIdentifier: function (options) { var self = this; - options = options || {}; + buildmessage.assertInCapture(); // Gather all the direct dependencies (that provide plugins) and // plugin dependencies' versions and organize them into arrays. We @@ -1068,6 +1072,7 @@ _.extend(Unipackage.prototype, { // `getBuildIdentifier`. addBuildIdentifierToVersion: function (options) { var self = this; + buildmessage.assertInCapture(); self.version = self.version + "+" + self.getBuildIdentifier(options); } diff --git a/tools/updater.js b/tools/updater.js index 3d4cf0bcac..f26c79c459 100644 --- a/tools/updater.js +++ b/tools/updater.js @@ -12,6 +12,7 @@ var catalog = require('./catalog.js'); var archinfo = require('./archinfo.js'); var unipackage = require('./unipackage.js'); var utils = require('./utils.js'); +var buildmessage = require('./buildmessage.js'); /** * Check to see if an update is available. If so, download and install @@ -147,21 +148,32 @@ var updateMeteorToolSymlink = function () { // symlink points to. Let's make sure we have that release on disk, // and then update the symlink. try { - tropohouse.default.maybeDownloadPackageForArchitectures( - {packageName: latestReleaseToolPackage, - version: latestReleaseToolVersion}, - [archinfo.host()]); - - _.each(latestRelease.packages, function (version, packageName) { - tropohouse.default.maybeDownloadPackageForArchitectures( - { packageName: packageName, version: version }, - ['browser', archinfo.host()]); + var messages = buildmessage.capture(function () { + buildmessage.enterJob({ + title: "downloading tool package " + latestRelease.tool + }, function () { + tropohouse.default.maybeDownloadPackageForArchitectures( + {packageName: latestReleaseToolPackage, + version: latestReleaseToolVersion}, + [archinfo.host()], + true); + }); + _.each(latestRelease.packages, function (pkgVersion, pkgName) { + buildmessage.enterJob({ + title: "downloading package " + pkgName + "@" + pkgVersion + }, function () { + tropohouse.default.maybeDownloadPackageForArchitectures( + {packageName: pkgName, version: pkgVersion}, + [archinfo.host()], + true); + }); + }); }); } catch (err) { - console.log("Could not download latest release:", - latestRelease.track + "@" + latestRelease.version); - // Return, since we are running in the background. - return; + return; // since we are running in the background. + } + if (messages.hasMessages()) { + return; // since we are running in the background } var toolUnipackage = new unipackage.Unipackage; @@ -177,9 +189,6 @@ var updateMeteorToolSymlink = function () { if (!toolRecord) throw Error("latest release has no tool?"); - console.log("XXX updating tool symlink for", - latestReleaseVersion.track + "@" + latestReleaseVersion.version); - tropohouse.default.replaceLatestMeteorSymlink( path.join(relativeToolPath, toolRecord.path, 'meteor')); } From 1976d4f4f1e978b4b4495cc0d574b32d61d2c715 Mon Sep 17 00:00:00 2001 From: ekatek Date: Thu, 31 Jul 2014 00:20:33 -0700 Subject: [PATCH 095/126] different error code for server errors --- tools/package-client.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/package-client.js b/tools/package-client.js index c0a364b3ab..6dc2e95d40 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -586,7 +586,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) }); } catch (err) { process.stderr.write(err.message + "\n"); - return 1; + return 3; } } @@ -620,7 +620,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) var uploadInfo = conn.call('createPackageVersion', uploadRec); } catch (err) { process.stderr.write("ERROR " + err.message + "\n"); - return 1; + return 3; } // XXX If package version already exists, print a nice error message @@ -638,7 +638,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) treeHash: sourceBundleResult.treeHash }); } catch (err) { process.stderr.write("ERROR " + err.message + "\n"); - return 1; + return 3; } } From 722ebace9adc1c2f6b55ad5fb91818bedb386955 Mon Sep 17 00:00:00 2001 From: ekatek Date: Thu, 31 Jul 2014 12:14:01 -0700 Subject: [PATCH 096/126] don't just crash on unknown packages when changing admin --- tools/commands-packages.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 51249e1437..7b5c10b07c 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -48,6 +48,11 @@ var getReleaseOrPackageRecord = function(name) { // record: package or track record // action: string for error handling var checkAuthorizedPackageMaintainer = function (record, action) { + if (!record) { + process.stderr.write("This package does not exist.\n"); + return 1; + } + var authorized = _.indexOf( _.pluck(record.maintainers, 'username'), auth.loggedInUsername()); if (authorized == -1) { From 8bcbd65344b0014aedeb700db17ad7f1c8e013a6 Mon Sep 17 00:00:00 2001 From: Matthew Arbesfeld Date: Thu, 31 Jul 2014 13:48:50 -0700 Subject: [PATCH 097/126] Separate "browser" target into web.browser/cordova Cordova projects often have a different set of files than web targets, so we would like to be able to target different client architectures in our bundles. Ideally, we allow the user to use arbitrary client architectures - but this patch is a step in the right direction by abstracting out more of the hard coded "browser"/"os" lines. We accomplish this separation in a backwards compatible way by allowing api.___ commands to target a "client" architecture. For example, api.addFiles('a.js', 'client') adds 'a.js' to both the 'client.browser' and 'client.cordova' targets. Effects on 0.9 packaging stuff: packages don't have to change, but the "data.json" file in ".meteor0" has "browser" in some places. We think we have to fix the troposphere code where this data.json is created. Some plugins will also be backwards-incompatible with this change, since many have a "clientArch.matches("browser")" line in the plugin code. Ideally, we fix plugins so that this stops being an issue, but for now package authors can just patch that line. At the compiled (unipackage) level the new names are 'web.browser' and 'web.cordova', replacing 'browser'. In package.js, the new names are 'client.browser' and 'client.cordova', serving as an adjunct to 'client'. --- .../constraint-solver-tests.js | 6 +- .../constraint-solver/constraint-solver.js | 62 +++++---- packages/less/plugin/compile-less.js | 2 +- packages/meteor/plugin/basic-file-types.js | 2 +- packages/star-translate/translator.js | 6 +- packages/stylus/plugin/compile-stylus.js | 2 +- .../templating/plugin/compile-templates.js | 3 +- packages/webapp/webapp_server.js | 2 +- tools/bundler.js | 65 +++++---- tools/commands-packages.js | 24 ++-- tools/commands.js | 4 +- tools/compiler.js | 28 ++-- tools/package-source.js | 130 +++++++++++------- tools/project.js | 4 +- tools/selftest.js | 21 ++- tools/tests/logs-mongo-auth.js | 1 + tools/tests/old/test-bundler-assets.js | 4 +- tools/tests/old/test-bundler-options.js | 2 +- tools/tropohouse.js | 2 +- tools/unipackage.js | 6 +- 20 files changed, 227 insertions(+), 149 deletions(-) diff --git a/packages/constraint-solver/constraint-solver-tests.js b/packages/constraint-solver/constraint-solver-tests.js index 68c7554164..4284e30e8e 100644 --- a/packages/constraint-solver/constraint-solver-tests.js +++ b/packages/constraint-solver/constraint-solver-tests.js @@ -17,7 +17,7 @@ var insertVersion = function (name, version, ecv, deps) { references: [ { arch: "os", targetSlice: "main", weak: false, implied: false, unordered: false }, - { arch: "browser", targetSlice: "main", weak: false, + { arch: "web", targetSlice: "main", weak: false, implied: false, unordered: false }] }; }); @@ -25,7 +25,7 @@ var insertVersion = function (name, version, ecv, deps) { earliestCompatibleVersion: ecv, dependencies: constructedDeps }); Builds.insert({ packageName: name, version: version, - buildArchitectures: "browser+os" }); + buildArchitectures: "web+os" }); }; insertVersion("sparky-forms", "1.1.2", "1.0.0", {"forms": "=1.0.1", "sparkle": "=2.1.1"}); insertVersion("sparky-forms", "1.0.0", "1.0.0", {"awesome-dropdown": "=1.4.0"}); @@ -505,7 +505,7 @@ function getCatalogStub (gems) { packageVersion.dependencies[name] = { constraint: convertConstraints(constraints)[0], // XXX pick first one only references: [{ - "arch": "browser" + "arch": "web" }, { "arch": "os" }] }; diff --git a/packages/constraint-solver/constraint-solver.js b/packages/constraint-solver/constraint-solver.js index d769b3030f..49ff0aa0e7 100644 --- a/packages/constraint-solver/constraint-solver.js +++ b/packages/constraint-solver/constraint-solver.js @@ -1,5 +1,12 @@ var semver = Npm.require('semver'); +// Copied from archinfo.matches() in tools/ +var archMatches = function (arch, baseArch) { + return arch.substr(0, baseArch.length) === baseArch && + (arch.length === baseArch.length || + arch.substr(baseArch.length, 1) === "."); +}; + ConstraintSolver = {}; // catalog is a catalog.Catalog object. We have to pass this in because @@ -60,9 +67,10 @@ ConstraintSolver.PackagesResolver.prototype._loadPackageInfo = function ( var unibuilds = {}; // XXX in theory there might be different archs but in practice they are - // always "os" and "browser". Fix this once we actually have different - // archs used. - _.each(["os", "browser"], function (arch) { + // always "os", "web.browser" and "web.cordova". Fix this once we + // actually have different archs used. + var allArchs = ["os", "web.browser", "web.cordova"]; + _.each(allArchs, function (arch) { var unitName = packageName + "#" + arch; unibuilds[unitName] = new ConstraintSolver.UnitVersion( unitName, version, versionDef.earliestCompatibleVersion); @@ -73,24 +81,28 @@ ConstraintSolver.PackagesResolver.prototype._loadPackageInfo = function ( self._ensurePackageInfoLoaded(depName); _.each(dep.references, function (ref) { - var unitName = packageName + "#" + ref.arch; - var unitVersion = unibuilds[unitName]; + _.each(allArchs, function (arch) { + if (archMatches(arch, ref.arch)) { + var unitName = packageName + "#" + arch; + var unitVersion = unibuilds[unitName]; - if (! unitVersion) - throw new Error("A non-standard arch " + ref.arch + " for package " + packageName); + if (! unitVersion) + throw new Error("A non-standard arch " + arch + " for package " + packageName); - var targetUnitName = depName + "#" + ref.arch; + var targetUnitName = depName + "#" + arch; - // Add the dependency if needed - if (! ref.weak) - unitVersion.addDependency(targetUnitName); + // Add the dependency if needed + if (! ref.weak) + unitVersion.addDependency(targetUnitName); - // Add a constraint if such exists - if (dep.constraint && dep.constraint !== "none") { - var constraint = - self.resolver.getConstraint(targetUnitName, dep.constraint); - unitVersion.addConstraint(constraint); - } + // Add a constraint if such exists + if (dep.constraint && dep.constraint !== "none") { + var constraint = + self.resolver.getConstraint(targetUnitName, dep.constraint); + unitVersion.addConstraint(constraint); + } + } + }); }); }); @@ -168,10 +180,11 @@ ConstraintSolver.PackagesResolver.prototype.resolve = function ( } // split every package name to one or more archs belonging to that package - // (["foobar"] => ["foobar#os", "foobar#browser"]) - // XXX for now just put #os and #browser + // (["foobar"] => ["foobar#os", "foobar#web.browser", ...]) + // XXX for now just hardcode in all of the known architectures options.upgrade = _.filter(_.flatten(_.map(options.upgrade, function (packageName) { - return [packageName + "#os", packageName + "#browser"]; + return [packageName + "#os", packageName + "#web.browser", + packageName + "#web.cordova"]; })), _.identity); var dc = self._splitDepsToConstraints(dependencies, constraints); @@ -202,7 +215,7 @@ ConstraintSolver.PackagesResolver.prototype.resolve = function ( var resultChoices = {}; _.each(res, function (uv) { // Since we don't yet define the interface for a an app to depend only on - // certain unibuilds of the packages (like only browser unibuilds) and we know + // certain unibuilds of the packages (like only web unibuilds) and we know // that each unibuild weakly depends on other sibling unibuilds of the same // version, we can safely output the whole package for each unibuild in the // result. @@ -245,7 +258,7 @@ ConstraintSolver.PackagesResolver.prototype.propagateExactDeps = }; // takes dependencies and constraints and rewrites the names from "foo" to -// "foo#os" and "foo#browser" +// "foo#os" and "foo#web.browser" and "foo#web.cordova" // XXX right now creates a dependency for every unibuild it can find ConstraintSolver.PackagesResolver.prototype._splitDepsToConstraints = function (inputDeps, inputConstraints) { @@ -280,8 +293,9 @@ ConstraintSolver.PackagesResolver.prototype._unibuildsForPackage = var self = this; var unibuildPrefix = packageName + "#"; var unibuilds = []; - // XXX hardcode os and browser - _.each(["os", "browser"], function (arch) { + // XXX hardcode all common architectures assuming that every package has the + // same set of architectures. + _.each(["os", "web.browser", "web.cordova"], function (arch) { if (self.resolver.unitsVersions[unibuildPrefix + arch]) unibuilds.push(unibuildPrefix + arch); }); diff --git a/packages/less/plugin/compile-less.js b/packages/less/plugin/compile-less.js index 7eccb5e42f..fbbf1e4eea 100644 --- a/packages/less/plugin/compile-less.js +++ b/packages/less/plugin/compile-less.js @@ -5,7 +5,7 @@ var Future = Npm.require('fibers/future'); Plugin.registerSourceHandler("less", function (compileStep) { // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('browser')) { + if (! compileStep.archMatches('web')) { // XXX in the future, might be better to emit some kind of a // warning if a stylesheet is included on the server, rather than // silently ignoring it. but that would mean you can't stick .css diff --git a/packages/meteor/plugin/basic-file-types.js b/packages/meteor/plugin/basic-file-types.js index 3c87328198..5cadb96ab3 100644 --- a/packages/meteor/plugin/basic-file-types.js +++ b/packages/meteor/plugin/basic-file-types.js @@ -4,7 +4,7 @@ Plugin.registerSourceHandler("css", function (compileStep) { // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('browser')) { + if (! compileStep.archMatches('web')) { // XXX in the future, might be better to emit some kind of a // warning if a stylesheet is included on the server, rather than // silently ignoring it. but that would mean you can't stick .css diff --git a/packages/star-translate/translator.js b/packages/star-translate/translator.js index bff404d034..a0ea3d71f2 100644 --- a/packages/star-translate/translator.js +++ b/packages/star-translate/translator.js @@ -39,8 +39,8 @@ StarTranslator._translate = function (bundlePath) { "builtBy": "Star translator", "programs": [ { - "name": "client", - "arch": "browser", + "name": "web.browser", + "arch": "web.browser", "path": "client.json" }, { @@ -99,7 +99,7 @@ StarTranslator._writeClientProg = function (bundlePath, clientProgPath) { "app.json"), 'utf8')); var clientManifest = { - "format": "browser-program-pre1", + "format": "web-program-pre1", "manifest": origClientManifest.manifest, // XXX Haven't updated this for the app.html -> head/body change, but // surely we don't need to because code in pre-star apps doesn't diff --git a/packages/stylus/plugin/compile-stylus.js b/packages/stylus/plugin/compile-stylus.js index 7e3c7372e5..556db16b64 100644 --- a/packages/stylus/plugin/compile-stylus.js +++ b/packages/stylus/plugin/compile-stylus.js @@ -6,7 +6,7 @@ var Future = Npm.require('fibers/future'); Plugin.registerSourceHandler("styl", function (compileStep) { // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('browser')) { + if (! compileStep.archMatches('web')) { // XXX in the future, might be better to emit some kind of a // warning if a stylesheet is included on the server, rather than // silently ignoring it. but that would mean you can't stick .css diff --git a/packages/templating/plugin/compile-templates.js b/packages/templating/plugin/compile-templates.js index b364737c2e..d91ad23360 100644 --- a/packages/templating/plugin/compile-templates.js +++ b/packages/templating/plugin/compile-templates.js @@ -2,8 +2,7 @@ var path = Npm.require('path'); var doHTMLScanning = function (compileStep, htmlScanner) { - // XXX use archinfo rather than rolling our own - if (! compileStep.arch.match(/^browser(\.|$)/)) + if (! compileStep.archMatches("web")) // XXX might be nice to throw an error here, but then we'd have to // make it so that packages.js ignores html files that appear in // the server directories in an app tree.. or, it might be nice to diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index af460e1ffe..bbfd8118db 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -417,7 +417,7 @@ var runWebAppServer = function () { __meteor_bootstrap__.configJson.client); clientDir = path.dirname(clientJsonPath); clientJson = JSON.parse(readUtf8FileSync(clientJsonPath)); - if (clientJson.format !== "browser-program-pre1") + if (clientJson.format !== "web-program-pre1") throw new Error("Unsupported format for client assets: " + JSON.stringify(clientJson.format)); diff --git a/tools/bundler.js b/tools/bundler.js index 0a561ee442..e2cf9545a5 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -51,19 +51,19 @@ // really the build tool can lay out the star however it wants. // // -// == Format of a program when arch is "browser.*" == +// == Format of a program when arch is "web.*" == // // Standard: // // /program.json // -// - format: "browser-program-pre1" for this version +// - format: "web-program-pre1" for this version // // - manifest: array of resources to serve with HTTP, each an object: // - path: path of file relative to program.json // - where: "client" // - type: "js", "css", or "asset" -// - cacheable: is it safe to ask the browser to cache this file (boolean) +// - cacheable: is it safe to ask the client to cache this file (boolean) // - url: relative url to download the resource, includes cache busting // parameter when used // - size: size of file in bytes @@ -387,7 +387,7 @@ var Target = function (options) { // PackageLoader to use for resolving package dependenices. self.packageLoader = options.packageLoader; - // Something like "browser.w3c" or "os" or "os.osx.x86_64" + // Something like "web.browser" or "os" or "os.osx.x86_64" self.arch = options.arch; // All of the Unibuilds that are to go into this target, in the order @@ -411,7 +411,7 @@ var Target = function (options) { self.nodeModulesDirectories = {}; // Static assets to include in the bundle. List of File. - // For browser targets, these are served over HTTP. + // For client targets, these are served over HTTP. self.asset = []; }; @@ -579,7 +579,7 @@ _.extend(Target.prototype, { _emitResources: function () { var self = this; - var isBrowser = archinfo.matches(self.arch, "browser"); + var isWeb = archinfo.matches(self.arch, "web"); var isOs = archinfo.matches(self.arch, "os"); // Copy their resources into the bundle in order @@ -607,7 +607,7 @@ _.extend(Target.prototype, { : stripLeadingSlash(resource.servePath); f.setTargetPathFromRelPath(relPath); - if (isBrowser) + if (isWeb) f.setUrlFromRelPath(resource.servePath); else { unibuildAssets[resource.path] = resource.data; @@ -622,7 +622,7 @@ _.extend(Target.prototype, { return; // already handled if (_.contains(["js", "css"], resource.type)) { - if (resource.type === "css" && ! isBrowser) + if (resource.type === "css" && ! isWeb) // XXX might be nice to throw an error here, but then we'd // have to make it so that package.js ignores css files // that appear in the server directories in an app tree @@ -636,7 +636,7 @@ _.extend(Target.prototype, { var relPath = stripLeadingSlash(resource.servePath); f.setTargetPathFromRelPath(relPath); - if (isBrowser) { + if (isWeb) { f.setUrlFromRelPath(resource.servePath); } @@ -673,8 +673,8 @@ _.extend(Target.prototype, { } if (_.contains(["head", "body"], resource.type)) { - if (! isBrowser) - throw new Error("HTML segments can only go to the browser"); + if (! isWeb) + throw new Error("HTML segments can only go to the client"); self[resource.type].push(resource.data); return; } @@ -760,8 +760,8 @@ var ClientTarget = function (options) { self.head = []; self.body = []; - if (! archinfo.matches(self.arch, "browser")) - throw new Error("ClientTarget targeting something that isn't a browser?"); + if (! archinfo.matches(self.arch, "web")) + throw new Error("ClientTarget targeting something that isn't a client?"); }; util.inherits(ClientTarget, Target); @@ -953,7 +953,7 @@ _.extend(ClientTarget.prototype, { // Control file builder.writeJson('program.json', { - format: "browser-program-pre1", + format: "web-program-pre1", manifest: manifest }); return "program.json"; @@ -1296,7 +1296,7 @@ var JsImageTarget = function (options) { Target.apply(this, arguments); if (! archinfo.matches(self.arch, "os")) - // Conceivably we could support targeting the browser as long as + // Conceivably we could support targeting the client as long as // no native node modules were used. No use case for that though. throw new Error("JsImageTarget targeting something unusual?"); }; @@ -1615,7 +1615,9 @@ var writeSiteArchive = function (targets, outputPath, options) { * * - buildOptions: may include * - minify: minify the CSS and JS assets (boolean, default false) - * - arch: the server architecture to target (defaults to archinfo.host()) + * - serverArch: the server architecture to target + * (defaults to archinfo.host()) + * - webArchs: an array of web archs to target * * - hasCachedBundle: true if we already have a cached bundle stored in * /build. When true, we only build the new client targets in the bundle. @@ -1648,7 +1650,8 @@ exports.bundle = function (options) { if (! release.usingRightReleaseForApp(appDir)) throw new Error("running wrong release for app?"); - var arch = buildOptions.arch || archinfo.host(); + var serverArch = buildOptions.serverArch || archinfo.host(); + var webArchs = buildOptions.webArchs || [ "web.browser" ]; var releaseName = release.current.isCheckout() ? "none" : release.current.name; @@ -1666,7 +1669,7 @@ exports.bundle = function (options) { }, function () { var packageLoader = project.project.getPackageLoader(); var downloaded = project.project._ensurePackagesExistOnDisk( - project.project.dependencies, { arch: arch, verbose: true }); + project.project.dependencies, { serverArch: serverArch, verbose: true }); if (_.keys(downloaded).length !== _.keys(project.project.dependencies).length) { @@ -1677,10 +1680,10 @@ exports.bundle = function (options) { var controlProgram = null; - var makeClientTarget = function (app) { + var makeClientTarget = function (app, webArch) { var client = new ClientTarget({ packageLoader: packageLoader, - arch: "browser" + arch: webArch }); client.make({ @@ -1695,7 +1698,7 @@ exports.bundle = function (options) { var makeBlankClientTarget = function () { var client = new ClientTarget({ packageLoader: packageLoader, - arch: "browser" + arch: "web.browser" }); client.make({ minify: buildOptions.minify, @@ -1708,7 +1711,7 @@ exports.bundle = function (options) { var makeServerTarget = function (app, clientTarget) { var targetOptions = { packageLoader: packageLoader, - arch: arch, + arch: serverArch, releaseName: releaseName }; if (clientTarget) @@ -1738,13 +1741,23 @@ exports.bundle = function (options) { appDir, exports.ignoreFiles); // Client - var client = makeClientTarget(app); - targets.client = client; + _.each(webArchs, function (arch) { + var client = makeClientTarget(app, arch); + targets[arch] = client; + }); + + // Create a browser client if one doesn't exist already. + var browserClient = targets["web.browser"]; + + if (! browserClient) { + browserClient = makeBlankClientTarget(app); + targets["web.browser"] = browserClient; + } // Server if (! options.hasCachedBundle) { - var server = makeServerTarget(app, client); - server.clientTarget = client; + var server = makeServerTarget(app, browserClient); + server.clientTarget = browserClient; targets.server = server; } } diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 51249e1437..1f54fea654 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -186,12 +186,15 @@ main.registerCommand({ } // Warn the user if their package is not good for all architectures. - if (compileResult.unipackage.buildArchitectures() !== "browser+os") { + var allArchs = compileResult.unipackage.buildArchitectures().split('+'); + if (_.any(allArchs, function (arch) { + return arch.match(/^os\./); + })) { process.stdout.write( "\nWARNING: Your package contains binary code and is only compatible with " + archinfo.host() + " architecture.\n" + "Please use publish-for-arch to publish new builds of the package.\n\n"); - }; + } // We are only publishing one package, so we should close the connection, and // then exit with the previous error code. @@ -763,15 +766,18 @@ main.registerCommand({ var myBuilds = _.pluck( catalog.official.getAllBuilds(name, version), 'buildArchitectures'); - // This package does not have different builds, so we don't care. - if (_.isEqual(myBuilds, ["browser+os"])) { - return versionRecord; + // Does this package only have a cross-platform build? + if (myBuilds.length === 1) { + var allArches = myBuilds[0].split('+'); + if (!_.any(allArches, function (arch) { + return arch.match(/^os\./); + })) { + return versionRecord; + } } // This package is only available for some architectures. - var myStringBuilds = ""; - _.each(myBuilds, function (build) { - myStringBuilds = myStringBuilds + build.split('+')[1] + " "; - }); + // XXX show in a more human way? + var myStringBuilds = myBuilds.join(' '); return _.extend({ buildArchitectures: myStringBuilds }, versionRecord); }; diff --git a/tools/commands.js b/tools/commands.js index ea5e91f620..ec420d4e20 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1308,7 +1308,7 @@ main.registerCommand({ }, function () { tmpTropo.maybeDownloadPackageForArchitectures( {packageName: toolPkg.package, version: toolPkg.constraint}, - [osArch], // XXX 'browser' too? + [osArch], // XXX 'web.browser' too? true); }); _.each(release.packages, function (pkgVersion, pkgName) { @@ -1317,7 +1317,7 @@ main.registerCommand({ }, function () { tmpTropo.maybeDownloadPackageForArchitectures( {packageName: pkgName, version: pkgVersion}, - [osArch], // XXX 'browser' too? + [osArch], // XXX 'web.browser' too? true); }); }); diff --git a/tools/compiler.js b/tools/compiler.js index 2e25c9acd0..390b777fd5 100644 --- a/tools/compiler.js +++ b/tools/compiler.js @@ -28,7 +28,7 @@ var compiler = exports; // end up as watched dependencies. (At least for now, packages only used in // target creation (eg minifiers and dev-bundle-fetcher) don't require you to // update BUILT_BY, though you will need to quit and rerun "meteor run".) -compiler.BUILT_BY = 'meteor/11'; +compiler.BUILT_BY = 'meteor/12'; // XXX where should this go? I'll make it a random utility function // for now @@ -457,11 +457,11 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // information. // - pathForSourceMap: If this file is to be included in a source map, // this is the name you should use for it in the map. - // - rootOutputPath: on browser targets, for resources such as + // - rootOutputPath: on web targets, for resources such as // stylesheet and static assets, this is the root URL that // will get prepended to the paths you pick for your output // files so that you get your own namespace, for example - // '/packages/foo'. null on non-browser targets + // '/packages/foo'. null on non-web targets // - fileOptions: any options passed to "api.add_files"; for // use by the plugin. The built-in "js" plugin uses the "bare" // option for files that shouldn't be wrapped in a closure. @@ -473,11 +473,11 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // file as a Buffer. If n is omitted you get the rest of the // file. // - appendDocument({ section: "head", data: "my markup" }) - // Browser targets only. Add markup to the "head" or "body" + // Web targets only. Add markup to the "head" or "body" // section of the document. // - addStylesheet({ path: "my/stylesheet.css", data: "my css", // sourceMap: "stringified json sourcemap"}) - // Browser targets only. Add a stylesheet to the + // Web targets only. Add a stylesheet to the // document. 'path' is a requested URL for the stylesheet that // may or may not ultimately be honored. (Meteor will add // appropriate tags to cause the stylesheet to be loaded. It @@ -498,10 +498,10 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // a closure, so that its vars are shared with other files // in the module. // - addAsset({ path: "my/image.png", data: Buffer }) - // Add a file to serve as-is over HTTP (browser targets) or + // Add a file to serve as-is over HTTP (web targets) or // to include as-is in the bundle (os targets). // This time `data` is a Buffer rather than a string. For - // browser targets, it will be served at the exact path you + // web targets, it will be served at the exact path you // request (concatenated with rootOutputPath). For server // targets, the file can be retrieved by passing path to // Assets.getText or Assets.getBinary. @@ -514,7 +514,7 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // line, column, and func are all optional. // // XXX for now, these handlers must only generate portable code - // (code that isn't dependent on the arch, other than 'browser' + // (code that isn't dependent on the arch, other than 'web' // vs 'os') -- they can look at the arch that is provided // but they can't rely on the running on that particular arch // (in the end, an arch-specific unibuild will be emitted only if @@ -572,9 +572,9 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, return ret; }, appendDocument: function (options) { - if (! archinfo.matches(inputSourceArch.arch, "browser")) + if (! archinfo.matches(inputSourceArch.arch, "web")) throw new Error("Document sections can only be emitted to " + - "browser targets"); + "web targets"); if (options.section !== "head" && options.section !== "body") throw new Error("'section' must be 'head' or 'body'"); if (typeof options.data !== "string") @@ -586,9 +586,9 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, }); }, addStylesheet: function (options) { - if (! archinfo.matches(inputSourceArch.arch, "browser")) + if (! archinfo.matches(inputSourceArch.arch, "web")) throw new Error("Stylesheets can only be emitted to " + - "browser targets"); + "web targets"); if (typeof options.data !== "string") throw new Error("'data' option to addStylesheet must be a string"); sourceIsWatched = true; @@ -605,8 +605,8 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, throw new Error("'data' option to addJavaScript must be a string"); if (typeof options.sourcePath !== "string") throw new Error("'sourcePath' option must be supplied to addJavaScript. Consider passing inputPath."); - if (options.bare && ! archinfo.matches(inputSourceArch.arch, "browser")) - throw new Error("'bare' option may only be used for browser targets"); + if (options.bare && ! archinfo.matches(inputSourceArch.arch, "web")) + throw new Error("'bare' option may only be used for web targets"); sourceIsWatched = true; js.push({ source: options.data, diff --git a/tools/package-source.js b/tools/package-source.js index 1ddf15adb0..6d45967c16 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -92,6 +92,17 @@ var loadOrderSort = function (templateExtensions) { }; }; +// XXX We currently have a 1 to 1 mapping between 'where' and 'arch'. +// In the future, we may let people specify different 'where' and 'arch'. +var mapWhereToArch = function (where) { + if (where === 'server') { + return 'os'; + } else { + // Transform client.* into web.* + return 'web.' + where.split('.').slice(1).join('.'); + } +}; + /////////////////////////////////////////////////////////////////////////////// // SourceArch /////////////////////////////////////////////////////////////////////////////// @@ -200,7 +211,7 @@ var PackageSource = function () { // Path that will be prepended to the URLs of all resources emitted // by this package (assuming they don't end up getting - // concatenated). For non-browser targets, the only effect this will + // concatenated). For non-web targets, the only effect this will // have is to change the actual on-disk paths of the files in the // bundle, for those that care to open up the bundle and look (but // it's still nice to get it right). @@ -293,6 +304,10 @@ var PackageSource = function () { // overall package versions file (if one exists). In the future, we can make // this option transparent to the user in package.js. self.noVersionFile = false; + + // The list of where that we can target. Doesn't include 'client' because + // it is expanded into 'client.*'. + self.allWheres = ['server', 'client.browser', 'client.cordova']; }; @@ -699,16 +714,35 @@ _.extend(PackageSource.prototype, { } // source files used - var sources = {client: [], server: []}; + var sources = {}; // symbols exported - var exports = {client: [], server: []}; + var exports = {}; // packages used and implied (keys are 'package', 'unordered', and // 'weak'). an "implied" package is a package that will be used by a unibuild // which uses us. - var uses = {client: [], server: []}; - var implies = {client: [], server: []}; + var uses = {}; + var implies = {}; + + _.each(self.allWheres, function (where) { + sources[where] = []; + exports[where] = []; + uses[where] = []; + implies[where] = []; + }); + + // Iterates over the list of target archs and calls f(arch) for all archs + // that match an element of 'wheres'. + var forAllMatchingWheres = function (wheres, f) { + _.each(wheres, function (where) { + _.each(self.allWheres, function (matchWhere) { + if (archinfo.matches(matchWhere, where)) { + f(matchWhere); + } + }); + }); + }; // For this old-style, on_use/on_test/where-based package, figure // out its dependencies by calling its on_xxx functions and seeing @@ -717,14 +751,11 @@ _.extend(PackageSource.prototype, { // We have a simple strategy. Call its on_xxx handler with no // 'where', which is what happens when the package is added // directly to an app, and see what files it adds to the client - // and the server. Call the former the client version of the - // package, and the latter the server version. Then, when a - // package is used, include it in both the client and the server - // by default. This simple strategy doesn't capture even 10% of - // the complexity possible with on_use, on_test, and where, but + // and the server. When a package is used, include it in both the client + // and the server by default. This simple strategy doesn't capture even + // 10% of the complexity possible with on_use, on_test, and where, but // probably is sufficient for virtually all packages that actually - // exist in the field, if not every single - // one. #OldStylePackageSupport + // exist in the field, if not every single one. #OldStylePackageSupport if (fileAndDepLoader) { var toArray = function (x) { @@ -733,25 +764,23 @@ _.extend(PackageSource.prototype, { return x ? [x] : []; }; - var allWheres = ['client', 'server']; var toWhereArray = function (where) { if (!(where instanceof Array)) { - where = where ? [where] : allWheres; + where = where ? [where] : self.allWheres; } where = _.uniq(where); - var realWhere = _.intersection(where, allWheres); - if (realWhere.length !== where.length) { - var badWheres = _.difference(where, allWheres); - // avoid using _.each so as to not add more frames to skip - for (var i = 0; i < badWheres.length; ++i) { + _.each(where, function (inputWhere) { + var isMatch = _.any(_.map(self.allWheres, function (actualWhere) { + return archinfo.matches(actualWhere, inputWhere); + })); + if (! isMatch) { buildmessage.error( - "Invalid 'where' argument: '" + badWheres[i] + "'", + "Invalid 'where' argument: '" + inputWhere + "'", // skip toWhereArray in addition to the actual API function - {useMyCaller: 1}); - }; - // recover by using the real ones only - } - return realWhere; + {useMyCaller: 2}); + } + }); + return where; }; var api = { @@ -759,8 +788,9 @@ _.extend(PackageSource.prototype, { // used. Can also take literal package objects, if you have // anonymous packages you want to use (eg, app packages) // - // @param where 'client', 'server', or an array of those. - // The default is ['client', 'server']. + // @param where 'web', 'web.browser', 'web.cordova', 'server', + // or an array of those. + // The default is ['web', 'server']. // // options can include: // @@ -808,7 +838,7 @@ _.extend(PackageSource.prototype, { } _.each(names, function (name) { - _.each(where, function (w) { + forAllMatchingWheres(where, function (w) { uses[w].push(_.extend(utils.splitConstraint(name), { unordered: options.unordered || false, weak: options.weak || false @@ -825,7 +855,7 @@ _.extend(PackageSource.prototype, { where = toWhereArray(where); _.each(names, function (name) { - _.each(where, function (w) { + forAllMatchingWheres(where, function (w) { // We don't allow weak or unordered implies, since the main // purpose of imply is to provide imports and plugins. implies[w].push(utils.splitConstraint(name)); @@ -841,7 +871,7 @@ _.extend(PackageSource.prototype, { where = toWhereArray(where); _.each(paths, function (path) { - _.each(where, function (w) { + forAllMatchingWheres(where, function (w) { var source = {relPath: path}; if (fileOptions) source.fileOptions = fileOptions; @@ -877,8 +907,9 @@ _.extend(PackageSource.prototype, { // Export symbols from this package. // // @param symbols String (eg "Foo") or array of String - // @param where 'client', 'server', or an array of those. - // The default is ['client', 'server']. + // @param where 'web', 'server', 'web.browser', 'web.cordova' + // or an array of those. + // The default is ['web', 'server']. // @param options 'testOnly', boolean. export: function (symbols, where, options) { // Support `api.export("FooTest", {testOnly: true})` without @@ -900,7 +931,7 @@ _.extend(PackageSource.prototype, { // recover by ignoring return; } - _.each(where, function (w) { + forAllMatchingWheres(where, function (w) { exports[w].push({name: symbol, testOnly: !!options.testOnly}); }); }); @@ -918,7 +949,11 @@ _.extend(PackageSource.prototype, { // packages and any remaining handlers. It violates the // principle of least surprise to half-run a handler // and then continue. - sources = {client: [], server: []}; + sources = {}; + _.each(self.allWheres, function (where) { + sources[where] = []; + }); + fileAndDepLoader = null; self.pluginInfo = {}; npmDependencies = null; @@ -944,9 +979,9 @@ _.extend(PackageSource.prototype, { // For all implies and uses, fill in the unspecified dependencies from the // release. - _.each(['server', 'client'], function (label) { - uses[label] = _.map(uses[label], setFromRel); - implies[label] = _.map(implies[label], setFromRel); + _.each(self.allWheres, function (label) { + uses[label] = _.map(uses[label], setFromRel); + implies[label] = _.map(implies[label], setFromRel); }); }; @@ -983,9 +1018,10 @@ _.extend(PackageSource.prototype, { files.rm_recursive(path.join(self.sourceRoot, '.npm', f)); }); - // Create source architectures, one for the server and one for the client. - _.each(["browser", "os"], function (arch) { - var where = (arch === "browser") ? "client" : "server"; + // Create source architectures, one for the server and one for each web + // arch. + _.each(self.allWheres, function (where) { + var arch = mapWhereToArch(where); // Everything depends on the package 'meteor', which sets up // the basic environment) (except 'meteor' itself, and js-analyze @@ -1065,15 +1101,17 @@ _.extend(PackageSource.prototype, { self.sourceRoot = appDir; self.serveRoot = path.sep; - _.each(["client", "server"], function (archName) { + _.each(self.allWheres, function (where) { // Determine used packages var project = require('./project.js').project; var names = project.getConstraints(); - var arch = archName === "server" ? "os" : "browser"; + var arch = mapWhereToArch(where); + // XXX what about /client.browser/* etc, these directories could also + // be for specific client targets. // Create unibuild var sourceArch = new SourceArch(self, { - name: archName, + name: where, arch: arch, uses: _.map(names, utils.dealConstraint) }); @@ -1118,7 +1156,7 @@ _.extend(PackageSource.prototype, { }); var otherUnibuildRegExp = - (archName === "server" ? /^client\/$/ : /^server\/$/); + (where === "server" ? /^client\/$/ : /^server\/$/); // The paths that we've called checkForInfiniteRecursion on. var seenPaths = {}; @@ -1188,7 +1226,7 @@ _.extend(PackageSource.prototype, { // Special case: on the client, JavaScript files in a // `client/compatibility` directory don't get wrapped in a closure. - if (archName === "client" && relPath.match(/\.js$/)) { + if (archinfo.matches(arch, "web") && relPath.match(/\.js$/)) { var clientCompatSubstr = path.sep + 'client' + path.sep + 'compatibility' + path.sep; if ((path.sep + relPath).indexOf(clientCompatSubstr) !== -1) @@ -1198,7 +1236,7 @@ _.extend(PackageSource.prototype, { }); // Now look for assets for this unibuild. - var assetDir = archName === "client" ? "public" : "private"; + var assetDir = archinfo.matches(arch, "web") ? "public" : "private"; var assetDirs = readAndWatchDirectory('', { include: [new RegExp('^' + assetDir + '/$')] }); diff --git a/tools/project.js b/tools/project.js index 7cabd810d1..94cbf7a545 100644 --- a/tools/project.js +++ b/tools/project.js @@ -649,7 +649,7 @@ _.extend(Project.prototype, { var self = this; buildmessage.assertInCapture(); options = options || {}; - var arch = options.arch || archinfo.host(); + var serverArch = options.serverArch || archinfo.host(); var verbose = options.verbose || !self.muted; var downloadedPackages = {}; _.each(versions, function (version, name) { @@ -657,7 +657,7 @@ _.extend(Project.prototype, { try { var available = tropohouse.default.maybeDownloadPackageForArchitectures( packageVersionInfo, - ['browser', arch], + [serverArch], // XXX 'web.browser' too? verbose /* print downloading message */ ); downloadedPackages[name] = version; diff --git a/tools/selftest.js b/tools/selftest.js index 48336d3d7c..afe1cada73 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -339,10 +339,15 @@ _.extend(OutputLog.prototype, { // 'fake-mongod' stub process to be started instead of 'mongod'. The // tellMongo method then becomes available on Runs for controlling // the stub. +// - clients +// - browserstack: true if browserstack clients should be used +// - port: the port that the clients should run on var Sandbox = function (options) { var self = this; - options = options || {}; + // default options + options = _.extend({ clients: {} }, options); + self.root = files.mkdtemp(); self.warehouse = null; @@ -369,13 +374,13 @@ var Sandbox = function (options) { self.clients = [ new PhantomClient({ host: 'localhost', - port: 3000 + port: options.clients.port || 3000 })]; if (options.clients && options.clients.browserstack) { self.clients.push(new BrowserStackClient({ host: 'localhost', - port: 3000 + port: options.clients.port || 3000 })); } @@ -410,6 +415,7 @@ _.extend(Sandbox.prototype, { // }); testWithAllClients: function (f) { var self = this; + var argsArray = _.compact(_.toArray(arguments).slice(1)); if (self.clients.length) { console.log("running test with " + self.clients.length + " client(s)."); @@ -421,7 +427,7 @@ _.extend(Sandbox.prototype, { console.log("testing with " + client.name + "..."); f(new Run(self.execPath, { sandbox: self, - args: [], + args: argsArray, cwd: self.cwd, env: self._makeEnv(), fakeMongo: self.fakeMongo, @@ -528,7 +534,7 @@ _.extend(Sandbox.prototype, { }; self.write(to, contents); }, - + // Delete a file in the sandbox. 'filename' is as in write(). unlink: function (filename) { var self = this; @@ -725,7 +731,7 @@ _.extend(Sandbox.prototype, { // Insert into builds. Assume the package is available for all // architectures. stubCatalog.collections.builds.push({ - buildArchitectures: "browser+os", + buildArchitectures: "web.browser+os", versionId: versionRec._id, build: buildRec.build, _id: utils.randomToken() @@ -1566,5 +1572,6 @@ _.extend(exports, { fail: fail, expectEqual: expectEqual, expectThrows: expectThrows, - getToolsPackage: getToolsPackage + getToolsPackage: getToolsPackage, + execFileSync: execFileSync }); diff --git a/tools/tests/logs-mongo-auth.js b/tools/tests/logs-mongo-auth.js index 5aba7a3939..88751ad0d9 100644 --- a/tools/tests/logs-mongo-auth.js +++ b/tools/tests/logs-mongo-auth.js @@ -43,6 +43,7 @@ var logsOrMongoForApp = function (sandbox, command, appName, options) { run.waitSecs(commandTimeoutSecs); var expectSuccess = selftest.markStack(function () { + run.waitSecs(2); run.match(matchString); run.expectExit(0); }); diff --git a/tools/tests/old/test-bundler-assets.js b/tools/tests/old/test-bundler-assets.js index 75cf9620e7..e4c8d18177 100644 --- a/tools/tests/old/test-bundler-assets.js +++ b/tools/tests/old/test-bundler-assets.js @@ -48,7 +48,7 @@ var runTest = function () { }); var clientManifest = JSON.parse( fs.readFileSync( - path.join(tmpOutputDir, "programs", "client", "program.json") + path.join(tmpOutputDir, "programs", "web.browser", "program.json") ) ); @@ -59,7 +59,7 @@ var runTest = function () { return m.url === file[0]; }); assert(manifestItem); - var diskPath = path.join(tmpOutputDir, "programs", "client", + var diskPath = path.join(tmpOutputDir, "programs", "web.browser", manifestItem.path); assert(fs.existsSync(diskPath)); assert.strictEqual(fs.readFileSync(diskPath, "utf8"), file[1]); diff --git a/tools/tests/old/test-bundler-options.js b/tools/tests/old/test-bundler-options.js index 1c954296a6..1cc05b2325 100644 --- a/tools/tests/old/test-bundler-options.js +++ b/tools/tests/old/test-bundler-options.js @@ -37,7 +37,7 @@ var setAppDir = function (appDir) { var runTest = function () { var readManifest = function (tmpOutputDir) { return JSON.parse(fs.readFileSync( - path.join(tmpOutputDir, "programs", "client", "program.json"), + path.join(tmpOutputDir, "programs", "web.browser", "program.json"), "utf8")).manifest; }; diff --git a/tools/tropohouse.js b/tools/tropohouse.js index 937e9edc66..3cb7dd6469 100644 --- a/tools/tropohouse.js +++ b/tools/tropohouse.js @@ -116,7 +116,7 @@ _.extend(exports.Tropohouse.prototype, { throw e; } if (packageLinkTarget) { - // The symlink will be of the form '.VERSION.RANDOMTOKEN++browser+os', + // The symlink will be of the form '.VERSION.RANDOMTOKEN++web.browser+os', // so this strips off the part before the '++'. // XXX maybe we should just read the unipackage.json instead of // depending on the symlink? diff --git a/tools/unipackage.js b/tools/unipackage.js index 3ddafa6d4f..aaa66863a4 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -156,7 +156,7 @@ _.extend(Unibuild.prototype, { importStubServePath: isApp && '/packages/global-imports.js', prelinkFiles: self.prelinkFiles, packageVariables: self.packageVariables, - includeSourceMapInstructions: archinfo.matches(self.arch, "browser"), + includeSourceMapInstructions: archinfo.matches(self.arch, "web"), name: self.pkg.name || null }); @@ -307,7 +307,7 @@ _.extend(Unipackage.prototype, { _.pluck(self.unibuilds, 'arch').concat(self._toolArchitectures()) ).sort(); // Ensure that our buildArchitectures string does not look like - // browser+os+os.osx.x86_64 + // web+os+os.osx.x86_64 // This would happen if there is an 'os' unibuild but a platform-specific // tool (eg, in meteor-tool). This would confuse catalog.getBuildsForArches // into thinking that it would work for Linux, since the 'os' means @@ -338,7 +338,7 @@ _.extend(Unipackage.prototype, { }, // Return the unibuild of the package to use for a given target architecture - // (eg, 'os.linux.x86_64' or 'browser'), or throw an exception if that + // (eg, 'os.linux.x86_64' or 'web'), or throw an exception if that // packages can't be loaded under these circumstances. getUnibuildAtArch: function (arch) { var self = this; From cee6d7401ee52d4e5e28d2db388bdba9f9021138 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 31 Jul 2014 13:56:19 -0700 Subject: [PATCH 098/126] fix orderKey regexp --- tools/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/utils.js b/tools/utils.js index e801beaae1..2fac0dcce4 100644 --- a/tools/utils.js +++ b/tools/utils.js @@ -228,7 +228,7 @@ exports.timeoutScaleFactor = timeoutScaleFactor; // the prerelease for a given release will sort before it. Because $ sorts // before '.', this means that 1.2 will sort before 1.2.3.) exports.defaultOrderKeyForReleaseVersion = function (v) { - var m = v.match(/^(\d{1,4}(?:\.\d{1,4})*)(?:-([-A-za-z]{1,15})(\d{0,4}))?$/); + var m = v.match(/^(\d{1,4}(?:\.\d{1,4})*)(?:-([-A-Za-z]{1,15})(\d{0,4}))?$/); if (!m) return null; var numberPart = m[1]; From 8adb56b2b34a28c5657072afc7fa1a1a4e1f3c44 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 31 Jul 2014 14:33:23 -0700 Subject: [PATCH 099/126] Fix typo in make-bootstrap-tarballs --- tools/commands.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/commands.js b/tools/commands.js index ec420d4e20..7d7edbb02d 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1322,7 +1322,7 @@ main.registerCommand({ }); }); }); - if (messages.hasMessage()) { + if (messages.hasMessages()) { process.stderr.write("\n" + messages.formatMessages()); return 1; } From bf0e0b61df4b5fd07f054cb51ca02aa092dbc88e Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Thu, 31 Jul 2014 15:05:56 -0700 Subject: [PATCH 100/126] Fix usage of timeoutScaleFactor in test-utils --- tools/test-utils.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/test-utils.js b/tools/test-utils.js index 80858e2520..f959254262 100644 --- a/tools/test-utils.js +++ b/tools/test-utils.js @@ -2,6 +2,7 @@ var _ = require('underscore'); var release = require('./release.js'); var unipackage = require('./unipackage.js'); var config = require('./config.js'); +var utils = require('./utils.js'); var randomString = function (charsCount) { var chars = 'abcdefghijklmnopqrstuvwxyz'; @@ -12,7 +13,7 @@ var randomString = function (charsCount) { return str; }; -exports.accountsCommandTimeoutSecs = 15 * exports.timeoutScaleFactor; +exports.accountsCommandTimeoutSecs = 15 * utils.timeoutScaleFactor; exports.randomString = randomString; From d7594fd72d3743d60d78f2ed0b0a17cfa3ee6b04 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 31 Jul 2014 15:16:52 -0700 Subject: [PATCH 101/126] Code change from packaging-PREVIEW branch --- tools/commands-packages.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 8b5bd6d978..3ad40c5137 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -55,7 +55,7 @@ var checkAuthorizedPackageMaintainer = function (record, action) { var authorized = _.indexOf( _.pluck(record.maintainers, 'username'), auth.loggedInUsername()); - if (authorized == -1) { + if (authorized == -1 && record.name.split(':').length > 1) { process.stderr.write('You are not an authorized maintainer of ' + record.name + ".\n"); process.stderr.write('Only authorized maintainers may ' + action + ".\n"); return 1;; @@ -437,7 +437,10 @@ main.registerCommand({ var auth = require("./auth.js"); var authorized = _.indexOf( _.pluck(trackRecord.maintainers, 'username'), auth.loggedInUsername()); - if (authorized == -1) { + // The split on ':' is a hack to postpone the auth checking for non-prefixed + // releases to the server, which will, in turn use a hack to let MDG members + // publish on non-prefixed releases until we implement organizations. + if (authorized == -1 && relConf.track.split(':').length > 1) { process.stderr.write('\n You are not an authorized maintainer of ' + relConf.track + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions.\n'); return 1; From 93b7a6fb1098ea5bf7147bb9b9cbbc170becf456 Mon Sep 17 00:00:00 2001 From: Justin SB Date: Thu, 31 Jul 2014 19:58:16 -0700 Subject: [PATCH 102/126] Add some more scenarios for benchmark --- examples/unfinished/benchmark/scenarios/scale20.json | 11 +++++++++++ examples/unfinished/benchmark/scenarios/scale40.json | 11 +++++++++++ 2 files changed, 22 insertions(+) create mode 100644 examples/unfinished/benchmark/scenarios/scale20.json create mode 100644 examples/unfinished/benchmark/scenarios/scale40.json diff --git a/examples/unfinished/benchmark/scenarios/scale20.json b/examples/unfinished/benchmark/scenarios/scale20.json new file mode 100644 index 0000000000..db357f2994 --- /dev/null +++ b/examples/unfinished/benchmark/scenarios/scale20.json @@ -0,0 +1,11 @@ + { + "params": { + "numCollections": 1, + "maxAgeSeconds": 60, + "insertsPerSecond": 20, + "updatesPerSecond": 20, + "removesPerSecond": 2, + "documentSize": 128, + "documentNumFields": 2 + } +} diff --git a/examples/unfinished/benchmark/scenarios/scale40.json b/examples/unfinished/benchmark/scenarios/scale40.json new file mode 100644 index 0000000000..53cab32bd1 --- /dev/null +++ b/examples/unfinished/benchmark/scenarios/scale40.json @@ -0,0 +1,11 @@ + { + "params": { + "numCollections": 1, + "maxAgeSeconds": 60, + "insertsPerSecond": 40, + "updatesPerSecond": 40, + "removesPerSecond": 4, + "documentSize": 128, + "documentNumFields": 2 + } +} From 9d5aa280c30df3e0d1d0799fab8a3c9304034e2b Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 1 Aug 2014 00:13:08 -0700 Subject: [PATCH 103/126] invalid release should be a buildmesage, not an error --- tools/package-source.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/package-source.js b/tools/package-source.js index 6d45967c16..a6e20af0b7 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -900,7 +900,7 @@ _.extend(PackageSource.prototype, { releaseRecord = catalog.official.getReleaseVersion( relInf[0], relInf[1], true); if (!releaseRecord) { - throw new Error("Unknown release "+ release); + buildmessage.error("Unknown release "+ release); } }, From 80b09735071e0b522be0fd63d84b9e523b70f490 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 1 Aug 2014 14:58:11 -0700 Subject: [PATCH 104/126] rename `bindToCurrentDataIfIsfunction` to `bindDataContext` (for clarity) --- packages/blaze/lookup.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/blaze/lookup.js b/packages/blaze/lookup.js index 31f5701fde..3bc97a5f6b 100644 --- a/packages/blaze/lookup.js +++ b/packages/blaze/lookup.js @@ -6,7 +6,9 @@ var bindIfIsFunction = function (x, target) { }; }; -var bindToCurrentDataIfIsFunction = function (x) { +// If `x` is a function, binds the value of `this` for that function +// to the current data context. +var bindDataContext = function (x) { if (typeof x === 'function') { return function () { var data = Blaze.getCurrentData(); @@ -22,6 +24,8 @@ var wrapHelper = function (f) { return Blaze.wrapCatchingExceptions(f, 'template helper'); }; +// !!! FIX THIS COMMENT !!! +// // Implements {{foo}} where `name` is "foo" // and `component` is the component the tag is found in // (the lexical "self," on which to look for methods). @@ -45,11 +49,11 @@ Blaze.View.prototype.lookup = function (name, _options) { return Blaze._parentData(name.length - 1, true /*_functionWrapped*/); } else if (template && (name in template)) { - return wrapHelper(bindToCurrentDataIfIsFunction(template[name])); + return wrapHelper(bindDataContext(template[name])); } else if (lookupTemplate && Template.__lookup__(name)) { return Template.__lookup__(name); } else if (UI._globalHelpers[name]) { - return wrapHelper(bindToCurrentDataIfIsFunction(UI._globalHelpers[name])); + return wrapHelper(bindDataContext(UI._globalHelpers[name])); } else { return function () { var isCalledAsFunction = (arguments.length > 0); From 6166abde14b3c49cfb22fdbdca38b933f7275805 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 1 Aug 2014 14:59:48 -0700 Subject: [PATCH 105/126] Reorg observe-sequence (in preparation for {{#each}} over objects) --- packages/observe-sequence/observe_sequence.js | 143 ++++++++++-------- 1 file changed, 77 insertions(+), 66 deletions(-) diff --git a/packages/observe-sequence/observe_sequence.js b/packages/observe-sequence/observe_sequence.js index b4a7dc9b71..29cbc04a5c 100644 --- a/packages/observe-sequence/observe_sequence.js +++ b/packages/observe-sequence/observe_sequence.js @@ -94,78 +94,19 @@ ObserveSequence = { } if (!seq) { - seqArray = []; - diffArray(lastSeqArray, seqArray, callbacks); + seqArray = seqChangedToEmpty(lastSeqArray, callbacks); } else if (seq instanceof Array) { - var idsUsed = {}; - seqArray = _.map(seq, function (item, index) { - var id; - if (typeof item === 'string') { - // ensure not empty, since other layers (eg DomRange) assume this as well - id = "-" + item; - } else if (typeof item === 'number' || - typeof item === 'boolean' || - item === undefined) { - id = item; - } else if (typeof item === 'object') { - id = (item && item._id) || index; - } else { - throw new Error("{{#each}} doesn't support arrays with " + - "elements of type " + typeof item); - } - - var idString = idStringify(id); - if (idsUsed[idString]) { - if (typeof item === 'object' && '_id' in item) - warn("duplicate id " + id + " in", seq); - id = Random.id(); - } else { - idsUsed[idString] = true; - } - - return { _id: id, item: item }; - }); - - diffArray(lastSeqArray, seqArray, callbacks); + seqArray = seqChangedToArray(lastSeqArray, seq, callbacks); } else if (isStoreCursor(seq)) { - var cursor = seq; - seqArray = []; - - var initial = true; // are we observing initial data from cursor? - activeObserveHandle = cursor.observe({ - addedAt: function (document, atIndex, before) { - if (initial) { - // keep track of initial data so that we can diff once - // we exit `observe`. - if (before !== null) - throw new Error("Expected initial data from observe in order"); - seqArray.push({ _id: document._id, item: document }); - } else { - callbacks.addedAt(document._id, document, atIndex, before); - } - }, - changedAt: function (newDocument, oldDocument, atIndex) { - callbacks.changedAt(newDocument._id, newDocument, oldDocument, - atIndex); - }, - removedAt: function (oldDocument, atIndex) { - callbacks.removedAt(oldDocument._id, oldDocument, atIndex); - }, - movedTo: function (document, fromIndex, toIndex, before) { - callbacks.movedTo( - document._id, document, fromIndex, toIndex, before); - } - }); - initial = false; - - // diff the old sequnce with initial data in the new cursor. this will - // fire `addedAt` callbacks on the initial data. - diffArray(lastSeqArray, seqArray, callbacks); - + var result /* [seqArray, activeObserveHandle] */ = + seqChangedToCursor(lastSeqArray, seq, callbacks); + seqArray = result[0]; + activeObserveHandle = result[1]; } else { throw badSequenceError(); } + diffArray(lastSeqArray, seqArray, callbacks); lastSeq = seq; lastSeqArray = seqArray; }); @@ -306,3 +247,73 @@ var diffArray = function (lastSeqArray, seqArray, callbacks) { } }); }; + +seqChangedToEmpty = function (lastSeqArray, callbacks) { + return []; +}; + +seqChangedToArray = function (lastSeqArray, array, callbacks) { + var idsUsed = {}; + var seqArray = _.map(array, function (item, index) { + var id; + if (typeof item === 'string') { + // ensure not empty, since other layers (eg DomRange) assume this as well + id = "-" + item; + } else if (typeof item === 'number' || + typeof item === 'boolean' || + item === undefined) { + id = item; + } else if (typeof item === 'object') { + id = (item && item._id) || index; + } else { + throw new Error("{{#each}} doesn't support arrays with " + + "elements of type " + typeof item); + } + + var idString = idStringify(id); + if (idsUsed[idString]) { + if (typeof item === 'object' && '_id' in item) + warn("duplicate id " + id + " in", array); + id = Random.id(); + } else { + idsUsed[idString] = true; + } + + return { _id: id, item: item }; + }); + + return seqArray; +}; + +seqChangedToCursor = function (lastSeqArray, cursor, callbacks) { + var initial = true; // are we observing initial data from cursor? + var seqArray = []; + + var observeHandle = cursor.observe({ + addedAt: function (document, atIndex, before) { + if (initial) { + // keep track of initial data so that we can diff once + // we exit `observe`. + if (before !== null) + throw new Error("Expected initial data from observe in order"); + seqArray.push({ _id: document._id, item: document }); + } else { + callbacks.addedAt(document._id, document, atIndex, before); + } + }, + changedAt: function (newDocument, oldDocument, atIndex) { + callbacks.changedAt(newDocument._id, newDocument, oldDocument, + atIndex); + }, + removedAt: function (oldDocument, atIndex) { + callbacks.removedAt(oldDocument._id, oldDocument, atIndex); + }, + movedTo: function (document, fromIndex, toIndex, before) { + callbacks.movedTo( + document._id, document, fromIndex, toIndex, before); + } + }); + initial = false; + + return [seqArray, observeHandle]; +}; From e7250a045fd3c6b817eb7562eb97032511a52d88 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 1 Aug 2014 15:01:26 -0700 Subject: [PATCH 106/126] Eliminate `_fetch` on handles returned from `cursor.observe()` This was originally introduced with f1b77fec966c8c97d53628bfd79196afb0c99498, but it looks like all of our tests now pass. (Maybe eliminating `rewind` in b5a0613f85002ee35732c01a27c0d9b1901ed0de made this no longer necessary?) If we find that this commit did break something, let's make sure to add a failing test before reverting. --- packages/minimongo/observe.js | 27 ------------------- packages/observe-sequence/observe_sequence.js | 2 +- 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/packages/minimongo/observe.js b/packages/minimongo/observe.js index 95b92d3d51..191b78333f 100644 --- a/packages/minimongo/observe.js +++ b/packages/minimongo/observe.js @@ -177,32 +177,5 @@ LocalCollection._observeFromObserveChanges = function (cursor, observeCallbacks) var handle = cursor.observeChanges(changeObserver.applyChange); suppressed = false; - if (changeObserver.ordered) { - // Fetches the current list of documents, in order, as an array. Can be - // called at any time. Internal API assumed by the `observe-sequence` - // package (used by Meteor UI for `#each` blocks). Only defined on ordered - // observes (those that listen on `addedAt` or similar). Continues to work - // after `stop()` is called on the handle. - // - // Because we already materialize the full OrderedDict of all documents, it - // seems nice to provide access to the view rather than making the data - // consumer reconstitute it. This gives the consumer a shot at doing - // something smart with the feed like proxying it, since firing callbacks - // like `changed` and `movedTo` basically requires omniscience (knowing old - // and new documents, old and new indices, and the correct value for - // `before`). - // - // NOTE: If called from an observe callback for a certain change, the result - // is *not* guaranteed to be a snapshot of the cursor up to that - // change. This is because the callbacks are invoked before updating docs. - handle._fetch = function () { - var docsArray = []; - changeObserver.docs.forEach(function (doc) { - docsArray.push(transform(EJSON.clone(doc))); - }); - return docsArray; - }; - } - return handle; }; diff --git a/packages/observe-sequence/observe_sequence.js b/packages/observe-sequence/observe_sequence.js index 29cbc04a5c..b736ff88ef 100644 --- a/packages/observe-sequence/observe_sequence.js +++ b/packages/observe-sequence/observe_sequence.js @@ -86,7 +86,7 @@ ObserveSequence = { // more up-to-date information (specifically, the state of the observe // before it was stopped, which may be older than the DB). if (activeObserveHandle) { - lastSeqArray = _.map(activeObserveHandle._fetch(), function (doc) { + lastSeqArray = _.map(lastSeq.fetch(), function (doc) { return {_id: doc._id, item: doc}; }); activeObserveHandle.stop(); From 581b51afcc48d67142fce396a176c41508368e9c Mon Sep 17 00:00:00 2001 From: ekatek Date: Fri, 1 Aug 2014 15:29:19 -0700 Subject: [PATCH 107/126] clean up misc, operations stuff --- scripts/admin/meteor-release-experimental.json | 6 ++++++ scripts/admin/meteor-release-official.json | 7 +++++++ tools/commands.js | 3 ++- tools/help.txt | 8 +------- 4 files changed, 16 insertions(+), 8 deletions(-) create mode 100644 scripts/admin/meteor-release-experimental.json create mode 100644 scripts/admin/meteor-release-official.json diff --git a/scripts/admin/meteor-release-experimental.json b/scripts/admin/meteor-release-experimental.json new file mode 100644 index 0000000000..05653a605d --- /dev/null +++ b/scripts/admin/meteor-release-experimental.json @@ -0,0 +1,6 @@ +{ + "track": "METEOR-CORE", + "version": "0.9.0-rc0", + "recommended": false, + "description": "An experimental release of meteor." +} diff --git a/scripts/admin/meteor-release-official.json b/scripts/admin/meteor-release-official.json new file mode 100644 index 0000000000..63627931ac --- /dev/null +++ b/scripts/admin/meteor-release-official.json @@ -0,0 +1,7 @@ +{ + "track": "METEOR-CORE", + "version": "0.9.0", + "recommended": false, + "official": true, + "description": "Meteor release, supported by MDG." +} diff --git a/tools/commands.js b/tools/commands.js index 7d7edbb02d..5cf368d9df 100644 --- a/tools/commands.js +++ b/tools/commands.js @@ -1362,7 +1362,8 @@ main.registerCommand({ main.registerCommand({ name: 'admin set-banners', minArgs: 1, - maxArgs: 1 + maxArgs: 1, + hidden: true, }, function (options) { var bannersFile = options.args[0]; try { diff --git a/tools/help.txt b/tools/help.txt index 5d24e7a365..1f2a82ebd6 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -531,13 +531,7 @@ Usage: meteor admin change-homepage Change the homepage containing package information. ->>> admin set-breaking -Mark a given version as containing breaking changes. -Usage: stuff - -Mark a given package version as containing breaking changes. - ->>> admin set-banners +>>> admin set-banners Set banners on published releases. Usage: meteor admin set-banners From 112198f3e8c7e1a4915065bdb2959c2553c40c92 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 1 Aug 2014 16:39:43 -0700 Subject: [PATCH 108/126] Don't tell users to add ~/.meteor to PATH Tilde expansion doesn't work in $PATH --- scripts/admin/install-engine.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/admin/install-engine.sh b/scripts/admin/install-engine.sh index 52b7692a62..463ea985ce 100644 --- a/scripts/admin/install-engine.sh +++ b/scripts/admin/install-engine.sh @@ -141,7 +141,7 @@ Couldn't write the launcher script. Please either: (1) Run the following as root: cp ~/.meteor/tools/latest/launch-meteor /usr/bin/meteor - (2) Add ~/.meteor to your path, or + (2) Add "$HOME/.meteor" to your path, or (3) Rerun this command to try again. Then to get started, take a look at 'meteor --help' or see the docs at @@ -153,7 +153,7 @@ else Now you need to do one of the following: - (1) Add ~/.meteor to your path, or + (1) Add "$HOME/.meteor" to your path, or (2) Run this command as root: cp ~/.meteor/tools/latest/launch-meteor /usr/bin/meteor From 6345d92f0757043814d121e3a0431214d2cfe3ef Mon Sep 17 00:00:00 2001 From: MaximDubrovin Date: Sat, 19 Jul 2014 07:13:27 +0600 Subject: [PATCH 109/126] move phantom_script.js setInterval into page.open callback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until I made this I always got only `````` with it's content, `````` was empty. It seems setInterval script was finished earlier then url content was loaded to the page. Maybe because I have subscriptions with response time lower then 100ms so they were ready very quickly — database server in the same data center. http://phantomjs.org/api/webpage/method/open.html --- packages/spiderable/phantom_script.js | 40 +++++++++++++++------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/spiderable/phantom_script.js b/packages/spiderable/phantom_script.js index 00835824fb..a4208d1547 100644 --- a/packages/spiderable/phantom_script.js +++ b/packages/spiderable/phantom_script.js @@ -1,22 +1,26 @@ // 'url' is assigned to in a statement before this. var page = require('webpage').create(); -page.open(url); -setInterval(function() { - var ready = page.evaluate(function () { - if (typeof Meteor !== 'undefined' - && typeof(Meteor.status) !== 'undefined' - && Meteor.status().connected) { - Deps.flush(); - return DDP._allSubscriptionsReady(); - } - return false; - }); - if (ready) { - var out = page.content; - out = out.replace(/]+>(.|\n|\r)*?<\/script\s*>/ig, ''); - out = out.replace('', ''); - console.log(out); +page.open(url, function(status) { + if (status === 'fail') phantom.exit(); - } -}, 100); + setInterval(function() { + var ready = page.evaluate(function () { + if (typeof Meteor !== 'undefined' + && typeof(Meteor.status) !== 'undefined' + && Meteor.status().connected) { + Deps.flush(); + return DDP._allSubscriptionsReady(); + } + return false; + }); + if (ready) { + var out = page.content; + out = out.replace(/]+>(.|\n|\r)*?<\/script\s*>/ig, ''); + out = out.replace('', ''); + console.log(out); + phantom.exit(); + } + }, 100); +}); + From 4b7adf83973c221eaefdaff76c735fb199263154 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 1 Aug 2014 17:04:24 -0700 Subject: [PATCH 110/126] Make events documentation more clear Fixes #2335. See also Issue #2202. --- docs/client/api.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index 876e0b877f..f1ac161bb2 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -2304,8 +2304,8 @@ Example: // Fires when any element with the 'accept' class is clicked 'click .accept': function (event) { ... }, - // Fires when 'accept' is clicked, or a key is pressed - 'keydown, click .accept': function (event) { ... } + // Fires when 'accept' is clicked or focused, or a key is pressed + 'click .accept, focus .accept, keypress': function (event) { ... } } Most events bubble up the document tree from their originating From 7d05640ea04f814ea2b5c3354445973a42f6fcde Mon Sep 17 00:00:00 2001 From: David Glasser Date: Fri, 1 Aug 2014 15:31:27 -0700 Subject: [PATCH 111/126] Make buildmessage fiber-aware Port a simplified version of Meteor.EnvironmentVariable and Meteor.bindEnvironment to fiber-helpers.js to deal with this. Identify uses of fiberHelpers.inFiber and switch them to either fiberHelpers.bindEnvironment (if the callback they are wrapping is semantically "part of" the context that creates the callback) or fiberHelpers.inBareFiber (otherwise). Without this, concurrency was causing the wrong buildmessage message sets and jobs to be active when builds yielded. --- tools/auth.js | 4 +- tools/buildmessage.js | 103 ++++++++++++++---------------- tools/deploy-galaxy.js | 1 - tools/deploy.js | 1 - tools/fiber-helpers.js | 118 ++++++++++++++++++++++++++++------- tools/main.js | 5 +- tools/run-all.js | 2 - tools/run-app.js | 22 ++++--- tools/run-mongo.js | 8 +-- tools/run-updater.js | 10 +-- tools/selftest.js | 17 +++-- tools/stats.js | 1 + tools/tests/releases.js | 10 ++- tools/tests/report-stats.js | 5 +- tools/tests/selftest-test.js | 5 +- tools/updater.js | 1 - tools/watch.js | 2 +- 17 files changed, 195 insertions(+), 120 deletions(-) diff --git a/tools/auth.js b/tools/auth.js index 2fae2bb5fe..ec4066d9b0 100644 --- a/tools/auth.js +++ b/tools/auth.js @@ -110,7 +110,7 @@ var sessionMethodCaller = function (methodName, options) { var conn = options.connection || openAccountsConnection(); conn.apply(methodName, args, fiberHelpers.firstTimeResolver(fut)); if (options.timeout !== undefined) { - var timer = setTimeout(fiberHelpers.inFiber(function () { + var timer = setTimeout(fiberHelpers.bindEnvironment(function () { if (!fut.isResolved()) fut.throw(new Error('Method call timed out')); }), options.timeout); @@ -821,7 +821,7 @@ exports.pollForRegistrationCompletion = function (options) { fut['return'](result); }); - var timer = setTimeout(fiberHelpers.inFiber(function () { + var timer = setTimeout(fiberHelpers.bindEnvironment(function () { if (! fut.isResolved()) { fut['return'](null); } diff --git a/tools/buildmessage.js b/tools/buildmessage.js index 26eeb99ead..863bdde618 100644 --- a/tools/buildmessage.js +++ b/tools/buildmessage.js @@ -1,6 +1,7 @@ var _ = require('underscore'); var files = require('./files.js'); var parseStack = require('./parse-stack.js'); +var fiberHelpers = require('./fiber-helpers.js'); var debugBuild = !!process.env.METEOR_DEBUG_BUILD; @@ -153,8 +154,9 @@ _.extend(MessageSet.prototype, { } }); -var currentMessageSet = null; -var currentJob = null; +var currentMessageSet = new fiberHelpers.EnvironmentVariable; +var currentJob = new fiberHelpers.EnvironmentVariable; +var currentNestingLevel = new fiberHelpers.EnvironmentVariable(0); // Create a new MessageSet, run `f` with that as the current // MessageSet for the purpose of accumulating and recovering from @@ -165,37 +167,29 @@ var currentJob = null; // begin capturing errors. Alternately you may pass `options` // (otherwise optional) and a job will be created for you based on // `options`. -// -// ** Not compatible with multifiber environments ** -// Using a single fiber to block on i/o is fine however. var capture = function (options, f) { - var originalMessageSet = currentMessageSet; var messageSet = new MessageSet; - currentMessageSet = messageSet; - - var originalJob = currentJob; - var job = null; - if (typeof options === "object") { - job = new Job(options); - currentMessageSet.jobs.push(job); - } else { - f = options; // options not actually provided - } - - currentJob = job; - - if (debugBuild) - console.log("START CAPTURE: " + options.title); - - try { - f(); - } finally { - currentMessageSet = originalMessageSet; - currentJob = originalJob; - if (debugBuild) - console.log("DONE CAPTURE: " + options.title); - } + currentMessageSet.withValue(messageSet, function () { + var job = null; + if (typeof options === "object") { + job = new Job(options); + messageSet.jobs.push(job); + } else { + f = options; // options not actually provided + } + currentJob.withValue(job, function () { + var nestingLevel = currentNestingLevel.get(); + currentNestingLevel.withValue(nestingLevel + 1, function () { + debugBuild && console.log("START CAPTURE", nestingLevel, options.title); + try { + f(); + } finally { + debugBuild && console.log("END CAPTURE", nestingLevel, options.title); + } + }); + }); + }); return messageSet; }; @@ -217,29 +211,26 @@ var enterJob = function (options, f) { options = {}; } - if (! currentMessageSet) { + if (! currentMessageSet.get()) { return f(); } var job = new Job(options); - var originalJob = currentJob; - if (originalJob) - originalJob.children.push(job); - currentMessageSet.jobs.push(job); - currentJob = job; + var originalJob = currentJob.get(); + originalJob && originalJob.children.push(job); + currentMessageSet.get().jobs.push(job); - if (debugBuild) - console.log("START: " + options.title); - - try { - var ret = f(); - } finally { - if (debugBuild) - console.log("DONE: " + options.title); - currentJob = originalJob; - } - - return ret; + return currentJob.withValue(job, function () { + var nestingLevel = currentNestingLevel.get(); + return currentNestingLevel.withValue(nestingLevel + 1, function () { + debugBuild && console.log("START", nestingLevel, options.title); + try { + return f(); + } finally { + debugBuild && console.log("DONE", nestingLevel, options.title); + } + }); + }); }; // If not inside a job, return false. Otherwise, return true if any @@ -252,7 +243,7 @@ var jobHasMessages = function () { return !! _.find(job.children, search); }; - return currentJob ? search(currentJob) : false; + return currentJob.get() ? search(currentJob.get()) : false; }; // Given a function f, return a "marked" version of f. The mark @@ -294,7 +285,7 @@ var error = function (message, options) { if (options.downcase) message = message.slice(0,1).toLowerCase() + message.slice(1); - if (! currentJob) + if (! currentJob.get()) throw new Error("Error: " + message); if (options.secondary && jobHasMessages()) @@ -318,7 +309,7 @@ var error = function (message, options) { delete info.useMyCaller; } - currentJob.addMessage(info); + currentJob.get().addMessage(info); }; // Record an exception. The message as well as any file and line @@ -330,7 +321,7 @@ var error = function (message, options) { // actually occurred, rather than the place where the exception was // thrown. var exception = function (error) { - if (! currentJob) { + if (! currentJob.get()) { // XXX this may be the wrong place to do this, but it makes syntax errors in // files loaded via unipackage.load have context. if (error instanceof files.FancySyntaxError) { @@ -345,7 +336,7 @@ var exception = function (error) { if (error instanceof files.FancySyntaxError) { // No stack, because FancySyntaxError isn't a real Error and has no stack // property! - currentJob.addMessage({ + currentJob.get().addMessage({ message: message, file: error.file, line: error.line, @@ -354,7 +345,7 @@ var exception = function (error) { } else { var stack = parseStack.parse(error); var locus = stack[0]; - currentJob.addMessage({ + currentJob.get().addMessage({ message: message, stack: stack, func: locus.func, @@ -366,12 +357,12 @@ var exception = function (error) { }; var assertInJob = function () { - if (! currentJob) + if (! currentJob.get()) throw new Error("Expected to be in a buildmessage job"); }; var assertInCapture = function () { - if (! currentMessageSet) + if (! currentMessageSet.get()) throw new Error("Expected to be in a buildmessage capture"); }; diff --git a/tools/deploy-galaxy.js b/tools/deploy-galaxy.js index 1340a0785b..d82c74277c 100644 --- a/tools/deploy-galaxy.js +++ b/tools/deploy-galaxy.js @@ -5,7 +5,6 @@ var path = require('path'); var fs = require('fs'); var uniload = require('./uniload.js'); var fiberHelpers = require('./fiber-helpers.js'); -var Fiber = require('fibers'); var httpHelpers = require('./http-helpers.js'); var auth = require('./auth.js'); var release = require('./release.js'); diff --git a/tools/deploy.js b/tools/deploy.js index 191af29615..1b3789fccb 100644 --- a/tools/deploy.js +++ b/tools/deploy.js @@ -13,7 +13,6 @@ var config = require('./config.js'); var auth = require('./auth.js'); var utils = require('./utils.js'); var _ = require('underscore'); -var inFiber = require('./fiber-helpers.js').inFiber; var Future = require('fibers/future'); var project = require('./project.js'); var stats = require('./stats.js'); diff --git a/tools/fiber-helpers.js b/tools/fiber-helpers.js index 9b94c4c3e8..03f31ce282 100644 --- a/tools/fiber-helpers.js +++ b/tools/fiber-helpers.js @@ -2,30 +2,6 @@ var _ = require("underscore"); var Fiber = require("fibers"); var Future = require("fibers/future"); -// Use this to wrap callbacks that need to run in a fiber, when -// passing callbacks to functions such as setTimeout that aren't -// callback-aware. (In app code we handle this with Meteor.setTimeout, -// but we don't have such a thing in the tools code yet.) -// -// Specifically, given a function f, this returns a new function that -// returns immediately but also runs f (with any provided arguments) -// in a new fiber. -// -// NOTE: It's probably better to not use callbacks. Instead you can -// use Futures to generate synchronous equivalents. -// -// XXX just suck it up and replace setTimeout and clearTimeout, -// globally, with fiberized versions? will this mess up npm dependencies? -exports.inFiber = function (func) { - return function (/*arguments*/) { - var self = this; - var args = arguments; - new Fiber(function () { - func.apply(self, args); - }).run(); - }; -}; - exports.parallelEach = function (collection, callback, context) { var futures = _.map(collection, function () { var args = _.toArray(arguments); @@ -89,3 +65,97 @@ exports.noYieldsAllowed = function (f) { Fiber.yield = savedYield; } }; + +// Borrowed from packages/meteor/dynamics_nodejs.js +// Used by buildmessage + +exports.nodeCodeMustBeInFiber = function () { + if (!Fiber.current) { + throw new Error("Meteor code must always run within a Fiber. " + + "Try wrapping callbacks that you pass to non-Meteor " + + "libraries with Meteor.bindEnvironment."); + } +}; + +var nextSlot = 0; +exports.EnvironmentVariable = function (defaultValue) { + var self = this; + self.slot = 'slot' + nextSlot++; + self.defaultValue = defaultValue; +}; + +_.extend(exports.EnvironmentVariable.prototype, { + get: function () { + var self = this; + exports.nodeCodeMustBeInFiber(); + + if (!Fiber.current._meteorDynamics) + return self.defaultValue; + if (!_.has(Fiber.current._meteorDynamics, self.slot)) + return self.defaultValue; + return Fiber.current._meteorDynamics[self.slot]; + }, + + withValue: function (value, func) { + var self = this; + exports.nodeCodeMustBeInFiber(); + + if (!Fiber.current._meteorDynamics) + Fiber.current._meteorDynamics = {}; + var currentValues = Fiber.current._meteorDynamics; + + var saved = _.has(currentValues, self.slot) + ? currentValues[self.slot] : self.defaultValue; + currentValues[self.slot] = value; + + try { + return func(); + } finally { + currentValues[self.slot] = saved; + } + } +}); + +// This is like Meteor.bindEnvironment. +// Experimentally, we are NOT including onException or _this in this version. +exports.bindEnvironment = function (func) { + exports.nodeCodeMustBeInFiber(); + + var boundValues = _.clone(Fiber.current._meteorDynamics || {}); + + return function (/* arguments */) { + var self = this; + var args = _.toArray(arguments); + + var runWithEnvironment = function () { + var savedValues = Fiber.current._meteorDynamics; + try { + // Need to clone boundValues in case two fibers invoke this + // function at the same time + Fiber.current._meteorDynamics = _.clone(boundValues); + return func.apply(self, args); + } finally { + Fiber.current._meteorDynamics = savedValues; + } + }; + + if (Fiber.current) + return runWithEnvironment(); + Fiber(runWithEnvironment).run(); + }; +}; + +// An alternative to bindEnvironment for the rare case where you +// want the callback you're passing to some Node function to start +// a new fiber but *NOT* to inherit the current environment. +// Eg, if you are trying to do the equivalent of start a background +// thread. +exports.inBareFiber = function (func) { + return function (/*arguments*/) { + var self = this; + var args = arguments; + new Fiber(function () { + func.apply(self, args); + }).run(); + }; +}; diff --git a/tools/main.js b/tools/main.js index b09c0e5a24..28766230be 100644 --- a/tools/main.js +++ b/tools/main.js @@ -230,7 +230,7 @@ var loadHelp = function () { }); }; -var longHelp = function (commandName) { +var longHelp = exports.longHelp = function (commandName) { commandName = commandName.trim(); var parts = commandName.length ? commandName.split(' ') : []; var node = commands; @@ -1165,6 +1165,3 @@ commandName + ": You're not in a Meteor project directory.\n" + process.exit(ret); }).run(); -// exports -main.longHelp = longHelp; - diff --git a/tools/run-all.js b/tools/run-all.js index bf14ea51a1..9dec9c890c 100644 --- a/tools/run-all.js +++ b/tools/run-all.js @@ -1,8 +1,6 @@ var _ = require('underscore'); var Future = require('fibers/future'); -var Fiber = require('fibers'); var files = require('./files.js'); -var inFiber = require('./fiber-helpers.js').inFiber; var release = require('./release.js'); var runLog = require('./run-log.js'); diff --git a/tools/run-app.js b/tools/run-app.js index c01285e6f4..dd7281bf8c 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -3,13 +3,13 @@ var path = require("path"); var _ = require('underscore'); var Future = require('fibers/future'); var Fiber = require('fibers'); +var fiberHelpers = require('./fiber-helpers.js'); var files = require('./files.js'); var watch = require('./watch.js'); var project = require('./project.js').project; var bundler = require('./bundler.js'); var release = require('./release.js'); var buildmessage = require('./buildmessage.js'); -var inFiber = require('./fiber-helpers.js').inFiber; var runLog = require('./run-log.js'); var catalog = require('./catalog.js'); var packageCache = require('./package-cache.js'); @@ -90,7 +90,7 @@ _.extend(AppProcess.prototype, { // Send stdout and stderr to the runLog var eachline = require('eachline'); - eachline(self.proc.stdout, 'utf8', function (line) { + eachline(self.proc.stdout, 'utf8', fiberHelpers.inBareFiber(function (line) { if (line.match(/^LISTENING\s*$/)) { // This is the child process telling us that it's ready to // receive connections. @@ -98,26 +98,26 @@ _.extend(AppProcess.prototype, { } else { runLog.logAppOutput(line); } - }); + })); - eachline(self.proc.stderr, 'utf8', function (line) { + eachline(self.proc.stderr, 'utf8', fiberHelpers.inBareFiber(function (line) { runLog.logAppOutput(line, true); - }); + })); // Watch for exit and for stdio to be fully closed (so that we don't miss // log lines). - self.proc.on('close', function (code, signal) { + self.proc.on('close', fiberHelpers.inBareFiber(function (code, signal) { self._maybeCallOnExit(code, signal); - }); + })); - self.proc.on('error', function (err) { + self.proc.on('error', fiberHelpers.inBareFiber(function (err) { runLog.log("=> Couldn't spawn process: " + err.message); // node docs say that it might make both an 'error' and a // 'close' callback, so we use a guard to make sure we only call // onExit once. self._maybeCallOnExit(); - }); + })); // This happens sometimes when we write a keepalive after the app // is dead. If we don't register a handler, we get a top level @@ -355,7 +355,9 @@ _.extend(AppRunner.prototype, { throw new Error("already started?"); self.startFuture = new Future; - self.fiber = new Fiber(function () { + // XXX I think it's correct to not try to use bindEnvironment here: + // the extra fiber should be independent of this one. + self.fiber = Fiber(function () { self._fiber(); }); self.fiber.run(); diff --git a/tools/run-mongo.js b/tools/run-mongo.js index ca730d4b7b..c21504e85b 100644 --- a/tools/run-mongo.js +++ b/tools/run-mongo.js @@ -6,12 +6,10 @@ var utils = require('./utils.js'); var release = require('./release.js'); var mongoExitCodes = require('./mongo-exit-codes.js'); var fiberHelpers = require('./fiber-helpers.js'); -var inFiber = fiberHelpers.inFiber; var runLog = require('./run-log.js'); var _ = require('underscore'); var uniload = require('./uniload.js'); -var Fiber = require('fibers'); var Future = require('fibers/future'); // Given a Mongo URL, open an interative Mongo shell on this terminal @@ -309,7 +307,7 @@ var launchMongo = function (options) { } }); - procExitHandler = inFiber(function (code, signal) { + procExitHandler = fiberHelpers.bindEnvironment(function (code, signal) { // Defang subHandle.stop(). proc = null; @@ -338,7 +336,7 @@ var launchMongo = function (options) { } }; - var stdoutOnData = inFiber(function (data) { + var stdoutOnData = fiberHelpers.bindEnvironment(function (data) { // note: don't use "else ifs" in this, because 'data' can have multiple // lines if (/config from self or any seed \(EMPTYCONFIG\)/.test(data)) { @@ -629,7 +627,7 @@ _.extend(MongoRunner.prototype, { if (self.errorCount < 3) { // Wait a second, then restart. - self.restartTimer = setTimeout(inFiber(function () { + self.restartTimer = setTimeout(fiberHelpers.bindEnvironment(function () { self.restartTimer = null; self._startOrRestart(); }), 1000); diff --git a/tools/run-updater.js b/tools/run-updater.js index d5f429ff75..936001e1cf 100644 --- a/tools/run-updater.js +++ b/tools/run-updater.js @@ -1,6 +1,6 @@ var _ = require('underscore'); var Fiber = require('fibers'); -var inFiber = require('./fiber-helpers.js').inFiber; +var fiberHelpers = require('./fiber-helpers.js'); var Updater = function () { var self = this; @@ -17,12 +17,14 @@ _.extend(Updater.prototype, { if (self.timer) throw new Error("already running?"); - // Check twice a day. - self.timer = setInterval(inFiber(function () { + // Check twice a day. (Should not share buildmessage state with + // the main fiber.) + self.timer = setInterval(fiberHelpers.inBareFiber(function () { self._check(); }), 12*60*60*1000); - // Also start a check now, but don't block on it. + // Also start a check now, but don't block on it. (This should + // not share buildmessage state with the main fiber.) new Fiber(function () { self._check(); }).run(); diff --git a/tools/selftest.js b/tools/selftest.js index afe1cada73..5a99be8543 100644 --- a/tools/selftest.js +++ b/tools/selftest.js @@ -84,6 +84,15 @@ var execFileSync = function (binary, args) { })().wait(); }; +var captureAndThrow = function (f) { + var messages = buildmessage.capture(function () { + f(); + }); + if (messages.hasMessages()) { + throw Error(messages.formatMessages()); + } +}; + /////////////////////////////////////////////////////////////////////////////// // Matcher /////////////////////////////////////////////////////////////////////////////// @@ -623,16 +632,13 @@ _.extend(Sandbox.prototype, { // build apps that contain core packages). var toolPackage, toolPackageDirectory; - var messages = buildmessage.capture(function () { + captureAndThrow(function () { toolPackage = getToolsPackage(); toolPackageDirectory = '.' + toolPackage.version + '.XXX++' + toolPackage.buildArchitectures(); toolPackage.saveToPath(path.join(self.warehouse, packagesDirectoryName, toolPackageName, toolPackageDirectory)); }); - if (messages.hasMessages()) { - throw Error(messages.formatMessages()); - } fs.symlinkSync(toolPackageDirectory, path.join(self.warehouse, packagesDirectoryName, @@ -1573,5 +1579,6 @@ _.extend(exports, { expectEqual: expectEqual, expectThrows: expectThrows, getToolsPackage: getToolsPackage, - execFileSync: execFileSync + execFileSync: execFileSync, + captureAndThrow: captureAndThrow }); diff --git a/tools/stats.js b/tools/stats.js index 3dca61b1a2..588d3bd571 100644 --- a/tools/stats.js +++ b/tools/stats.js @@ -72,6 +72,7 @@ var recordPackages = function () { // We do this inside a new fiber to avoid blocking anything on talking // to the package stats server. If we can't connect, for example, we // don't care; we'll just miss out on recording these packages. + // This also gives it its own buildmessage state. Fiber(function () { var userAgentInfo = { diff --git a/tools/tests/releases.js b/tools/tests/releases.js index 8c35f16d4a..1ee2248cc4 100644 --- a/tools/tests/releases.js +++ b/tools/tests/releases.js @@ -19,7 +19,10 @@ selftest.define("springboard", ['checkout', 'net'], function () { }); var run; - var toolsPackage = selftest.getToolsPackage(); + var toolsPackage; + selftest.captureAndThrow(function() { + toolsPackage = selftest.getToolsPackage(); + }); var toolsVersion = toolsPackage.name + '@' + toolsPackage.version; @@ -132,7 +135,10 @@ selftest.define("writing versions file", ['checkout', 'net'], function () { }); var run; - var toolsPackage = selftest.getToolsPackage(); + var toolsPackage; + selftest.captureAndThrow(function() { + toolsPackage = selftest.getToolsPackage(); + }); var toolsVersion = toolsPackage.name + '@' + toolsPackage.version; diff --git a/tools/tests/report-stats.js b/tools/tests/report-stats.js index 0e437812bb..c79dcd3ceb 100644 --- a/tools/tests/report-stats.js +++ b/tools/tests/report-stats.js @@ -35,7 +35,10 @@ var checkMeta = function (appPackages, sessionId, useFakeRelease) { } if (useFakeRelease) { - var toolsPackage = selftest.getToolsPackage(); + var toolsPackage; + selftest.captureAndThrow(function() { + toolsPackage = selftest.getToolsPackage(); + }); expectedUserAgentInfo.meteorReleaseTrack = "METEOR-CORE"; expectedUserAgentInfo.meteorReleaseVersion = diff --git a/tools/tests/selftest-test.js b/tools/tests/selftest-test.js index 7d53543190..928a4865e3 100644 --- a/tools/tests/selftest-test.js +++ b/tools/tests/selftest-test.js @@ -11,7 +11,10 @@ selftest.define("selftest-from-warehouse", ['checkout'], function () { }); var run; - var toolsPackage = selftest.getToolsPackage(); + var toolsPackage; + selftest.captureAndThrow(function() { + toolsPackage = selftest.getToolsPackage(); + }); var toolsVersion = toolsPackage.name + '@' + toolsPackage.version; diff --git a/tools/updater.js b/tools/updater.js index f26c79c459..ff7dd0b1ed 100644 --- a/tools/updater.js +++ b/tools/updater.js @@ -1,7 +1,6 @@ var path = require('path'); var fs = require('fs'); var _ = require('underscore'); -var inFiber = require('./fiber-helpers.js').inFiber; var files = require('./files.js'); var tropohouse = require('./tropohouse.js'); var httpHelpers = require('./http-helpers.js'); diff --git a/tools/watch.js b/tools/watch.js index aed13b978e..8d6195aa3b 100644 --- a/tools/watch.js +++ b/tools/watch.js @@ -449,7 +449,7 @@ _.extend(Watcher.prototype, { }); if (!self.stopped && !self.justCheckOnce) { - setTimeout(fiberHelpers.inFiber(function () { + setTimeout(fiberHelpers.inBareFiber(function () { self._checkDirectories(true); }), 500); } From 5ff811731050f7718a08b7199baa120e75c42e5b Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 4 Aug 2014 19:06:15 -0700 Subject: [PATCH 112/126] Update comments after looking into e7250a0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It seems that when ObserveSequence observed a Cursor, it used to stop() the observe when the main autorun was invalidated, creating a “gap” during which no callbacks would be received (or fired). This was before we used Deps.nonreactive in the main autorun (shielding the cursor.observe from being stopped). Now the observe is only stopped upon re-run or when the ObserveSequence is stopped, and there is no gap. Removed references to this gap from comments. I believe the current code is correct, and in addition, we could now optimize the cursor-to-same-cursor case (and basically not do anything if seq===lastSeq and is a Cursor, i.e. not stop the old handle, create a new handle, or diff). --- packages/observe-sequence/observe_sequence.js | 5 ++--- packages/observe-sequence/observe_sequence_tests.js | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/observe-sequence/observe_sequence.js b/packages/observe-sequence/observe_sequence.js index b736ff88ef..b9273f2ab4 100644 --- a/packages/observe-sequence/observe_sequence.js +++ b/packages/observe-sequence/observe_sequence.js @@ -82,10 +82,9 @@ ObserveSequence = { Deps.nonreactive(function () { var seqArray; // same structure as `lastSeqArray` above. - // If we were previously observing a cursor, replace lastSeqArray with - // more up-to-date information (specifically, the state of the observe - // before it was stopped, which may be older than the DB). if (activeObserveHandle) { + // If we were previously observing a cursor, replace lastSeqArray with + // more up-to-date information. Then stop the old observe. lastSeqArray = _.map(lastSeq.fetch(), function (doc) { return {_id: doc._id, item: doc}; }); diff --git a/packages/observe-sequence/observe_sequence_tests.js b/packages/observe-sequence/observe_sequence_tests.js index 340de1179c..5fdb311989 100644 --- a/packages/observe-sequence/observe_sequence_tests.js +++ b/packages/observe-sequence/observe_sequence_tests.js @@ -439,9 +439,8 @@ Tinytest.add('observe-sequence - cursor to same cursor', function (test) { }, [ {addedAt: ["13", {_id: "13", rank: 1}, 0, null]}, {addedAt: ["24", {_id: "24", rank: 2}, 1, null]}, - // even if the cursor changes to the same cursor, we diff to see if we - // missed anything during the invalidation, which leads to these - // 'changedAt' events. + // even if the cursor changes to the same cursor, we do a diff, + // which leads to these 'changedAt' events. {changedAt: ["13", {_id: "13", rank: 1}, {_id: "13", rank: 1}, 0]}, {changedAt: ["24", {_id: "24", rank: 2}, {_id: "24", rank: 2}, 1]}, {addedAt: ["78", {_id: "78", rank: 3}, 2, null]} From 07852ebcbec97e9d076188029448e470a8017a58 Mon Sep 17 00:00:00 2001 From: Emily Stark Date: Mon, 4 Aug 2014 20:38:33 -0700 Subject: [PATCH 113/126] Fix galaxy login --- tools/auth.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/auth.js b/tools/auth.js index ec4066d9b0..8cab010bb3 100644 --- a/tools/auth.js +++ b/tools/auth.js @@ -541,7 +541,7 @@ var logInToGalaxy = function (galaxyName) { var authorizeResult; try { - sendAuthorizeRequest( + authorizeResult = sendAuthorizeRequest( galaxyClientId, galaxyRedirect, encodeURIComponent(JSON.stringify(stateInfo)) From ddc3657e4f4de6a5a20867b95a38c376bda7b9e2 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 4 Aug 2014 21:17:07 -0700 Subject: [PATCH 114/126] Watch files which fail before emitting a resource Regression introduced by the CSS watching code (specifically, f230eba62) by the sourceIsWatched check. We need to be able to tell the difference between "source handler didn't do anything because there was an error" and "source handler didn't do anything because it's web-specific and this is an os arch". A simple fix would have been to interpret compileStep.error as "sourceIsWatched = true", but I didn't think of that until after doing it the slightly more complicated but more precise way :) Also, ensure that if the runner rebuilds the client and there's an error, it properly kills the app process (and the watchers and the keepalive interval, etc). --- packages/less/plugin/compile-less.js | 11 +--- packages/meteor/plugin/basic-file-types.js | 11 +--- packages/stylus/plugin/compile-stylus.js | 11 +--- .../templating/plugin/compile-templates.js | 11 +--- tools/compiler.js | 28 ++++------ tools/run-app.js | 51 ++++++++++--------- .../css-injection-test/css-injection-test.js | 4 +- .../hot-code-push-test.html | 1 + tools/tests/hot-code-push.js | 28 ++++++++-- tools/unipackage.js | 3 +- 10 files changed, 70 insertions(+), 89 deletions(-) create mode 100644 tools/tests/apps/hot-code-push-test/hot-code-push-test.html diff --git a/packages/less/plugin/compile-less.js b/packages/less/plugin/compile-less.js index fbbf1e4eea..581a5b35a7 100644 --- a/packages/less/plugin/compile-less.js +++ b/packages/less/plugin/compile-less.js @@ -3,16 +3,7 @@ var path = Npm.require('path'); var less = Npm.require('less'); var Future = Npm.require('fibers/future'); -Plugin.registerSourceHandler("less", function (compileStep) { - // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('web')) { - // XXX in the future, might be better to emit some kind of a - // warning if a stylesheet is included on the server, rather than - // silently ignoring it. but that would mean you can't stick .css - // at the top level of your app, which is kind of silly. - return; - } - +Plugin.registerSourceHandler("less", {archMatching: 'web'}, function (compileStep) { var source = compileStep.read().toString('utf8'); var options = { filename: compileStep.inputPath, diff --git a/packages/meteor/plugin/basic-file-types.js b/packages/meteor/plugin/basic-file-types.js index 5cadb96ab3..39d78e2b35 100644 --- a/packages/meteor/plugin/basic-file-types.js +++ b/packages/meteor/plugin/basic-file-types.js @@ -2,16 +2,7 @@ we can't exactly define the *.js source file handler in a *.js source file. */ -Plugin.registerSourceHandler("css", function (compileStep) { - // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('web')) { - // XXX in the future, might be better to emit some kind of a - // warning if a stylesheet is included on the server, rather than - // silently ignoring it. but that would mean you can't stick .css - // at the top level of your app, which is kind of silly. - return; - } - +Plugin.registerSourceHandler("css", {archMatching: 'web'}, function (compileStep) { compileStep.addStylesheet({ data: compileStep.read().toString('utf8'), path: compileStep.inputPath diff --git a/packages/stylus/plugin/compile-stylus.js b/packages/stylus/plugin/compile-stylus.js index 556db16b64..008dc556aa 100644 --- a/packages/stylus/plugin/compile-stylus.js +++ b/packages/stylus/plugin/compile-stylus.js @@ -4,16 +4,7 @@ var nib = Npm.require('nib'); var path = Npm.require('path'); var Future = Npm.require('fibers/future'); -Plugin.registerSourceHandler("styl", function (compileStep) { - // XXX annoying that this is replicated in .css, .less, and .styl - if (! compileStep.archMatches('web')) { - // XXX in the future, might be better to emit some kind of a - // warning if a stylesheet is included on the server, rather than - // silently ignoring it. but that would mean you can't stick .css - // at the top level of your app, which is kind of silly. - return; - } - +Plugin.registerSourceHandler("styl", {archMatching: 'web'}, function (compileStep) { var f = new Future; stylus(compileStep.read().toString('utf8')) .use(nib()) diff --git a/packages/templating/plugin/compile-templates.js b/packages/templating/plugin/compile-templates.js index d91ad23360..e8b8dbdfbf 100644 --- a/packages/templating/plugin/compile-templates.js +++ b/packages/templating/plugin/compile-templates.js @@ -1,15 +1,6 @@ var path = Npm.require('path'); var doHTMLScanning = function (compileStep, htmlScanner) { - - if (! compileStep.archMatches("web")) - // XXX might be nice to throw an error here, but then we'd have to - // make it so that packages.js ignores html files that appear in - // the server directories in an app tree.. or, it might be nice to - // make html files actually work on the server (against jsdom or - // something) - return; - // XXX the way we deal with encodings here is sloppy .. should get // religion on that var contents = compileStep.read().toString('utf8'); @@ -53,7 +44,7 @@ var doHTMLScanning = function (compileStep, htmlScanner) { }; Plugin.registerSourceHandler( - "html", {isTemplate: true}, + "html", {isTemplate: true, archMatching: 'web'}, function (compileStep) { doHTMLScanning(compileStep, html_scanner); } diff --git a/tools/compiler.js b/tools/compiler.js index 390b777fd5..f43ecd204b 100644 --- a/tools/compiler.js +++ b/tools/compiler.js @@ -335,7 +335,7 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, _.each(activePluginPackages, function (otherPkg) { _.each(otherPkg.getSourceHandlers(), function (sourceHandler, ext) { // XXX comparing function text here seems wrong. - if (ext in allHandlers && + if (_.has(allHandlers, ext) && allHandlers[ext].toString() !== sourceHandler.handler.toString()) { buildmessage.error( "conflict: two packages included in " + @@ -344,10 +344,16 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, (otherPkg.name || "the app") + ", " + "are both trying to handle ." + ext); // Recover by just going with the first handler we saw - } else { - allHandlers[ext] = sourceHandler.handler; - sourceExtensions[ext] = !!sourceHandler.isTemplate; + return; } + // Is this handler only registered for, say, "web", and we're building, + // say, "os"? + if (sourceHandler.archMatching && + !archinfo.matches(inputSourceArch.arch, sourceHandler.archMatching)) { + return; + } + allHandlers[ext] = sourceHandler.handler; + sourceExtensions[ext] = !!sourceHandler.isTemplate; }); }); @@ -403,12 +409,6 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, var file = watch.readAndWatchFileWithHash(sourceWatchSet, absPath); var contents = file.contents; - // Only add the source file to the WatchSet if it's actually added to - // the build. This is a hacky workaround because plugins do not register - // themselves as "client" or "server", so we need to detect whether a file - // is actually added to the client/server program. - var sourceIsWatched = false; - sources.push(relPath); if (contents === null) { @@ -579,7 +579,6 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, throw new Error("'section' must be 'head' or 'body'"); if (typeof options.data !== "string") throw new Error("'data' option to appendDocument must be a string"); - sourceIsWatched = true; resources.push({ type: options.section, data: new Buffer(options.data, 'utf8') @@ -591,7 +590,6 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, "web targets"); if (typeof options.data !== "string") throw new Error("'data' option to addStylesheet must be a string"); - sourceIsWatched = true; resources.push({ type: "css", refreshable: true, @@ -607,7 +605,6 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, throw new Error("'sourcePath' option must be supplied to addJavaScript. Consider passing inputPath."); if (options.bare && ! archinfo.matches(inputSourceArch.arch, "web")) throw new Error("'bare' option may only be used for web targets"); - sourceIsWatched = true; js.push({ source: options.data, sourcePath: options.sourcePath, @@ -619,7 +616,6 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, addAsset: function (options) { if (! (options.data instanceof Buffer)) throw new Error("'data' option to addAsset must be a Buffer"); - sourceIsWatched = true; addAsset(options.data, options.path); }, error: function (options) { @@ -642,9 +638,7 @@ var compileUnibuild = function (unipackage, inputSourceArch, packageLoader, // handler might already have emitted resources) } - if (sourceIsWatched) { - watchSet.merge(sourceWatchSet); - } + watchSet.merge(sourceWatchSet); }); // *** Run Phase 1 link diff --git a/tools/run-app.js b/tools/run-app.js index dd7281bf8c..dd76935230 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -571,34 +571,37 @@ _.extend(AppRunner.prototype, { // source file to change. Or, for stop() to be called. var ret = runFuture.wait(); - while (ret.outcome === 'changed-refreshable') { - // We stay in this loop as long as only refreshable assets have changed. - // When ret.refreshable becomes false, we restart the server. - bundleResult = bundleApp(); - if (bundleResult.errors) { - return { - outcome: 'bundle-fail', - bundleResult: bundleResult - }; + try { + while (ret.outcome === 'changed-refreshable') { + // We stay in this loop as long as only refreshable assets have changed. + // When ret.refreshable becomes false, we restart the server. + bundleResult = bundleApp(); + if (bundleResult.errors) { + return { + outcome: 'bundle-fail', + bundleResult: bundleResult + }; + } + + // Establish a watcher on the new files. + setupClientWatcher(); + + // Notify the server that new client assets have been added to the build. + process.kill(appProcess.proc.pid, 'SIGUSR2'); + runLog.logClientRestart(); + + self.runFuture = new Future; + ret = self.runFuture.wait(); } + } finally { + self.runFuture = null; - // Establish a watcher on the new files. - setupClientWatcher(); + self.proxy.setMode("hold"); + appProcess.stop(); - // Notify the server that new client assets have been added to the build. - process.kill(appProcess.proc.pid, 'SIGUSR2'); - runLog.logClientRestart(); - - self.runFuture = new Future; - ret = self.runFuture.wait(); + serverWatcher && serverWatcher.stop(); + clientWatcher && clientWatcher.stop(); } - self.runFuture = null; - - self.proxy.setMode("hold"); - appProcess.stop(); - - serverWatcher && serverWatcher.stop(); - clientWatcher && clientWatcher.stop(); return ret; }, diff --git a/tools/tests/apps/css-injection-test/css-injection-test.js b/tools/tests/apps/css-injection-test/css-injection-test.js index ad4dfeb84c..c78d3f3bdd 100644 --- a/tools/tests/apps/css-injection-test/css-injection-test.js +++ b/tools/tests/apps/css-injection-test/css-injection-test.js @@ -25,7 +25,7 @@ if (Meteor.isClient) { oldBackgroundColor = backgroundColor(); Meteor.call("newStylesheet", ++numCssChanges, oldBackgroundColor); waitingForCssReloadToComplete = false; - Meteor.clearTimeout(handle); + Meteor.clearInterval(handle); } }, 500); } @@ -44,4 +44,4 @@ if (Meteor.isServer) { console.log("background-color: " + backgroundColor); } }); -} \ No newline at end of file +} diff --git a/tools/tests/apps/hot-code-push-test/hot-code-push-test.html b/tools/tests/apps/hot-code-push-test/hot-code-push-test.html new file mode 100644 index 0000000000..25bd658c06 --- /dev/null +++ b/tools/tests/apps/hot-code-push-test/hot-code-push-test.html @@ -0,0 +1 @@ +bla diff --git a/tools/tests/hot-code-push.js b/tools/tests/hot-code-push.js index 34a058d149..a2bb67a5b4 100644 --- a/tools/tests/hot-code-push.js +++ b/tools/tests/hot-code-push.js @@ -37,20 +37,20 @@ selftest.define("css hot code push", function (options) { // rgba(0, 0, 0, 0). run.match(/background-color: (transparent|rgba\(0, 0, 0, 0\))/); - // The server restarts if a new css file is added. + // The server does NOT restart if a new css file is added. s.write("test.css", "body { background-color: red; }"); - run.match("server restarted"); + run.match("Client modified -- refreshing"); run.match("numCssChanges: 1"); run.match("background-color: rgb(255, 0, 0)"); s.write("test.css", "body { background-color: blue; }"); - run.match("refreshing"); + run.match("Client modified -- refreshing"); run.match("numCssChanges: 2"); run.match("background-color: rgb(0, 0, 255)"); - // The server restarts if a css file is removed. + // The server does NOT restart if a css file is removed. s.unlink("test.css"); - run.match("server restarted"); + run.match("Client modified -- refreshing"); run.match("numCssChanges: 3"); run.match(/background-color: (transparent|rgba\(0, 0, 0, 0\))/); run.stop(); @@ -119,6 +119,24 @@ selftest.define("javascript hot code push", function (options) { run.match("client connected: 0"); run.match("jsVar: undefined"); + // Break the HTML file. This should kill the server, and print errors. + // (It would be reasonable behavior for this to NOT kill the server, since + // it only affects the client. But this is a regression test for a bug where + // fixing the HTML file wouldn't actually restart the server; that's the + // important part of this test.) + s.write("hot-code-push-test.html", ">"); + run.match("Errors prevented startup"); + run.match("bad formatting in HTML template"); + // Fix it. It should notice, and restart. The client will restart too. + s.write("hot-code-push-test.html", ""); + run.match("server restarted"); + run.match("client connected: 0"); + // Write something else to it. The client should restart. + s.write("hot-code-push-test.html", "foo"); + run.match("Client modified -- refreshing"); + run.match("client connected: 1"); + run.match("jsVar: undefined"); + // Add appcache and ensure that the browser still reloads. s.write(".meteor/packages", "standard-app-packages \n appcache"); run.match("added appcache"); diff --git a/tools/unipackage.js b/tools/unipackage.js index aaa66863a4..b455429d85 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -401,7 +401,8 @@ _.extend(Unipackage.prototype, { self.sourceHandlers[extension] = { handler: handler, - isTemplate: !!options.isTemplate + isTemplate: !!options.isTemplate, + archMatching: options.archMatching }; } }; From 835eb66377c01f5dfbe247e56800e5f79978376f Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 5 Aug 2014 12:56:07 -0700 Subject: [PATCH 115/126] a tiny bit more info to help debug an error --- packages/livedata/stream_client_nodejs.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/livedata/stream_client_nodejs.js b/packages/livedata/stream_client_nodejs.js index 718ee3fe41..a473fc5059 100644 --- a/packages/livedata/stream_client_nodejs.js +++ b/packages/livedata/stream_client_nodejs.js @@ -54,7 +54,7 @@ _.extend(LivedataTest.ClientStream.prototype, { // But _launchConnection calls _cleanup which closes previous connections. // It's our belief that this stifles future 'open' events, but maybe // we are wrong? - throw new Error("Got open from inactive client"); + throw new Error("Got open from inactive client " + !!self.client); } if (self._forcedToDisconnect) { From 98a9b5b31ac388e4be66437185e74161bf0a40d3 Mon Sep 17 00:00:00 2001 From: ekatek Date: Sun, 3 Aug 2014 01:25:20 -0700 Subject: [PATCH 116/126] don't check authorization on the client --- tools/commands-packages.js | 48 +++++++------------------------------- tools/package-client.js | 33 +++++++++++++++++++++++--- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 3ad40c5137..8f1e594248 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -38,28 +38,7 @@ var getReleaseOrPackageRecord = function(name) { if (rec) rel = true; } - return { record: rec, release: rel }; -}; - - -// Checks to see if you are an authorized maintainer for a given -// release/package. If you are not, calls process.exit -// explaining that you can't take that action. -// record: package or track record -// action: string for error handling -var checkAuthorizedPackageMaintainer = function (record, action) { - if (!record) { - process.stderr.write("This package does not exist.\n"); - return 1; - } - - var authorized = _.indexOf( - _.pluck(record.maintainers, 'username'), auth.loggedInUsername()); - if (authorized == -1 && record.name.split(':').length > 1) { - process.stderr.write('You are not an authorized maintainer of ' + record.name + ".\n"); - process.stderr.write('Only authorized maintainers may ' + action + ".\n"); - return 1;; - } + return { record: rec, isRelease: rel }; }; // Returns a pretty list suitable for showing to the user. Input is an @@ -434,18 +413,17 @@ main.registerCommand({ '. If you are creating a new track, use the --create-track flag.\n'); return 1; } - var auth = require("./auth.js"); - var authorized = _.indexOf( - _.pluck(trackRecord.maintainers, 'username'), auth.loggedInUsername()); - // The split on ':' is a hack to postpone the auth checking for non-prefixed - // releases to the server, which will, in turn use a hack to let MDG members - // publish on non-prefixed releases until we implement organizations. - if (authorized == -1 && relConf.track.split(':').length > 1) { + + // We are going to call the server to check if we are authorized, so that when + // we implement things like organizations, we are not handicapped by the + // user's meteor version. + if (!packageClient.amIAuthorized(relConf.track, true)) { process.stderr.write('\n You are not an authorized maintainer of ' + relConf.track + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions.\n'); return 1; - } + }; } + process.stdout.write(". OK!\n"); // This is sort of a hidden option to just take your entire meteor checkout @@ -766,7 +744,7 @@ main.registerCommand({ } var versionRecords; var label; - if (!allRecord.release) { + if (!allRecord.isRelease) { label = "package"; var getRelevantRecord = function (version) { var versionRecord = @@ -1598,8 +1576,6 @@ main.registerCommand({ var record = fullRecord.record; if (!options.list) { - checkAuthorizedPackageMaintainer(record, " add or remove maintainers"); - try { var conn = packageClient.loggedInPackagesConnection(); } catch (err) { @@ -1667,8 +1643,6 @@ main.registerCommand({ return 1; } - checkAuthorizedPackageMaintainer(record, " recommend or unrecommend release"); - try { var conn = packageClient.loggedInPackagesConnection(); } catch (err) { @@ -1722,8 +1696,6 @@ main.registerCommand({ return 1; } - checkAuthorizedPackageMaintainer(record, " set earliest recommended version"); - try { var conn = packageClient.loggedInPackagesConnection(); } catch (err) { @@ -1767,8 +1739,6 @@ main.registerCommand({ return 1; } - checkAuthorizedPackageMaintainer(record, " change repository URL"); - try { var conn = packageClient.loggedInPackagesConnection(); } catch (err) { diff --git a/tools/package-client.js b/tools/package-client.js index 6dc2e95d40..803fb7fe05 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -498,9 +498,8 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) process.stderr.write("Publish failed. \n"); return 1; } - var authorized = _.indexOf( - _.pluck(packRecord.maintainers, 'username'), auth.loggedInUsername()); - if (authorized == -1 && name.indexOf(":") !== -1) { + + if (!amIAuthorized(name, false)) { process.stderr.write('You are not an authorized maintainer of ' + name + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions. \n'); return 1; @@ -647,3 +646,31 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) return 0; }; + +// Call the server to ask if we are authorized to update this release or +// package. This is a way to save time before sending data to the server. It +// will mostly ignore most errors (just in case we have a flaky network connection or +// something) and let the method deal with those. +// +// If this returns FALSE, then we are NOT authorized. +// Otherwise, return true. +var amIAuthorized = function (name, isRelease) { + var conn = openPackageServerConnection(); + var methodName = "amIAuthorized" + + (isRelease ? "Release" : "Package"); + + try { + conn.call(methodName, name); + } catch (err) { + conn.close(); + if (err.error === 404) { + return false; + } + + // We don't know what this error is. Probably we can't contact the server, + // or the like. It would be a pity to fail all operations with the server + // just because a preliminary check fails, so return true for now. + return true; + } + return true; +}; From 5e4c8cd27640629d0d6125a411f7d0aaa7cf5147 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 11:01:42 -0700 Subject: [PATCH 117/126] search --mine --- tools/commands-packages.js | 57 ++++++++++++++++++++++++++++++++++---- tools/help.txt | 6 +++- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 8f1e594248..aa9e929762 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -724,13 +724,25 @@ main.registerCommand({ main.registerCommand({ name: 'search', - minArgs: 1, + minArgs: 0, maxArgs: 1, options: { - details: { type: Boolean, required: false } + details: { type: Boolean, required: false }, + mine: {type: Boolean, required: false } }, }, function (options) { + if (options.details && options.mine) { + process.stderr.write("You must select a specific package by name to view details. \n"); + return 1; + } + + if (!options.mine && options.args.length === 0) { + process.stderr.write("You must search for packages by name or substring. \n"); + throw new main.ShowUsage; + } + + catalog.official.refresh(); if (options.details) { @@ -832,15 +844,50 @@ main.registerCommand({ } process.stdout.write(maintain + "\n"); } else { - var search = options.args[0]; + var allPackages = catalog.official.getAllPackageNames(); var allReleases = catalog.official.getAllReleaseTracks(); var matchingPackages = []; var matchingReleases = []; + var selector; + if (options.mine) { + var myUserName = auth.loggedInUsername(); + if (!myUserName) { + // But couldn't you just grep the data.json for any maintainer? Yeah, + // but that's temporary, and won't work once organizations are around. + process.stderr.write("Please login so we know who you are. \n"); + auth.doUsernamePasswordLogin({}); + myUserName = auth.loggedInUsername(); + } + // In the future, we should consider checking this on the server, but I + // suspect the main use of this command will be to deal with the automatic + // migration and uncommon in everyday use. From that perspective, it makes + // little sense to require you to be online to find out what packages you + // own; and the consequence of not mentioning your group packages until + // you update to a new version of meteor is not that dire. + selector = function (packageName, isRelease) { + var record; + if (!isRelease) { + record = catalog.official.getPackage(packageName); + } else { + record = catalog.official.getReleaseTrack(packageName); + } + if (_.indexOf(_.pluck(record.maintainers, 'username'), myUserName) !== -1) { + return true; + } + return false; + }; + } else { + var search = options.args[0]; + selector = function (packageName) { + return packageName.match(search); + }; + } + _.each(allPackages, function (pack) { - if (pack.match(search)) { + if (selector(pack, false)) { var vr = catalog.official.getLatestVersion(pack); if (vr) { matchingPackages.push( @@ -849,7 +896,7 @@ main.registerCommand({ } }); _.each(allReleases, function (track) { - if (track.match(search)) { + if (selector(track, true)) { var vr = catalog.official.getDefaultReleaseVersion(track); if (vr) { var vrlong = diff --git a/tools/help.txt b/tools/help.txt index 1f2a82ebd6..ebb1a7aeca 100644 --- a/tools/help.txt +++ b/tools/help.txt @@ -473,6 +473,7 @@ debugging the Meteor packaging tools themselves. Search through the package server database. Usage: meteor search meteor search --details + meteor search --mine Search through the meteor package&release database for names containing the specified substring. @@ -481,9 +482,12 @@ Use --details to get detailed information on a specific package or release. Use --details @ to get more information on a specific version of a package or release. +Use --mine to get the list of packages on which you are one of the authorized +maintainers. + Options: --details show detailed information on a specific package or release - + --mine list packages on which you are the maintainer >>> admin maintainers View or change authorized maintainers for a package From 64ddf8a9797309d356ee926703ae54c9fbef56bb Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 11:04:15 -0700 Subject: [PATCH 118/126] fixing the amIAuthorized issues --- tools/commands-packages.js | 2 +- tools/package-client.js | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index aa9e929762..ddd2535473 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -417,7 +417,7 @@ main.registerCommand({ // We are going to call the server to check if we are authorized, so that when // we implement things like organizations, we are not handicapped by the // user's meteor version. - if (!packageClient.amIAuthorized(relConf.track, true)) { + if (!packageClient.amIAuthorized(relConf.track,conn, true)) { process.stderr.write('\n You are not an authorized maintainer of ' + relConf.track + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions.\n'); return 1; diff --git a/tools/package-client.js b/tools/package-client.js index 803fb7fe05..1a07b75a47 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -499,7 +499,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) return 1; } - if (!amIAuthorized(name, false)) { + if (!amIAuthorized(name, conn, false)) { process.stderr.write('You are not an authorized maintainer of ' + name + ".\n"); process.stderr.write('Only authorized maintainers may publish new versions. \n'); return 1; @@ -654,8 +654,7 @@ exports.publishPackage = function (packageSource, compileResult, conn, options) // // If this returns FALSE, then we are NOT authorized. // Otherwise, return true. -var amIAuthorized = function (name, isRelease) { - var conn = openPackageServerConnection(); +var amIAuthorized = function (name, conn, isRelease) { var methodName = "amIAuthorized" + (isRelease ? "Release" : "Package"); @@ -663,7 +662,7 @@ var amIAuthorized = function (name, isRelease) { conn.call(methodName, name); } catch (err) { conn.close(); - if (err.error === 404) { + if (err.error === 401) { return false; } From bcf85a42b935e72f6cf453c6664e8b4902b53051 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 11:13:56 -0700 Subject: [PATCH 119/126] don't print out add messages when no versions file present --- tools/project.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/project.js b/tools/project.js index 94cbf7a545..6a9b6f4fcf 100644 --- a/tools/project.js +++ b/tools/project.js @@ -349,7 +349,7 @@ _.extend(Project.prototype, { return 1; // Show the user the messageLog of packages we added. - if (!self.muted) { + if (!self.muted && !_.isEmpty(versions)) { _.each(messageLog, function (msg) { process.stdout.write(msg + "\n"); }); From ecaa9e02313ac4364fdde90a244e44e1cb8b797f Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 11:19:47 -0700 Subject: [PATCH 120/126] download all dependencies to be on the safe side --- tools/commands-packages.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index ddd2535473..25463f2f74 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -141,9 +141,9 @@ main.registerCommand({ if (buildmessage.jobHasMessages()) return; // already have errors, so skip the build - var directDeps = - compiler.determineBuildTimeDependencies(packageSource).directDependencies; - project._ensurePackagesExistOnDisk(directDeps); + var deps = + compiler.determineBuildTimeDependencies(packageSource).packageDependencies; + project._ensurePackagesExistOnDisk(deps); compileResult = compiler.compile(packageSource, { officialBuild: true }); }); From 82323fcfa935c05ed3b163e382103fa074312e92 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 13:06:43 -0700 Subject: [PATCH 121/126] do not close the connection --- tools/package-client.js | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/package-client.js b/tools/package-client.js index 1a07b75a47..1e1df33078 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -661,7 +661,6 @@ var amIAuthorized = function (name, conn, isRelease) { try { conn.call(methodName, name); } catch (err) { - conn.close(); if (err.error === 401) { return false; } From 62db7b694c4cbf06a44da740ac6e09c3b187a607 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 5 Aug 2014 13:17:26 -0700 Subject: [PATCH 122/126] api.versionsFrom may only be called once --- tools/package-source.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tools/package-source.js b/tools/package-source.js index a6e20af0b7..f7ad5aa23c 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -884,6 +884,13 @@ _.extend(PackageSource.prototype, { // you don't fill in dependencies for some of your implies/uses, we will // look at the packages listed in the release to figure that out. versionsFrom: function (release) { + if (releaseRecord) { + buildmessage.error("api.versionsFrom may only be specified once.", + { useMyCaller: true }); + // recover by ignoring + return; + } + // If you don't specify a track, use our default. if (release.indexOf('@') === -1) { release = catalog.complete.DEFAULT_TRACK + "@" + release; From ad708ca5699719acbc00a73dd2d77a43cf7c7681 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 13:31:24 -0700 Subject: [PATCH 123/126] this should fix local versionsFrom in most cases and is technically right --- tools/package-source.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/package-source.js b/tools/package-source.js index f7ad5aa23c..184a5b371d 100644 --- a/tools/package-source.js +++ b/tools/package-source.js @@ -904,7 +904,7 @@ _.extend(PackageSource.prototype, { // catalog may not be initialized, but we are pretty sure that the // releases are there anyway. This is not the right way to do this // long term. - releaseRecord = catalog.official.getReleaseVersion( + releaseRecord = catalog.complete.getReleaseVersion( relInf[0], relInf[1], true); if (!releaseRecord) { buildmessage.error("Unknown release "+ release); From 80a87f7f666b334e9b31bb63f56a78b4f8d38083 Mon Sep 17 00:00:00 2001 From: ekatek Date: Tue, 5 Aug 2014 14:16:52 -0700 Subject: [PATCH 124/126] fetch dependencies for publish-for-arch --- tools/commands-packages.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/commands-packages.js b/tools/commands-packages.js index 25463f2f74..bb25b0a34d 100644 --- a/tools/commands-packages.js +++ b/tools/commands-packages.js @@ -268,6 +268,15 @@ main.registerCommand({ if (buildmessage.jobHasMessages()) return; + + // Now compile it! Once again, everything should compile, and if + // it doesn't we should fail. Hopefully, of course, we have + // tested our stuff before deciding to publish it to the package + // server, but we need to be careful. + var deps = + compiler.determineBuildTimeDependencies(packageSource).packageDependencies; + project._ensurePackagesExistOnDisk(deps); + unipkg = compiler.compile(packageSource, { officialBuild: true }).unipackage; From 0422dfb5c6957c5fd08602c9c37fc15168dcee82 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 5 Aug 2014 15:48:27 -0700 Subject: [PATCH 125/126] fix updateServerPackageData test --- tools/tests/package-tests.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tools/tests/package-tests.js b/tools/tests/package-tests.js index d67a47c80b..2ed3dba2f1 100644 --- a/tools/tests/package-tests.js +++ b/tools/tests/package-tests.js @@ -449,10 +449,6 @@ selftest.define("update server package data unit test", ["net", "test-package-server"], function () { var packageStorageFileDir = files.mkdtemp("update-server-package-data"); var packageStorageFile = path.join(packageStorageFileDir, "data.json"); - var opts = { - packageStorageFile: packageStorageFile, - useShortPages: true - }; var s = new Sandbox(); var run; @@ -462,7 +458,9 @@ selftest.define("update server package data unit test", // Get the current data from the server. Once we publish new packages, // we'll check that all this data still appears on disk and hasn't // been overwritten. - var data = packageClient.updateServerPackageData({ syncToken: {} }); + var data = packageClient.updateServerPackageData({ syncToken: {} }, { + packageStorageFile: packageStorageFile + }); var packageNames = []; @@ -475,7 +473,10 @@ selftest.define("update server package data unit test", packageNames.push(packageName); }); - var newData = packageClient.updateServerPackageData(data); + var newData = packageClient.updateServerPackageData(data, { + packageStorageFile: packageStorageFile, + useShortPages: true + }); var newOnDiskData = packageClient.loadCachedServerData(packageStorageFile); // Check that we didn't lose any data. From 59e5b031f8508550076e5ec745c568359df31731 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Tue, 5 Aug 2014 15:49:46 -0700 Subject: [PATCH 126/126] use just one connection in updateServerPackageData --- tools/package-client.js | 115 ++++++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 51 deletions(-) diff --git a/tools/package-client.js b/tools/package-client.js index 1e1df33078..74b0182b74 100644 --- a/tools/package-client.js +++ b/tools/package-client.js @@ -82,10 +82,11 @@ exports.loadCachedServerData = function (packageStorageFile) { return ret; }; -// Opens a connection to the server, requests and returns new package data that -// we haven't cached on disk. We assume that data is cached chronologically, so -// essentially, we are asking for a diff from the last time that we did this. +// Requests and returns one page of new package data that we haven't cached on +// disk. We assume that data is cached chronologically, so essentially, we are +// asking for a diff from the last time that we did this. // Takes in: +// - conn: the connection to use (does not have to be logged in) // - syncToken: a syncToken object to be sent to the server that // represents the last time that we talked to the server. // - _optionsForTest: @@ -99,25 +100,20 @@ exports.loadCachedServerData = function (packageStorageFile) { // // Throws a ServiceConnection.ConnectionTimeoutError if the method call // times out. -var loadRemotePackageData = function (syncToken, _optionsForTest) { +var loadRemotePackageData = function (conn, syncToken, _optionsForTest) { _optionsForTest = _optionsForTest || {}; - var conn = openPackageServerConnection(); var syncOpts; if (_optionsForTest && _optionsForTest.useShortPages) { syncOpts = { shortPagesForTest: _optionsForTest.useShortPages }; } - try { - var collectionData; - if (syncOpts) { - collectionData = conn.call( - 'syncNewPackageData', syncToken, syncOpts); - } else { - collectionData = conn.call( - 'syncNewPackageData', syncToken); - } - } finally { - conn.close(); + var collectionData; + if (syncOpts) { + collectionData = conn.call( + 'syncNewPackageData', syncToken, syncOpts); + } else { + collectionData = conn.call( + 'syncNewPackageData', syncToken); } return collectionData; }; @@ -184,50 +180,67 @@ var writePackageDataToDisk = function (syncToken, data, options) { // - useShortPages: Boolean. Request short pages of ~3 records from the // server, instead of ~100 that it would send otherwise exports.updateServerPackageData = function (cachedServerData, _optionsForTest) { + var self = this; _optionsForTest = _optionsForTest || {}; + var done = false; - var sources = []; - if (cachedServerData.collections) { - sources.push(cachedServerData.collections); - } - var syncToken = cachedServerData.syncToken; - var remoteData; - try { - remoteData = loadRemotePackageData(syncToken, { - useShortPages: _optionsForTest.useShortPages - }); - } catch (err) { - process.stderr.write("ERROR " + err.message + "\n"); - if (err instanceof ServiceConnection.ConnectionTimeoutError) { - return null; - } else { - throw err; + var conn = openPackageServerConnection(); + + var getSomeData = function () { + var sources = []; + if (cachedServerData.collections) { + sources.push(cachedServerData.collections); + } + var syncToken = cachedServerData.syncToken; + var remoteData; + try { + remoteData = loadRemotePackageData(conn, syncToken, { + useShortPages: _optionsForTest.useShortPages + }); + } catch (err) { + process.stderr.write("ERROR " + err.message + "\n"); + if (err instanceof ServiceConnection.ConnectionTimeoutError) { + cachedServerData = null; + done = true; + return; + } else { + throw err; + } } - } - // If there is no new data from the server, don't bother writing things to - // disk. - if (_.isEqual(remoteData.collections, {})) { - return cachedServerData; - } + // If there is no new data from the server, don't bother writing things to + // disk. + // XXX fix for resetData? + if (_.isEqual(remoteData.collections, {})) { + done = true; + return; + } - sources.push(remoteData.collections); - var allCollections = mergeCollections(sources); - var data = { - syncToken: remoteData.syncToken, - formatVersion: "1.0", - collections: allCollections + sources.push(remoteData.collections); + var allCollections = mergeCollections(sources); + var data = { + syncToken: remoteData.syncToken, + formatVersion: "1.0", + collections: allCollections + }; + writePackageDataToDisk(remoteData.syncToken, data, { + packageStorageFile: _optionsForTest.packageStorageFile + }); + + cachedServerData = data; + if (remoteData.upToDate) + done = true; }; - writePackageDataToDisk(remoteData.syncToken, data, { - packageStorageFile: _optionsForTest.packageStorageFile - }); - // If we are not done, keep trying! - if (!remoteData.upToDate) { - data = this.updateServerPackageData(data, _optionsForTest); + try { + while (!done) { + getSomeData(); + } + } finally { + conn.close(); } - return data; + return cachedServerData; }; // Returns a logged-in DDP connection to the package server, or null if