From 0b700b286a6bc29dd2d4d8e8403533c8cd846bf2 Mon Sep 17 00:00:00 2001 From: Tim Haines Date: Thu, 28 Feb 2013 15:58:09 -0800 Subject: [PATCH 001/450] Include avatar and language in serviceData when user signs in to Twitter --- packages/accounts-twitter/twitter_server.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/accounts-twitter/twitter_server.js b/packages/accounts-twitter/twitter_server.js index 7ba252cca3..2904e7b17f 100644 --- a/packages/accounts-twitter/twitter_server.js +++ b/packages/accounts-twitter/twitter_server.js @@ -3,13 +3,22 @@ Accounts.oauth.registerService('twitter', 1, function(oauthBinding) { var identity = oauthBinding.get('https://api.twitter.com/1.1/account/verify_credentials.json').data; - return { - serviceData: { + var serviceData = { id: identity.id_str, screenName: identity.screen_name, accessToken: oauthBinding.accessToken, accessTokenSecret: oauthBinding.accessTokenSecret - }, + }; + + // include helpful fields from twitter + // https://dev.twitter.com/docs/api/1.1/get/account/verify_credentials + var whitelisted = ['profile_image_url', 'profile_image_url_https', 'lang']; + + var fields = _.pick(identity, whitelisted); + _.extend(serviceData, fields); + + return { + serviceData: serviceData, options: { profile: { name: identity.name From 6f059f0508f59d8794a1352ae957ffe87994b0b0 Mon Sep 17 00:00:00 2001 From: Sean McCann Date: Thu, 7 Mar 2013 20:44:41 -0500 Subject: [PATCH 002/450] Remove trailing whitespace As per the "Whitespace" section of the Meteor style guide, remove trailing whitespace. https://github.com/meteor/meteor/wiki/Meteor-Style-Guide --- app/lib/files.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/app/lib/files.js b/app/lib/files.js index 250af92a17..8ad2c8403b 100644 --- a/app/lib/files.js +++ b/app/lib/files.js @@ -198,7 +198,7 @@ var files = module.exports = { else return path.join(__dirname, '..', '..'); }, - + // returns a list of places where packages can be found. // 1. directories set via process.env.PACKAGES_DIRS // 2. default is packages/ in the meteor directory @@ -207,11 +207,11 @@ var files = module.exports = { var package_dirs = [path.join(__dirname, '..', '..', 'packages')]; if (process.env.PACKAGE_DIRS) package_dirs = process.env.PACKAGE_DIRS.split(':').concat(package_dirs); - + return package_dirs; }, - - // search package dirs for a package named name. + + // search package dirs for a package named name. // undefined if the package isn't in any dir get_package_dir: function (name) { var ret; @@ -222,7 +222,7 @@ var files = module.exports = { return true; } }); - + return ret; }, From 39fe1509e152e2598d5c628bed25a1a0db51f526 Mon Sep 17 00:00:00 2001 From: Sean McCann Date: Fri, 8 Mar 2013 00:59:20 -0500 Subject: [PATCH 003/450] Fix spelling error in find_upwards() comments --- app/lib/files.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/files.js b/app/lib/files.js index 8ad2c8403b..a1c178a24b 100644 --- a/app/lib/files.js +++ b/app/lib/files.js @@ -127,7 +127,7 @@ var files = module.exports = { }, // given a predicate function and a starting path, traverse upwards - // from the path until we find a path that satisfys the predicate. + // from the path until we find a path that satisfies the predicate. // // returns either the path to the lowest level directory that passed // the test or null for none found. if starting path isn't given, use From 4bf1b76a71c7665fb9443708f01a6b9b9f1ddd26 Mon Sep 17 00:00:00 2001 From: Morten Henriksen Date: Thu, 14 Mar 2013 11:50:36 +0100 Subject: [PATCH 004/450] Fixed #801 --- packages/handlebars/evaluate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/handlebars/evaluate.js b/packages/handlebars/evaluate.js index a9e7ddd19c..1accbc99c0 100644 --- a/packages/handlebars/evaluate.js +++ b/packages/handlebars/evaluate.js @@ -35,7 +35,7 @@ Handlebars._default_helpers = { if (data && data.length > 0) return _.map(data, function(x, i) { // infer a branch key from the data - var branch = (x._id || (typeof x === 'string' ? x : null) || + var branch = ((x && x._id) || (typeof x === 'string' ? x : null) || Spark.UNIQUE_LABEL); return Spark.labelBranch(branch, function() { return options.fn(x); From 30643c5008f0e12d8fe6cee0ade214950c635f0c Mon Sep 17 00:00:00 2001 From: Morten Henriksen Date: Thu, 14 Mar 2013 14:03:39 +0100 Subject: [PATCH 005/450] Added som tests for fix #801 --- packages/templating/templating_tests.html | 5 +++++ packages/templating/templating_tests.js | 7 +++++++ 2 files changed, 12 insertions(+) diff --git a/packages/templating/templating_tests.html b/packages/templating/templating_tests.html index d44420cd02..d57f9ce36b 100644 --- a/packages/templating/templating_tests.html +++ b/packages/templating/templating_tests.html @@ -346,3 +346,8 @@ + + + diff --git a/packages/templating/templating_tests.js b/packages/templating/templating_tests.js index 5c3d9717e5..dd56e22fa1 100644 --- a/packages/templating/templating_tests.js +++ b/packages/templating/templating_tests.js @@ -1068,3 +1068,10 @@ Tinytest.add("templating - tricky branch labels", function (test) { div.kill(); Meteor.flush(); }); + +Tinytest.add('templating - each falsy Issue #801', function (test) { + //Minor test for issue #801 + Template.test_template_issue801.values = function() { return [1,2,null,undefined]; }; + var frag = Meteor.render(Template.test_template_issue801); + test.equal(canonicalizeHtml(DomUtils.fragmentToHtml(frag)), "12null"); +}); From 0ead1fbf4123d270e164a38d5734298dc053addd Mon Sep 17 00:00:00 2001 From: Morten Henriksen Date: Thu, 14 Mar 2013 14:32:06 +0100 Subject: [PATCH 006/450] Fixed reactive each Mentioned by Chris Mather on google talk --- packages/templating/deftemplate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/templating/deftemplate.js b/packages/templating/deftemplate.js index 4e459883bd..8578e1ca18 100644 --- a/packages/templating/deftemplate.js +++ b/packages/templating/deftemplate.js @@ -18,7 +18,7 @@ arg, function (item) { return Spark.labelBranch( - item._id || Spark.UNIQUE_LABEL, function () { + (item && item._id) || Spark.UNIQUE_LABEL, function () { var html = Spark.isolate(_.bind(options.fn, null, item)); return Spark.setDataContext(item, html); }); From 00c3f488370ebc0f8dce241e1b4045622e67ac0b Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Thu, 14 Mar 2013 15:04:35 -0700 Subject: [PATCH 007/450] Fix annoying bug in opera where deleting from a doc as you iterate doesn't work right. --- packages/minimongo/modify.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/minimongo/modify.js b/packages/minimongo/modify.js index 1501d552a6..e699a262ff 100644 --- a/packages/minimongo/modify.js +++ b/packages/minimongo/modify.js @@ -55,11 +55,14 @@ LocalCollection._modify = function (doc, mod) { } } - // move new document into place - for (var k in doc) { + // move new document into place. + _.each(_.keys(doc), function (k) { + // Note: this used to be for (var k in doc) however, this does not + // work right in Opera. Deleting from a doc while iterating over it + // would sometimes cause opera to skip some keys. if (k !== '_id') delete doc[k]; - } + }); for (var k in new_doc) { doc[k] = new_doc[k]; } From c8d3e0dd8954c72cdd631b14cbe89fa5dd93dd0b Mon Sep 17 00:00:00 2001 From: David Glasser Date: Thu, 14 Mar 2013 22:20:30 -0700 Subject: [PATCH 008/450] Fix allow_tests instability. Meteor.apply('login', [], {wait: true}); Meteor.call('setUpTests'); // creates some publishers Meteor.subscribe('oneOfThosePublishers'); The client will send the subscribe call before the setUpTests call due to login's wait-ness, and then the subscribe will fail. We were basically doing that in allow_tests. --- packages/mongo-livedata/allow_tests.js | 149 ++++++++++++++----------- 1 file changed, 85 insertions(+), 64 deletions(-) diff --git a/packages/mongo-livedata/allow_tests.js b/packages/mongo-livedata/allow_tests.js index fb934c113c..b574c739b8 100644 --- a/packages/mongo-livedata/allow_tests.js +++ b/packages/mongo-livedata/allow_tests.js @@ -2,8 +2,88 @@ if (Meteor.isServer) { // Set up allow/deny rules for test collections - Meteor.methods({ - setUpAllowTestsCollections: function (nonce, idGeneration) { + + var allowCollections = {}; + + // We create the collections in the publisher (instead of using a method or + // something) because if we made them with a method, we'd need to follow the + // method with some subscribes, and it's possible that the method call would + // be delayed by a wait method and the subscribe messages would be sent before + // it and fail due to the collection not yet existing. So we are very hacky + // and use a publish. + Meteor.publish("allowTests", function (nonce, idGeneration) { + var cursors = []; + var needToConfigure = undefined; + + // helper for defining a collection. we are careful to create just one + // Meteor.Collection even if the sub body is rerun, by caching them. + var defineCollection = function(name, insecure, transform) { + var fullName = name + idGeneration + nonce; + + var collection; + if (_.has(allowCollections, fullName)) { + collection = allowCollections[fullName]; + if (needToConfigure === true) + throw new Error("collections inconsistently exist"); + needToConfigure = false; + } else { + collection = new Meteor.Collection( + fullName, {idGeneration: idGeneration, transform: transform}); + allowCollections[fullName] = collection; + if (needToConfigure === false) + throw new Error("collections inconsistently don't exist"); + needToConfigure = true; + collection._insecure = insecure; + var m = {}; + m["clear-collection-" + fullName] = function() { + collection.remove({}); + }; + Meteor.methods(m); + } + + cursors.push(collection.find()); + return collection; + }; + + var insecureCollection = defineCollection( + "collection-insecure", true /*insecure*/); + // totally locked down collection + var lockedDownCollection = defineCollection( + "collection-locked-down", false /*insecure*/); + // resticted collection with same allowed modifications, both with and + // without the `insecure` package + var restrictedCollectionDefaultSecure = defineCollection( + "collection-restrictedDefaultSecure", false /*insecure*/); + var restrictedCollectionDefaultInsecure = defineCollection( + "collection-restrictedDefaultInsecure", true /*insecure*/); + var restrictedCollectionForUpdateOptionsTest = defineCollection( + "collection-restrictedForUpdateOptionsTest", true /*insecure*/); + var restrictedCollectionForPartialAllowTest = defineCollection( + "collection-restrictedForPartialAllowTest", true /*insecure*/); + var restrictedCollectionForPartialDenyTest = defineCollection( + "collection-restrictedForPartialDenyTest", true /*insecure*/); + var restrictedCollectionForFetchTest = defineCollection( + "collection-restrictedForFetchTest", true /*insecure*/); + var restrictedCollectionForFetchAllTest = defineCollection( + "collection-restrictedForFetchAllTest", true /*insecure*/); + var restrictedCollectionWithTransform = defineCollection( + "withTransform", false, function (doc) { + return doc.a; + }); + + if (needToConfigure) { + restrictedCollectionWithTransform.allow({ + insert: function (userId, doc) { + return doc.foo === "foo"; + }, + update: function (userId, doc) { + return doc.foo === "foo"; + }, + remove: function (userId, doc) { + return doc.bar === "bar"; + } + }); + // two calls to allow to verify that either validator is sufficient. var allows = [{ insert: function(userId, doc) { @@ -44,65 +124,6 @@ if (Meteor.isServer) { } }]; - - // helper for defining a collection, subscribing to it, and defining - // a method to clear it - var defineCollection = function(name, insecure, transform) { - var fullName = name + idGeneration + nonce; - var collection = new Meteor.Collection( - fullName, {idGeneration: idGeneration, transform: transform}); - collection._insecure = insecure; - - Meteor.publish("collection-" + fullName, function() { - return collection.find(); - }); - - var m = {}; - m["clear-collection-" + fullName] = function() { - collection.remove({}); - }; - Meteor.methods(m); - return collection; - }; - - var insecureCollection = defineCollection( - "collection-insecure", true /*insecure*/); - // totally locked down collection - var lockedDownCollection = defineCollection( - "collection-locked-down", false /*insecure*/); - // resticted collection with same allowed modifications, both with and - // without the `insecure` package - var restrictedCollectionDefaultSecure = defineCollection( - "collection-restrictedDefaultSecure", false /*insecure*/); - var restrictedCollectionDefaultInsecure = defineCollection( - "collection-restrictedDefaultInsecure", true /*insecure*/); - var restrictedCollectionForUpdateOptionsTest = defineCollection( - "collection-restrictedForUpdateOptionsTest", true /*insecure*/); - var restrictedCollectionForPartialAllowTest = defineCollection( - "collection-restrictedForPartialAllowTest", true /*insecure*/); - var restrictedCollectionForPartialDenyTest = defineCollection( - "collection-restrictedForPartialDenyTest", true /*insecure*/); - var restrictedCollectionForFetchTest = defineCollection( - "collection-restrictedForFetchTest", true /*insecure*/); - var restrictedCollectionForFetchAllTest = defineCollection( - "collection-restrictedForFetchAllTest", true /*insecure*/); - var restrictedCollectionWithTransform = defineCollection( - "withTransform", false, function (doc) { - return doc.a; - }); - - restrictedCollectionWithTransform.allow({ - insert: function (userId, doc) { - return doc.foo === "foo"; - }, - update: function (userId, doc) { - return doc.foo === "foo"; - }, - remove: function (userId, doc) { - return doc.bar === "bar"; - } - }); - _.each([ restrictedCollectionDefaultSecure, restrictedCollectionDefaultInsecure, @@ -168,6 +189,8 @@ if (Meteor.isServer) { update: function() { return true; } }); } + + return cursors; }); } @@ -180,7 +203,7 @@ if (Meteor.isClient) { // Tell the server to make, configure, and publish a set of collections unique // to our test run. Since the method does not unblock, this will complete // running on the server before anything else happens. - Meteor.call('setUpAllowTestsCollections', nonce, idGeneration); + Meteor.subscribe('allowTests', nonce, idGeneration); // helper for defining a collection, subscribing to it, and defining // a method to clear it @@ -189,8 +212,6 @@ if (Meteor.isClient) { var collection = new Meteor.Collection( fullName, {idGeneration: idGeneration, transform: transform}); - Meteor.subscribe("collection-" + fullName); - collection.callClearMethod = function (callback) { Meteor.call("clear-collection-" + fullName, callback); }; From d97b0b6e0d3cbe568ffbfd6b66efb20f11c5fdcb Mon Sep 17 00:00:00 2001 From: Chris Mather Date: Fri, 15 Mar 2013 15:42:05 -0700 Subject: [PATCH 009/450] Update History.md with recent pull request merges. --- History.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/History.md b/History.md index 6af7ffee18..56cd69c217 100644 --- a/History.md +++ b/History.md @@ -1,6 +1,14 @@ ## vNEXT +* {{#each}} helper can now iterate over falsy values without throwing an + exception. #815, #801 + +* Twitter login now stores profile_image_url and profile_image_url_https + attributes in the user.services.twitter namespace. #788 + +Patches contributed by GitHub users raix, blackcoat, timhaines + ## v0.5.9 * Fix regression in 0.5.8 that prevented users from editing their own From 83b99f8c8a513970fb856da36f0345288604d680 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Fri, 15 Mar 2013 18:57:01 -0700 Subject: [PATCH 010/450] test for added from two different subs --- packages/livedata/livedata_tests.js | 65 +++++++++++++++++++++++++ packages/livedata/session_view_tests.js | 19 ++++++++ 2 files changed, 84 insertions(+) diff --git a/packages/livedata/livedata_tests.js b/packages/livedata/livedata_tests.js index 02a8f9a6a4..ac1b4ae1ab 100644 --- a/packages/livedata/livedata_tests.js +++ b/packages/livedata/livedata_tests.js @@ -400,6 +400,71 @@ Tinytest.add("livedata - setUserId error when called from server", function(test } }); + +if (Meteor.isServer) { + var pubHandles = {}; +}; +Meteor.methods({ + "livedata/setup" : function (id) { + if (Meteor.isServer) { + pubHandles[id] = {}; + Meteor.publish("pub1"+id, function () { + pubHandles[id].pub1 = this; + this.ready(); + }); + Meteor.publish("pub2"+id, function () { + pubHandles[id].pub2 = this; + this.ready(); + }); + + } + }, + "livedata/pub1go" : function (id) { + if (Meteor.isServer) { + + pubHandles[id].pub1.added("MultiPubCollection" + id, "foo", {a: "aa"}); + return 1; + } + return 0; + }, + "livedata/pub2go" : function (id) { + if (Meteor.isServer) { + pubHandles[id].pub2.added("MultiPubCollection" + id , "foo", {b: "bb"}); + return 2; + } + return 0; + } +}); + +if (Meteor.isClient) { + (function () { + var MultiPub; + var id = Random.id(); + testAsyncMulti("livedata - added from two different subs", [ + function (test, expect) { + Meteor.call('livedata/setup', id, expect(function () {})); + }, + function (test, expect) { + MultiPub = new Meteor.Collection("MultiPubCollection" + id); + var sub1 = Meteor.subscribe("pub1"+id, expect(function () {})); + var sub2 = Meteor.subscribe("pub2"+id, expect(function () {})); + }, + function (test, expect) { + Meteor.call("livedata/pub1go", id, expect(function (err, res) {test.equal(res, 1);})); + }, + function (test, expect) { + test.equal(MultiPub.findOne("foo"), {_id: "foo", a: "aa"}); + }, + function (test, expect) { + Meteor.call("livedata/pub2go", id, expect(function (err, res) {test.equal(res, 2);})); + }, + function (test, expect) { + test.equal(MultiPub.findOne("foo"), {_id: "foo", a: "aa", b: "bb"}); + } + ]); + })(); +}; + if (Meteor.isClient) { testAsyncMulti("livedata - overlapping universal subs", [ function (test, expect) { diff --git a/packages/livedata/session_view_tests.js b/packages/livedata/session_view_tests.js index 3e9b71aac6..ad6f09e2db 100644 --- a/packages/livedata/session_view_tests.js +++ b/packages/livedata/session_view_tests.js @@ -52,6 +52,25 @@ Tinytest.add('livedata - sessionview - exists reveal', function (test) { v.expectNoResult(); }); +Tinytest.add('livedata - sessionview - added a second field in another sub', function (test) { + var v = newView(test); + + v.added("A", "A1", {a: "foo"}); + v.expectResult({fun: 'added', id: "A1", fields: {a: "foo"}}); + v.expectNoResult(); + + v.added("B", "A1", {a: "foo", b: "bar"}); + v.expectResult({fun: 'changed', 'id': "A1", changed: {b: "bar"}}); + + v.removed("A", "A1"); + v.expectNoResult(); + + v.removed("B", "A1"); + v.expectResult({fun: 'removed', id: "A1"}); + v.expectNoResult(); +}); + + Tinytest.add('livedata - sessionview - field reveal', function (test) { var v = newView(test); From 70f1412c0a8ce4fab1f667b2f3e9b766c8791389 Mon Sep 17 00:00:00 2001 From: David Glasser Date: Mon, 18 Mar 2013 10:22:37 -0700 Subject: [PATCH 011/450] Avoid crash on spiderable errors. Fixes #832. --- packages/spiderable/spiderable.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/spiderable/spiderable.js b/packages/spiderable/spiderable.js index dd298bef05..2df1c73224 100644 --- a/packages/spiderable/spiderable.js +++ b/packages/spiderable/spiderable.js @@ -80,7 +80,7 @@ } else { // phantomjs failed. Don't send the error, instead send the // normal page. - if (error.code === 127) + if (error && error.code === 127) Meteor._debug("spiderable: phantomjs not installed. Download and install from http://phantomjs.org/"); else Meteor._debug("spiderable: phantomjs failed:", error, "\nstderr:", stderr); From 331177d002e750c22f94eb5077903ab6702fc223 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Mon, 18 Mar 2013 15:26:55 -0700 Subject: [PATCH 012/450] Comment only. --- packages/spark/spark.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/spark/spark.js b/packages/spark/spark.js index e6c27cf9fc..97ed9006c1 100644 --- a/packages/spark/spark.js +++ b/packages/spark/spark.js @@ -931,8 +931,14 @@ Spark.list = function (cursor, itemFunc, elseFunc) { _.bind(renderer.annotate, renderer) : function (html) { return html; }; - // Templates should have access to data and methods added by the transformer, - // but observeChanges doesn't transform, so we have to do it here. + // Templates should have access to data and methods added by the + // transformer, but observeChanges doesn't transform, so we have to do + // it here. + // + // NOTE: this is a little bit of an abstraction violation. Ideally, + // the only thing Spark should know about Minimongo is the contract of + // observeChanges. In theory, anything that implements observeChanges + // could be passed to Spark.list. But meh. var transformedDoc = function (doc) { if (cursor.getTransform && cursor.getTransform()) return cursor.getTransform()(EJSON.clone(doc)); From 1bc0a0cff8fc6221a8feabcb586b77fa871b192a Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Mon, 18 Mar 2013 20:53:55 -0700 Subject: [PATCH 013/450] Clean up deploy-examples. Now it finds examples automatically. --- admin/deploy-examples.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/admin/deploy-examples.sh b/admin/deploy-examples.sh index a53d572bdc..a9a4a7fcdf 100755 --- a/admin/deploy-examples.sh +++ b/admin/deploy-examples.sh @@ -1,10 +1,18 @@ #!/bin/bash -cd $METEOR_HOME/examples; + +set -e + +cd `dirname $0` +cd ../examples + + read -p "Prefix? " PREFIX; -for EXAMPLE in leaderboard todos wordplay parties -do - cd $EXAMPLE; - echo "meteor deploy $@ $PREFIX-$EXAMPLE;" - meteor deploy $@ $PREFIX-$EXAMPLE; - cd ..; + +for EXAMPLE in * ; do + if [ -d "$EXAMPLE/.meteor" ] ; then + cd $EXAMPLE; + echo "meteor deploy $@ $PREFIX-$EXAMPLE;" + meteor deploy $@ $PREFIX-$EXAMPLE; + cd ..; + fi done From b475c953584e2e183d417bc17ad8bcb7357ac480 Mon Sep 17 00:00:00 2001 From: Naomi Seyfer Date: Tue, 19 Mar 2013 11:23:10 -0700 Subject: [PATCH 014/450] Old note about old event syntax was confusing people on IRC --- docs/client/api.html | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/docs/client/api.html b/docs/client/api.html index f37852d418..bad079157c 100644 --- a/docs/client/api.html +++ b/docs/client/api.html @@ -1806,12 +1806,6 @@ new event handlers in addition to the existing ones. See [Event Maps](#eventmaps) for a detailed description of the event map format and how event handling works in Meteor. -{{#note}} -This syntax replaces the previous syntax: `Template.myTemplate.events = {...}`, -but for now, the old syntax still works. -{{/note}} - - {{> api_box template_helpers}} Each template has a local dictionary of helpers that are made available to it, @@ -2392,7 +2386,7 @@ computation. {{> api_box deps_nonreactive }} Calls `func()` with `Deps.currentComputation` temporarily set to -`null`. If `func` accesses reactive data sources, these data sources +`null`. If `func` accesses reactive data sources, these data sources will never cause a rerun of the enclosing computation. {{> api_box deps_active }} From cb12b195823cde79b760f5f925bb6939d73046cc Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Thu, 13 Dec 2012 16:25:09 -0800 Subject: [PATCH 015/450] Implement files.extractTarGz(...) utility --- app/lib/files.js | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/app/lib/files.js b/app/lib/files.js index a1c178a24b..432429b623 100644 --- a/app/lib/files.js +++ b/app/lib/files.js @@ -1,6 +1,9 @@ var fs = require("fs"); var path = require('path'); var _ = require('underscore'); +var zlib = require("zlib"); +var tar = require("tar"); +var Future = require('fibers/future'); var files = module.exports = { // A sort comparator to order files into load order. @@ -363,6 +366,40 @@ var files = module.exports = { } } throw new Error("failed to make tempory directory in " + tmp_dir); + }, + + // Takes a buffer containing `.tar.gz` data and extracts the archive into + // a destination directory. + extractTarGz: function (buffer, destPath) { + var future, error; + + future = new Future; + zlib.gunzip(buffer, function (e, result) { + error = e; + future['return'](result); + }); + var unzippedBuffer = future.wait(); + if (error) + throw error; + + future = new Future; + var extractor = new tar.Extract({ path: destPath }) + .on('error', function (e) { + error = e; + future['return'](false); + }) + .on('end', function () { + future['return'](true); + }); + // write the unzippedBuffer to the tar extractor; these calls + // cause the tar to be extracted to disk. + extractor.write(unzippedBuffer); + extractor.end(); + future.wait(); + if (error) + throw error; + + // succeed! no return value. } }; From 25c3a683faa5d8c8afd98e113912af6705b4a0b2 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 13 Dec 2012 18:06:08 -0800 Subject: [PATCH 016/450] Directory reorg to mirror the structure of the engine repo --- {app/lib => lib}/app.html.in | 0 {app/lib => lib}/bundler.js | 0 {app/meteor => lib}/deploy.js | 0 {app/lib => lib}/files.js | 0 {app/meteor => lib}/meteor.js | 0 {app/lib => lib}/mongo_exit_codes.js | 0 {app/lib => lib}/mongo_runner.js | 0 {app/lib => lib}/packages.js | 0 {app/meteor => lib}/post-upgrade.js | 0 {app/lib => lib}/project.js | 0 {app/meteor => lib}/run.js | 0 {app/meteor => lib}/skel/.meteor/.gitignore | 0 {app/meteor => lib}/skel/.meteor/packages | 0 {app/meteor => lib}/skel/~name~.css | 0 {app/meteor => lib}/skel/~name~.html | 0 {app/meteor => lib}/skel/~name~.js | 0 {app/lib => lib}/unsupported.html | 0 {app/meteor => lib}/update.js | 0 {app/lib => lib}/updater.js | 0 {admin => tools/admin}/build-release.sh | 0 {admin => tools/admin}/cut-release.sh | 0 {admin => tools/admin}/debian/changelog | 0 {admin => tools/admin}/debian/compat | 0 {admin => tools/admin}/debian/control | 0 {admin => tools/admin}/debian/copyright | 0 {admin => tools/admin}/debian/docs | 0 {admin => tools/admin}/debian/meteor.install | 0 {admin => tools/admin}/debian/meteor.links | 0 {admin => tools/admin}/debian/rules | 0 {admin => tools/admin}/debian/source/format | 0 {admin => tools/admin}/find-new-npm-versions.sh | 0 {admin => tools/admin}/increment-version.js | 0 {admin => tools/admin}/install-s3.sh | 0 {admin => tools/admin}/manifest.json | 0 {admin => tools/admin}/meteor.spec | 0 {admin => tools/admin}/node.sh | 0 {admin => tools/admin}/spark-standalone.sh | 0 {admin => tools}/cli-test.sh | 0 {admin => tools}/generate-dev-bundle.sh | 0 39 files changed, 0 insertions(+), 0 deletions(-) rename {app/lib => lib}/app.html.in (100%) rename {app/lib => lib}/bundler.js (100%) rename {app/meteor => lib}/deploy.js (100%) rename {app/lib => lib}/files.js (100%) rename {app/meteor => lib}/meteor.js (100%) rename {app/lib => lib}/mongo_exit_codes.js (100%) rename {app/lib => lib}/mongo_runner.js (100%) rename {app/lib => lib}/packages.js (100%) rename {app/meteor => lib}/post-upgrade.js (100%) rename {app/lib => lib}/project.js (100%) rename {app/meteor => lib}/run.js (100%) rename {app/meteor => lib}/skel/.meteor/.gitignore (100%) rename {app/meteor => lib}/skel/.meteor/packages (100%) rename {app/meteor => lib}/skel/~name~.css (100%) rename {app/meteor => lib}/skel/~name~.html (100%) rename {app/meteor => lib}/skel/~name~.js (100%) rename {app/lib => lib}/unsupported.html (100%) rename {app/meteor => lib}/update.js (100%) rename {app/lib => lib}/updater.js (100%) rename {admin => tools/admin}/build-release.sh (100%) rename {admin => tools/admin}/cut-release.sh (100%) rename {admin => tools/admin}/debian/changelog (100%) rename {admin => tools/admin}/debian/compat (100%) rename {admin => tools/admin}/debian/control (100%) rename {admin => tools/admin}/debian/copyright (100%) rename {admin => tools/admin}/debian/docs (100%) rename {admin => tools/admin}/debian/meteor.install (100%) rename {admin => tools/admin}/debian/meteor.links (100%) rename {admin => tools/admin}/debian/rules (100%) rename {admin => tools/admin}/debian/source/format (100%) rename {admin => tools/admin}/find-new-npm-versions.sh (100%) rename {admin => tools/admin}/increment-version.js (100%) rename {admin => tools/admin}/install-s3.sh (100%) rename {admin => tools/admin}/manifest.json (100%) rename {admin => tools/admin}/meteor.spec (100%) rename {admin => tools/admin}/node.sh (100%) rename {admin => tools/admin}/spark-standalone.sh (100%) rename {admin => tools}/cli-test.sh (100%) rename {admin => tools}/generate-dev-bundle.sh (100%) diff --git a/app/lib/app.html.in b/lib/app.html.in similarity index 100% rename from app/lib/app.html.in rename to lib/app.html.in diff --git a/app/lib/bundler.js b/lib/bundler.js similarity index 100% rename from app/lib/bundler.js rename to lib/bundler.js diff --git a/app/meteor/deploy.js b/lib/deploy.js similarity index 100% rename from app/meteor/deploy.js rename to lib/deploy.js diff --git a/app/lib/files.js b/lib/files.js similarity index 100% rename from app/lib/files.js rename to lib/files.js diff --git a/app/meteor/meteor.js b/lib/meteor.js similarity index 100% rename from app/meteor/meteor.js rename to lib/meteor.js diff --git a/app/lib/mongo_exit_codes.js b/lib/mongo_exit_codes.js similarity index 100% rename from app/lib/mongo_exit_codes.js rename to lib/mongo_exit_codes.js diff --git a/app/lib/mongo_runner.js b/lib/mongo_runner.js similarity index 100% rename from app/lib/mongo_runner.js rename to lib/mongo_runner.js diff --git a/app/lib/packages.js b/lib/packages.js similarity index 100% rename from app/lib/packages.js rename to lib/packages.js diff --git a/app/meteor/post-upgrade.js b/lib/post-upgrade.js similarity index 100% rename from app/meteor/post-upgrade.js rename to lib/post-upgrade.js diff --git a/app/lib/project.js b/lib/project.js similarity index 100% rename from app/lib/project.js rename to lib/project.js diff --git a/app/meteor/run.js b/lib/run.js similarity index 100% rename from app/meteor/run.js rename to lib/run.js diff --git a/app/meteor/skel/.meteor/.gitignore b/lib/skel/.meteor/.gitignore similarity index 100% rename from app/meteor/skel/.meteor/.gitignore rename to lib/skel/.meteor/.gitignore diff --git a/app/meteor/skel/.meteor/packages b/lib/skel/.meteor/packages similarity index 100% rename from app/meteor/skel/.meteor/packages rename to lib/skel/.meteor/packages diff --git a/app/meteor/skel/~name~.css b/lib/skel/~name~.css similarity index 100% rename from app/meteor/skel/~name~.css rename to lib/skel/~name~.css diff --git a/app/meteor/skel/~name~.html b/lib/skel/~name~.html similarity index 100% rename from app/meteor/skel/~name~.html rename to lib/skel/~name~.html diff --git a/app/meteor/skel/~name~.js b/lib/skel/~name~.js similarity index 100% rename from app/meteor/skel/~name~.js rename to lib/skel/~name~.js diff --git a/app/lib/unsupported.html b/lib/unsupported.html similarity index 100% rename from app/lib/unsupported.html rename to lib/unsupported.html diff --git a/app/meteor/update.js b/lib/update.js similarity index 100% rename from app/meteor/update.js rename to lib/update.js diff --git a/app/lib/updater.js b/lib/updater.js similarity index 100% rename from app/lib/updater.js rename to lib/updater.js diff --git a/admin/build-release.sh b/tools/admin/build-release.sh similarity index 100% rename from admin/build-release.sh rename to tools/admin/build-release.sh diff --git a/admin/cut-release.sh b/tools/admin/cut-release.sh similarity index 100% rename from admin/cut-release.sh rename to tools/admin/cut-release.sh diff --git a/admin/debian/changelog b/tools/admin/debian/changelog similarity index 100% rename from admin/debian/changelog rename to tools/admin/debian/changelog diff --git a/admin/debian/compat b/tools/admin/debian/compat similarity index 100% rename from admin/debian/compat rename to tools/admin/debian/compat diff --git a/admin/debian/control b/tools/admin/debian/control similarity index 100% rename from admin/debian/control rename to tools/admin/debian/control diff --git a/admin/debian/copyright b/tools/admin/debian/copyright similarity index 100% rename from admin/debian/copyright rename to tools/admin/debian/copyright diff --git a/admin/debian/docs b/tools/admin/debian/docs similarity index 100% rename from admin/debian/docs rename to tools/admin/debian/docs diff --git a/admin/debian/meteor.install b/tools/admin/debian/meteor.install similarity index 100% rename from admin/debian/meteor.install rename to tools/admin/debian/meteor.install diff --git a/admin/debian/meteor.links b/tools/admin/debian/meteor.links similarity index 100% rename from admin/debian/meteor.links rename to tools/admin/debian/meteor.links diff --git a/admin/debian/rules b/tools/admin/debian/rules similarity index 100% rename from admin/debian/rules rename to tools/admin/debian/rules diff --git a/admin/debian/source/format b/tools/admin/debian/source/format similarity index 100% rename from admin/debian/source/format rename to tools/admin/debian/source/format diff --git a/admin/find-new-npm-versions.sh b/tools/admin/find-new-npm-versions.sh similarity index 100% rename from admin/find-new-npm-versions.sh rename to tools/admin/find-new-npm-versions.sh diff --git a/admin/increment-version.js b/tools/admin/increment-version.js similarity index 100% rename from admin/increment-version.js rename to tools/admin/increment-version.js diff --git a/admin/install-s3.sh b/tools/admin/install-s3.sh similarity index 100% rename from admin/install-s3.sh rename to tools/admin/install-s3.sh diff --git a/admin/manifest.json b/tools/admin/manifest.json similarity index 100% rename from admin/manifest.json rename to tools/admin/manifest.json diff --git a/admin/meteor.spec b/tools/admin/meteor.spec similarity index 100% rename from admin/meteor.spec rename to tools/admin/meteor.spec diff --git a/admin/node.sh b/tools/admin/node.sh similarity index 100% rename from admin/node.sh rename to tools/admin/node.sh diff --git a/admin/spark-standalone.sh b/tools/admin/spark-standalone.sh similarity index 100% rename from admin/spark-standalone.sh rename to tools/admin/spark-standalone.sh diff --git a/admin/cli-test.sh b/tools/cli-test.sh similarity index 100% rename from admin/cli-test.sh rename to tools/cli-test.sh diff --git a/admin/generate-dev-bundle.sh b/tools/generate-dev-bundle.sh similarity index 100% rename from admin/generate-dev-bundle.sh rename to tools/generate-dev-bundle.sh From 58c6f55676f98da5e4bac1afea0b299e82bbe19e Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 13 Dec 2012 19:05:26 -0800 Subject: [PATCH 017/450] Apply changes made on engine repo --- .gitignore | 3 +- lib/bundler.js | 9 +- lib/files.js | 36 +---- lib/meteor.js | 22 +-- lib/packages.js | 259 +++++++++++++++++++++++++++---- lib/project.js | 20 ++- lib/run.js | 8 +- lib/skel/.meteor/version | 1 + meteor | 4 +- packages/handlebars/package.js | 4 +- packages/handlebars/parse.js | 6 +- packages/session/package.js | 3 +- packages/showdown/package.js | 3 +- packages/templating/package.js | 6 +- {app/server => server}/server.js | 0 tools/cli-test.sh | 51 +++++- 16 files changed, 322 insertions(+), 113 deletions(-) create mode 100644 lib/skel/.meteor/version rename {app/server => server}/server.js (100%) diff --git a/.gitignore b/.gitignore index cd47ae6e54..bcc5e6b119 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store +cache *~ /dev_bundle /dev_bundle*.tar.gz @@ -11,4 +12,4 @@ *.sublime-workspace TAGS *.log -*.out \ No newline at end of file +*.out diff --git a/lib/bundler.js b/lib/bundler.js index 2c172f89c6..a30533d27b 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -43,6 +43,7 @@ var fs = require('fs'); var uglify = require('uglify-js'); var cleanCSS = require('clean-css'); var _ = require('underscore'); +var project = require(path.join(__dirname, 'project.js')); // files to ignore when bundling. node has no globs, so use regexps var ignore_files = [ @@ -499,8 +500,10 @@ _.extend(Bundle.prototype, { var dependencies_json = {core: [], app: [], packages: {}}; var is_app = files.is_app_dir(project_dir); - if (is_app) + if (is_app) { dependencies_json.app.push(path.join('.meteor', 'packages')); + dependencies_json.app.push(path.join('.meteor', 'version')); + } // --- Set up build area --- @@ -717,8 +720,8 @@ exports.bundle = function (project_dir, output_path, options) { // Create a bundle, add the project packages.flush(); var bundle = new Bundle; - var project = packages.get_for_dir(project_dir, ignore_files); - bundle.use(project); + var pkg = packages.get_for_dir(project_dir, ignore_files); // `package` is a reserved keyword + bundle.use(pkg); // Include tests if requested if (options.include_tests) { diff --git a/lib/files.js b/lib/files.js index 432429b623..5fe9f59a33 100644 --- a/lib/files.js +++ b/lib/files.js @@ -185,7 +185,7 @@ var files = module.exports = { // an installation.) in_checkout: function () { try { - if (fs.existsSync(path.join(__dirname, '..', '..', '.git'))) + if (fs.existsSync(path.join(__dirname, '..', '.git'))) return true; } catch (e) { console.log(e);} @@ -195,38 +195,10 @@ var files = module.exports = { // Return the root of dev_bundle (probably /usr/local/meteor in an // install, or (checkout root)/dev_bundle in a checkout..) get_dev_bundle: function () { - if (files.in_checkout()) { - return path.join(__dirname, '..', '..', 'dev_bundle'); - } + if (files.in_checkout()) + return path.join(__dirname, '..', 'dev_bundle'); else - return path.join(__dirname, '..', '..'); - }, - - // returns a list of places where packages can be found. - // 1. directories set via process.env.PACKAGES_DIRS - // 2. default is packages/ in the meteor directory - // XXX: 3. a per project directory? (vendor/packages in rails parlance?) - get_package_dirs: function() { - var package_dirs = [path.join(__dirname, '..', '..', 'packages')]; - if (process.env.PACKAGE_DIRS) - package_dirs = process.env.PACKAGE_DIRS.split(':').concat(package_dirs); - - return package_dirs; - }, - - // search package dirs for a package named name. - // undefined if the package isn't in any dir - get_package_dir: function (name) { - var ret; - _.find(this.get_package_dirs(), function(package_dir) { - var dir = path.join(package_dir, name); - if (fs.existsSync(path.join(dir, 'package.js'))) { - ret = dir; - return true; - } - }); - - return ret; + return path.join(__dirname, '..'); }, // Return the directory that contains the core tool (the top-level diff --git a/lib/meteor.js b/lib/meteor.js index 2b0d738594..91c1bb73f6 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -2,7 +2,7 @@ var Fiber = require('fibers'); Fiber(function () { var path = require('path'); - var files = require(path.join(__dirname, '..', 'lib', 'files.js')); + var files = require(path.join(__dirname, 'files.js')); var _ = require('underscore'); var deploy = require(path.join(__dirname, 'deploy')); var fs = require("fs"); @@ -71,7 +71,7 @@ Fiber(function () { var find_mongo_port = function (cmd, callback) { var app_dir = require_project(cmd); - var mongo_runner = require(path.join(__dirname, '..', 'lib', 'mongo_runner.js')); + var mongo_runner = require(path.join(__dirname, 'mongo_runner.js')); mongo_runner.find_mongo_port(app_dir, callback); }; @@ -280,8 +280,8 @@ Fiber(function () { } var app_dir = require_project('add'); - var packages = require(path.join(__dirname, '..', 'lib', 'packages.js')); - var project = require(path.join(__dirname, '..', 'lib', 'project.js')); + var packages = require(path.join(__dirname, 'packages.js')); + var project = require(path.join(__dirname, 'project.js')); var all = packages.list(); var using = {}; _.each(project.get_packages(app_dir), function (name) { @@ -317,8 +317,8 @@ Fiber(function () { } var app_dir = require_project('remove'); - var packages = require(path.join(__dirname, '..', 'lib', 'packages.js')); - var project = require(path.join(__dirname, '..', 'lib', 'project.js')); + var packages = require(path.join(__dirname, 'packages.js')); + var project = require(path.join(__dirname, 'project.js')); var using = {}; _.each(project.get_packages(app_dir), function (name) { using[name] = true; @@ -352,7 +352,7 @@ Fiber(function () { if (argv.using) { var app_dir = require_project('list --using'); - var using = require(path.join(__dirname, '..', 'lib', 'project.js')).get_packages(app_dir); + var using = require(path.join(__dirname, 'project.js')).get_packages(app_dir); if (using.length) { _.each(using, function (name) { @@ -369,7 +369,7 @@ Fiber(function () { return; } - var list = require(path.join(__dirname, '..', 'lib', 'packages.js')).list(); + var list = require(path.join(__dirname, 'packages.js')).list(); var names = _.keys(list); names.sort(); var pkgs = []; @@ -377,7 +377,7 @@ Fiber(function () { pkgs.push(list[name]); }); process.stdout.write("\n" + - require(path.join(__dirname, '..', 'lib', 'packages.js')).format_list(pkgs) + + require(path.join(__dirname, 'packages.js')).format_list(pkgs) + "\n"); } }); @@ -411,7 +411,7 @@ Fiber(function () { var bundle_path = path.join(build_dir, 'bundle'); var output_path = path.resolve(argv._[0]); // get absolute path - var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); + var bundler = require(path.join(__dirname, 'bundler.js')); var errors = bundler.bundle(app_dir, bundle_path); if (errors) { process.stdout.write("Errors prevented bundling:\n"); @@ -644,7 +644,7 @@ Fiber(function () { } if (argv.version) { - var updater = require(path.join(__dirname, '..', 'lib', 'updater.js')); + var updater = require(path.join(__dirname, 'updater.js')); var sha = updater.git_sha(); process.stdout.write("Meteor version " + updater.CURRENT_VERSION); diff --git a/lib/packages.js b/lib/packages.js index 3a0c3a6a62..4de640aa43 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -2,6 +2,14 @@ var path = require('path'); var _ = require('underscore'); var files = require(path.join(__dirname, 'files.js')); var fs = require('fs'); +var https = require('https'); +var Future = require('fibers/future'); +var request = require('request'); + +// XXX make this work with packages.meteor.com. A CNAME record isn't +// good enough because then our SSL certs are wrong (or maybe we can +// get this to work somehow else) +var PACKAGES_URLBASE = 'https://d3fm2vapipm3k9.cloudfront.net'; // Under the hood, packages in the library (/package/foo), and user // applications, are both Packages -- they are just represented @@ -76,35 +84,52 @@ var Package = function () { throw new Error("This package has already registered a handler for " + extension); self.extensions[extension] = callback; + }, + + // Same as node's default `require` but is relative to the + // package's directory. Regular `require` doesn't work well + // because we read the package.js file and `runInThisContext` it + // separately as a string. This means that paths are relative to + // the top-level meteor.js script rather than the location of + // package.js + require: function(filename) { + return require(path.join(self.source_root, filename)); } }; }; _.extend(Package.prototype, { - init_from_library: function (name) { + // @returns {Boolean} was the package found in any local package sets? + initFromLocalPackageSets: function (name) { var self = this; - self.name = name; - self.source_root = files.get_package_dir(name); - self.serve_root = path.join(path.sep, 'packages', name); + var setsDir = path.join(__dirname, '..', 'local-package-sets'); + if (!fs.existsSync(setsDir)) + return false; - if (!self.source_root) - throw new Error("The package named " + self.name + " does not exist."); + var sets = fs.readdirSync(setsDir); + var found = false; + _.each(sets, function (set) { - var fullpath = path.join(self.source_root, 'package.js'); - var code = fs.readFileSync(fullpath).toString(); - // \n is necessary in case final line is a //-comment - var wrapped = "(function(Package,require){" + code + "\n})"; - // XXX it'd be nice to runInNewContext so that the package - // setup code can't mess with our globals, but objects that - // come out of runInNewContext have bizarro antimatter - // prototype chains and break 'instanceof Array'. for now, - // steer clear - var func = require('vm').runInThisContext(wrapped, fullpath, true); - // XXX would be nice to eliminate require. packages like - // 'templating' use this to load other code to run at - // bundle-time. and to pull in, eg, 'fs' and 'path' to access - // the file system - func(self.declarationFuncs, require); + if (found) + return; + + var setDir = path.join(setsDir, set); + if (fs.statSync(setDir).isDirectory()) { + var packageNames = fs.readdirSync(setDir); + if (_.contains(packageNames, name)) { + found = true; + self._initFromPackageDir(name, + path.join(setsDir, set, name)); + } + } + }); + + return found; + }, + + initFromPackageCache: function (name, version) { + this._initFromPackageDir(name, + path.join(__dirname, '..', 'cache', 'packages', name, version)); }, init_from_app_dir: function (app_dir, ignore_files) { @@ -196,6 +221,37 @@ _.extend(Package.prototype, { }); }, + _initFromPackageDir: function (name, dir) { + var self = this; + self.name = name; + self.source_root = dir; + self.serve_root = path.join(path.sep, 'packages', name); + + if (!fs.existsSync(self.source_root)) + throw new Error("The package named " + self.name + " does not exist."); + + // We use string concatenation to load package.js rather than + // directly `require`ing it because that allows us to simplify the + // package API (such as supporting Package.on_use rather than + // something like Package.current().on_use) + + var fullpath = path.join(self.source_root, 'package.js'); + var code = fs.readFileSync(fullpath).toString(); + // \n is necessary in case final line is a //-comment + var wrapped = "(function(Package,require){" + code + "\n})"; + // XXX it'd be nice to runInNewContext so that the package + // setup code can't mess with our globals, but objects that + // come out of runInNewContext have bizarro antimatter + // prototype chains and break 'instanceof Array'. for now, + // steer clear + var func = require('vm').runInThisContext(wrapped, fullpath, true); + // XXX would be nice to eliminate require. packages like + // 'templating' use this to load other code to run at + // bundle-time. and to pull in, eg, 'fs' and 'path' to access + // the file system + func(self.declarationFuncs, require); + }, + init_from_collection: function (collection_dir) { var self = this; self.name = null; @@ -216,13 +272,36 @@ _.extend(Package.prototype, { var package_cache = {}; var packages = module.exports = { + + // @param manifest {Object} parsed manifest file for appropriate meteor release + // + // XXX we should probably instead refactor packages to be a class + // rather than a global object, in which case this would be passed + // into the constructor and ensureManifestSet() would no longer be + // needed. + setManifest: function (manifest) { + if (this.manifest) + throw new Error("Can't set manifest twice"); + this.manifest = manifest; + }, + + ensureManifestSet: function() { + if (!this.manifest) + throw new Error( + "No manifest set. You need to call setManifest before taking this action."); + }, + // get a package by name. also maps package objects to themselves. get: function (name) { + var self = this; if (name instanceof Package) return name; if (!(name in package_cache)) { var pkg = new Package; - pkg.init_from_library(name); + if (!pkg.initFromLocalPackageSets(name)) { + self.ensureManifestSet(); + pkg.initFromPackageCache(name, self.manifest.packages[name]); + } package_cache[name] = pkg; } @@ -265,20 +344,42 @@ var packages = module.exports = { package_cache = {}; }, - // get all packages in the directory, in a map from package name to - // a package object. + // get all packages available, in both local package sets and in the package + // cache (in case we are in an app directory) + // returns {Object} maps name to Package list: function () { - var ret = {}; + var self = this; + var list = {}; - _.each(files.get_package_dirs(), function(dir) { - _.each(fs.readdirSync(dir), function (name) { - // skip .meteor directory - if (fs.existsSync(path.join(dir, name, 'package.js'))) - ret[name] = packages.get(name); + var setsDir = path.join(__dirname, '..', 'local-package-sets'); + if (fs.existsSync(setsDir)) { + var sets = fs.readdirSync(setsDir); + _.each(sets, function (set) { + var setDir = path.join(setsDir, set); + if (fs.statSync(setDir).isDirectory()) { + var packageNames = fs.readdirSync(path.join(setsDir, set)); + _.each(packageNames, function (name) { + if (!list[name]) + list[name] = packages.get(name); + else + throw new Error("Found package " + name + " more than once in local package sets."); + }); + } }); - }) + } - return ret; + if (self.manifest) { + _.each(self.manifest.packages, function(version, name) { + // don't even look for packages if they've already been + // overridden (though this `if` isn't necessary for + // correctness, since `packages.get` looks for packages in the + // override directories first anyways) + if (!list[name]) + list[name] = packages.get(name); + }); + } + + return list; }, // returns a pretty list suitable for showing to the user. input is @@ -308,5 +409,97 @@ var packages = module.exports = { }); return out; + }, + + existsInPackageCache: function (name, version) { + return fs.existsSync(path.join(__dirname, '..', 'cache', 'packages', name, version)); + }, + + // fetches the manifest file for the given release version. also fetches + // all of the missing versioned packages referenced from the manifest + // @param releaseVersion {String} eg "0.1" + // @returns {Object} parsed manifest file + populateCacheForReleaseVersion: function(releaseVersion) { + var self = this; + var future = new Future; + var manifestDir = path.join(__dirname, '..', 'cache'); + files.mkdir_p(manifestDir, 0755); + var manifestPath = path.join(manifestDir, releaseVersion + '.json'); + + // load the manifest from s3, and store in the cache + try { + // a variation on request that has a standard (error, result) callback. + // this way we can use Future.wrap + var request2 = function(url, cb) { + request(url, function(error, response, body) { + cb(error, body); + }); + }; + + var manifest = Future.wrap(request2)( + PACKAGES_URLBASE + "/manifest/" + releaseVersion + ".json").wait(); + fs.writeFileSync(manifestPath, manifest); + return JSON.parse(manifest); + } catch (e) { + console.error( + "Can't find manifest for meteor release version " + releaseVersion); + throw e; + } + }, + + // Look for ./.meteor/version. If it exists, presumable since we're + // inside a project load the manifest corresponding to that meteor + // release version + loadManifestForProjectIfExists: function () { + var self = this; + var project = require(path.join(__dirname, 'project.js')); + if (fs.existsSync('.meteor/version')) { + var releaseVersion = project.getMeteorReleaseVersion(); + var manifestPath = path.join( + __dirname, '..', 'cache', 'manifest', releaseVersion + '.json'); + + var manifest; + if (fs.existsSync(manifestPath)) { + + // read from cache + manifest = JSON.parse(fs.readFileSync(manifestPath)); + packages.setManifest(manifest); + } else { + + // grow cache with new manifest and packages + manifest = packages.populateCacheForReleaseVersion(releaseVersion); + packages.setManifest(manifest); + } + + var exec = require('child_process').exec; + var Future = require('fibers/future'); + var futures = []; + _.each(manifest.packages, function (version, name) { + if (!self.existsInPackageCache(name, version)) { + var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version); + files.mkdir_p(packageDir); + + var future = new Future; + futures.push(future); + exec( + "curl " + PACKAGES_URLBASE + "/packages/" + name + "/" + + version + ".tar.gz" + + "| tar xvz", + {cwd: packageDir}, + function (error, stdout, stderr) { + console.log('stdout: ' + stdout); + console.log('stderr: ' + stderr); + if (error !== null) { + console.log('exec error: ' + error); + } + future['return'](); + }); + } + }); + + Future.wait(futures); + } } -} +}; + +packages.loadManifestForProjectIfExists(); diff --git a/lib/project.js b/lib/project.js index fdf7353b8b..19594ea8b9 100644 --- a/lib/project.js +++ b/lib/project.js @@ -4,8 +4,8 @@ var _ = require('underscore'); var project = module.exports = { - _get_lines: function (app_dir) { - var raw = fs.readFileSync(path.join(app_dir, '.meteor', 'packages'), 'utf8'); + _get_lines: function (file) { + var raw = fs.readFileSync(file, 'utf8'); var lines = raw.split(/\r*\n\r*/); // strip blank lines at the end @@ -19,6 +19,10 @@ var project = module.exports = { return lines; }, + _get_packages_lines: function (app_dir) { + return project._get_lines(path.join(app_dir, '.meteor', 'packages')); + }, + _trim_line: function (line) { var match = line.match(/^([^#]*)#/); if (match) @@ -32,11 +36,11 @@ var project = module.exports = { lines.join('\n') + '\n', 'utf8'); }, - // Packages used by this project. + // Package names used by this project. get_packages: function (app_dir) { var ret = []; - _.each(project._get_lines(app_dir), function (line) { + _.each(project._get_packages_lines(app_dir), function (line) { line = project._trim_line(line); if (line !== '') ret.push(line); @@ -45,8 +49,12 @@ var project = module.exports = { return ret; }, + getMeteorReleaseVersion: function (appDir) { + return project._trim_line(project._get_lines(path.join(appDir, '.meteor', 'version'))[0]); + }, + add_package: function (app_dir, name) { - var lines = project._get_lines(app_dir); + var lines = project._get_packages_lines(app_dir); // detail: if the file starts with a comment, try to keep a single // blank line after the comment (unless the user removes it) @@ -59,7 +67,7 @@ var project = module.exports = { remove_package: function (app_dir, name) { // XXX assume no special regexp characters - var lines = _.reject(project._get_lines(app_dir), function (line) { + var lines = _.reject(project._get_packages_lines(app_dir), function (line) { return project._trim_line(line) === name; }); project._write_packages(app_dir, lines); diff --git a/lib/run.js b/lib/run.js index 23c776c0d9..1143ccf3f4 100644 --- a/lib/run.js +++ b/lib/run.js @@ -337,12 +337,8 @@ var DependencyWatcher = function (deps, app_dir, relativeFiles, on_change) { // Additional list of specific files that are interesting. self.specific_files = {}; - for (var pkg in (deps.packages || {})) { - _.each(deps.packages[pkg], function (file) { - self.specific_files[path.join(files.get_package_dir(pkg), file)] - = true; - }); - }; + // #PackageOverride use this mechanism to support local package + // override directories _.each(relativeFiles, function (file) { self.specific_files[file] = true; diff --git a/lib/skel/.meteor/version b/lib/skel/.meteor/version new file mode 100644 index 0000000000..49d59571fb --- /dev/null +++ b/lib/skel/.meteor/version @@ -0,0 +1 @@ +0.1 diff --git a/meteor b/meteor index 2f7aa3573d..0fc4b07c03 100755 --- a/meteor +++ b/meteor @@ -83,11 +83,11 @@ if [ -d "$SCRIPT_DIR/.git" ] || [ -f "$SCRIPT_DIR/.git" ]; then fi DEV_BUNDLE="$SCRIPT_DIR/dev_bundle" - METEOR="$SCRIPT_DIR/app/meteor/meteor.js" + METEOR="$SCRIPT_DIR/lib/meteor.js" else # In an install DEV_BUNDLE=$(dirname "$SCRIPT_DIR") - METEOR="$DEV_BUNDLE/app/meteor/meteor.js" + METEOR="$DEV_BUNDLE/lib/meteor.js" fi diff --git a/packages/handlebars/package.js b/packages/handlebars/package.js index a6303f9fce..256fa6a260 100644 --- a/packages/handlebars/package.js +++ b/packages/handlebars/package.js @@ -9,11 +9,13 @@ Package.describe({ summary: "Simple semantic templating language" }); -require(path.join('..', '..', 'packages', 'handlebars', 'parse.js')); // XXX lame!! +Package.require('parse.js'); // needed at bundle time Package.on_use(function (api) { // XXX should only be sent if we have handlebars templates in the app.. api.add_files('evaluate.js', 'client'); + api.add_files('parse.js', 'server'); // needed on server for tests + api.use('underscore', 'client'); }); diff --git a/packages/handlebars/parse.js b/packages/handlebars/parse.js index 6ba6879090..a08e743d5f 100644 --- a/packages/handlebars/parse.js +++ b/packages/handlebars/parse.js @@ -36,11 +36,7 @@ Handlebars.to_json_ast = function (code) { var req = (typeof require === 'undefined' ? __meteor_bootstrap__.require : require); var path = req('path'); - - var _ = global._; - if (! _) - _ = req(path.join('..', '..', 'packages', 'underscore', 'underscore.js')); // XXX super lame - + var _ = req("underscore"); var ast = req("handlebars").parse(code); // Recreate Handlebars.Exception to properly report error messages diff --git a/packages/session/package.js b/packages/session/package.js index 2b2ca13bd9..4a5f7e53db 100644 --- a/packages/session/package.js +++ b/packages/session/package.js @@ -5,8 +5,7 @@ Package.describe({ internal: true }); -// XXX hack -- need a way to use a package at bundle time -var _ = require(path.join('..', '..', 'packages', 'underscore', 'underscore.js')); +var _ = require('underscore'); // needed at bundle time Package.on_use(function (api) { api.use(['underscore', 'deps'], 'client'); diff --git a/packages/showdown/package.js b/packages/showdown/package.js index 95b53fd166..4fd274154d 100644 --- a/packages/showdown/package.js +++ b/packages/showdown/package.js @@ -8,8 +8,7 @@ Package.describe({ summary: "Markdown-to-HTML processor" }); -// XXX hack -- need a way to use a package at bundle time -var _ = require(path.join('..', '..', 'packages', 'underscore', 'underscore.js')); +var _ = require('underscore'); Package.on_use(function (api, where) { where = where || ["client", "server"]; diff --git a/packages/templating/package.js b/packages/templating/package.js index 48a9b64423..2fb38b2020 100644 --- a/packages/templating/package.js +++ b/packages/templating/package.js @@ -40,9 +40,7 @@ Package.register_extension( // religion on that var contents = fs.readFileSync(source_path); - // XXX super lame! we actually have to give paths relative to - // app/inner/app.js, since that's who's evaling us. - var html_scanner = require(path.join('..', '..', 'packages', 'templating', 'html_scanner.js')); + var html_scanner = Package.require('html_scanner.js'); var results = html_scanner.scan(contents.toString('utf8'), source_path); if (results.head) @@ -84,12 +82,12 @@ Package.on_test(function (api) { api.use('tinytest'); api.use('htmljs'); api.use(['test-helpers', 'domutils', 'session'], 'client'); + api.use('handlebars', 'server'); api.add_files([ 'templating_tests.js', 'templating_tests.html' ], 'client'); api.add_files([ - path.join('..', 'handlebars', 'parse.js'), // XXX hacky 'html_scanner.js', 'scanner_tests.js' ], 'server'); diff --git a/app/server/server.js b/server/server.js similarity index 100% rename from app/server/server.js rename to server/server.js diff --git a/tools/cli-test.sh b/tools/cli-test.sh index 8f1b262b8f..c76f446b09 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -4,7 +4,8 @@ # To test the installed meteor, pass in --global cd `dirname $0` -METEOR=`pwd`/../meteor +METEOR_DIR=`pwd`/.. +METEOR=$METEOR_DIR/meteor if [ -z "$NODE" ]; then NODE=`pwd`/node.sh @@ -12,14 +13,15 @@ fi #If this ever takes more options, use getopt if [ "$1" == "--global" ]; then - METEOR=meteor + METEOR_DIR=/usr/local/meteor + METEOR=/usr/local/bin/meteor fi DIR=`mktemp -d -t meteor-cli-test-XXXXXXXX` -trap 'echo FAILED ; rm -rf "$DIR" >/dev/null 2>&1' EXIT +trap 'echo FAILED ; rm -rfd `find $METEOR_DIR -name __tmp`; rm -rf "$DIR" >/dev/null 2>&1' EXIT cd "$DIR" -set -e +set -e -x ## Begin actual tests @@ -162,7 +164,46 @@ EOF $METEOR -p $PORT --settings='settings.json' --once > /dev/null -# XXX more tests here! + +# prepare die.js so that we have a server that loads packages and dies +cat > die.js < $METEOR_DIR/local-package-sets/__tmp/a-package-named-bar/package.js < /dev/null +$METEOR -p $PORT --once | grep "loaded a-package-named-bar" > /dev/null + +rm -rf $METEOR_DIR/local-package-sets/__tmp/ + + +echo "... local-package-sets -- overridden package" + +mkdir -p $METEOR_DIR/local-package-sets/__tmp/accounts-ui/ +cat > $METEOR_DIR/local-package-sets/__tmp/accounts-ui/package.js <&1 | grep "accounts-ui - overridden" > /dev/null +$METEOR list | grep "accounts-ui - overridden" > /dev/null + +rm -rf $METEOR_DIR/local-package-sets/__tmp/ + + +# remove die.js, we're done with package tests. +rm die.js + From 21cd1bee2f59c19380e0a2e6c303edb354cb20e3 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 14 Dec 2012 16:16:30 -0800 Subject: [PATCH 018/450] Move node out of admin/ into tools/; also fixes it --- tools/{admin => }/node.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{admin => }/node.sh (100%) diff --git a/tools/admin/node.sh b/tools/node.sh similarity index 100% rename from tools/admin/node.sh rename to tools/node.sh From 067ec181391b095cc2eaef313e77769d8b41e0ae Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Fri, 14 Dec 2012 16:18:03 -0800 Subject: [PATCH 019/450] untargz in process when fetching packages --- lib/files.js | 9 +++++++++ lib/packages.js | 38 +++++++++++++++++++++----------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/files.js b/lib/files.js index 5fe9f59a33..6abf92e13d 100644 --- a/lib/files.js +++ b/lib/files.js @@ -4,6 +4,7 @@ var _ = require('underscore'); var zlib = require("zlib"); var tar = require("tar"); var Future = require('fibers/future'); +var request = require('request'); var files = module.exports = { // A sort comparator to order files into load order. @@ -372,6 +373,14 @@ var files = module.exports = { throw error; // succeed! no return value. + }, + + // A thin wrapper around request(...) that makes the response "body" + // the main callback argument. This facilitates using `Future.wrap`. + getUrl: function (urlOrOptions, callback) { + return request(urlOrOptions, function (error, response, body) { + callback(error, body, response); + }); } }; diff --git a/lib/packages.js b/lib/packages.js index 4de640aa43..7d1920e202 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -412,7 +412,11 @@ var packages = module.exports = { }, existsInPackageCache: function (name, version) { - return fs.existsSync(path.join(__dirname, '..', 'cache', 'packages', name, version)); + // Look for presence of "package.js" file in directory so we don't count + // an empty dir as a package. An empty dir could be left by a failed + // package untarring, for example. + return fs.existsSync(path.join(__dirname, '..', 'cache', 'packages', name, version, + 'package.js')); }, // fetches the manifest file for the given release version. also fetches @@ -477,27 +481,27 @@ var packages = module.exports = { _.each(manifest.packages, function (version, name) { if (!self.existsInPackageCache(name, version)) { var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version); - files.mkdir_p(packageDir); + var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + + version + ".tar.gz"; - var future = new Future; - futures.push(future); - exec( - "curl " + PACKAGES_URLBASE + "/packages/" + name + "/" - + version + ".tar.gz" - + "| tar xvz", - {cwd: packageDir}, - function (error, stdout, stderr) { - console.log('stdout: ' + stdout); - console.log('stderr: ' + stderr); - if (error !== null) { - console.log('exec error: ' + error); - } - future['return'](); + console.log("Fetching " + packageUrl + "..."); + futures.push(Future.wrap(function (cb) { + files.getUrl({url: packageUrl, encoding: null}, function (error, result) { + if (! error && result) + result = { buffer: result, packageDir: packageDir }; + cb(error, result); }); - } + })()); + } }); Future.wait(futures); + + _.each(futures, function (f) { + var result = f.get(); + files.mkdir_p(result.packageDir); + files.extractTarGz(result.buffer, result.packageDir); + }); } } }; From 55349ca97424755df753830d2e5daaa7aff304d7 Mon Sep 17 00:00:00 2001 From: Nick Martin Date: Fri, 14 Dec 2012 17:26:42 -0800 Subject: [PATCH 020/450] Use packages.meteor.com instead of cloudfront. --- lib/packages.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/packages.js b/lib/packages.js index 7d1920e202..5db876456f 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -6,10 +6,7 @@ var https = require('https'); var Future = require('fibers/future'); var request = require('request'); -// XXX make this work with packages.meteor.com. A CNAME record isn't -// good enough because then our SSL certs are wrong (or maybe we can -// get this to work somehow else) -var PACKAGES_URLBASE = 'https://d3fm2vapipm3k9.cloudfront.net'; +var PACKAGES_URLBASE = 'https://packages.meteor.com'; // Under the hood, packages in the library (/package/foo), and user // applications, are both Packages -- they are just represented From 460911c4064e3392f254e1b627ff1f7ef99cd72f Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 14 Dec 2012 17:45:26 -0800 Subject: [PATCH 021/450] Use files.getUrl instead of ad-hoc solution --- lib/packages.js | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/lib/packages.js b/lib/packages.js index 5db876456f..e86869504c 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -429,15 +429,7 @@ var packages = module.exports = { // load the manifest from s3, and store in the cache try { - // a variation on request that has a standard (error, result) callback. - // this way we can use Future.wrap - var request2 = function(url, cb) { - request(url, function(error, response, body) { - cb(error, body); - }); - }; - - var manifest = Future.wrap(request2)( + var manifest = Future.wrap(files.getUrl)( PACKAGES_URLBASE + "/manifest/" + releaseVersion + ".json").wait(); fs.writeFileSync(manifestPath, manifest); return JSON.parse(manifest); From 1bb59b22d8d45cf00201c85c679c4e2e5e558589 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 17 Dec 2012 19:36:06 -0800 Subject: [PATCH 022/450] fix meteor create (path change) --- lib/meteor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/meteor.js b/lib/meteor.js index 91c1bb73f6..b3d24e015b 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -169,7 +169,7 @@ Fiber(function () { var new_argv = opt.argv; var appname; - var example_dir = path.join(__dirname, '..', '..', 'examples'); + var example_dir = path.join(__dirname, '..', 'examples'); var examples = _.reject(fs.readdirSync(example_dir), function (e) { return (e === 'unfinished' || e === 'other' || e[0] === '.'); }); From b145dc48de4471c96a9efa1232bcdaea306b9169 Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 17 Dec 2012 19:42:25 -0800 Subject: [PATCH 023/450] make install.sh work on engine branch* * XXX this copies the meteor source in lib into the same lib directory as dev_bundle/lib! that's not right, but it works. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index c20affa9d4..d979b57f6b 100755 --- a/install.sh +++ b/install.sh @@ -33,7 +33,7 @@ function CPR { tar -c --exclude .meteor/local "$1" | tar -x -C "$2" } cp meteor "$TARGET_DIR/bin" -CPR app "$TARGET_DIR" +CPR lib "$TARGET_DIR" CPR packages "$TARGET_DIR" CPR examples "$TARGET_DIR" rm -rf "$TARGET_DIR"/examples/unfinished From 8527cf2739d10af8fa88ba709790f27f6fa7a8fe Mon Sep 17 00:00:00 2001 From: David Greenspan Date: Mon, 17 Dec 2012 19:43:24 -0800 Subject: [PATCH 024/450] use in-process node-tar for all create and extract --- lib/deploy.js | 5 ++--- lib/files.js | 10 ++++++++++ lib/meteor.js | 25 +++++++++++-------------- lib/update.js | 51 +++++++++++++++++++++++++-------------------------- 4 files changed, 48 insertions(+), 43 deletions(-) diff --git a/lib/deploy.js b/lib/deploy.js index d06953c13c..9ced23df1d 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -116,8 +116,7 @@ var bundle_and_deploy = function (options) { // When it hits the wire, all these opts will be URL-encoded. if (settings !== undefined) rpcOptions.settings = settings; - var tar = child_process.spawn( - 'tar', ['czf', '-', 'bundle'], {cwd: build_dir}); + var tar = files.createTarGzStream(path.join(build_dir, 'bundle')); var rpc = meteor_rpc('deploy', 'POST', site, rpcOptions, function (err, body) { if (err) { @@ -168,7 +167,7 @@ var bundle_and_deploy = function (options) { } }); - tar.stdout.pipe(rpc); + tar.pipe(rpc); }; var delete_app = function (url) { diff --git a/lib/files.js b/lib/files.js index 6abf92e13d..3f39129fc7 100644 --- a/lib/files.js +++ b/lib/files.js @@ -6,6 +6,8 @@ var tar = require("tar"); var Future = require('fibers/future'); var request = require('request'); +var fstream = require('fstream'); + var files = module.exports = { // A sort comparator to order files into load order. sort: function (a, b) { @@ -375,6 +377,14 @@ var files = module.exports = { // succeed! no return value. }, + // Tar-gzips a directory, returning a stream that can then + // be piped as needed. The tar archive will contain a top-level + // directory named after dirPath. + createTarGzStream: function (dirPath) { + return fstream.Reader({ path: dirPath, type: 'Directory' }).pipe( + tar.Pack()).pipe(zlib.createGzip()); + }, + // A thin wrapper around request(...) that makes the response "body" // the main callback argument. This facilitates using `Future.wrap`. getUrl: function (urlOrOptions, callback) { diff --git a/lib/meteor.js b/lib/meteor.js index b3d24e015b..cf8b721fe7 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -422,20 +422,17 @@ Fiber(function () { process.exit(1); } - var cp = require('child_process'); - cp.execFile('/usr/bin/env', - ['tar', 'czf', output_path, 'bundle'], - {cwd: build_dir}, - function (error, stdout, stderr) { - if (error !== null) { - console.log(JSON.stringify(error)); - process.stderr.write("couldn't run tar\n"); - } else { - process.stdout.write(stdout); - process.stderr.write(stderr); - } - files.rm_recursive(build_dir); - }); + var out = fs.createWriteStream(output_path); + + out.on('error', function (err) { + console.log(JSON.stringify(err)); + process.stderr.write("Couldn't create tarball\n"); + }); + out.on('close', function () { + files.rm_recursive(build_dir); + }); + + files.createTarGzStream(path.join(build_dir, 'bundle')).pipe(out); } }); diff --git a/lib/update.js b/lib/update.js index 7f6a17ec07..f9e300384b 100644 --- a/lib/update.js +++ b/lib/update.js @@ -172,36 +172,31 @@ updater.get_manifest(function (manifest) { "-" + arch + "-" + manifest.version + ".tar.gz"; download_callback = function (tar_path) { - var base_dir = path.join(__dirname, "..", ".."); + var base_dir = path.join(__dirname, ".."); var tmp_dir = path.join(base_dir, "tmp"); // XXX error check! try { fs.mkdirSync(tmp_dir, 0755); } catch (err) { } - // open pipe to tar - var tar_proc = spawn("tar", ["-C", tmp_dir, "-xzf", tar_path]); + try { + console.log('installing upgrade...'); + files.extractTarGz(fs.readFileSync(tar_path), + tmp_dir); + } catch (e) { + console.log(e); + console.log("Error: package download failed."); + return; + } - tar_proc.stderr.setEncoding('utf8'); - tar_proc.stderr.on('data', function (data) { - console.log(data); - }); + // untar complete. swap directories + var old_base_dir = base_dir + ".old"; + if (fs.existsSync(old_base_dir)) + files.rm_recursive(old_base_dir); // rm -rf !! - tar_proc.on('exit', function (code, signal) { - if (code !== 0 || signal) { - console.log("Error: package download failed."); - return; - } + fs.renameSync(base_dir, old_base_dir); + fs.renameSync(path.join(old_base_dir, "tmp", "meteor"), base_dir); - // untar complete. swap directories - var old_base_dir = base_dir + ".old"; - if (fs.existsSync(old_base_dir)) - files.rm_recursive(old_base_dir); // rm -rf !! - - fs.renameSync(base_dir, old_base_dir); - fs.renameSync(path.join(old_base_dir, "tmp", "meteor"), base_dir); - - // success! - run_post_upgrade(); - }); + // success! + run_post_upgrade(); }; } @@ -242,9 +237,13 @@ updater.get_manifest(function (manifest) { res.on('end', function () { download_stream.end(); console.log("... finished download"); - download_callback(download_path); - // don't remove temp dir here, download_callback is probably still - // using it. + // create a Fiber so that code in download_callback can use + // sync calls having Futures. + Fiber(function () { + download_callback(download_path); + // don't remove temp dir here, download_callback is probably still + // using it. + }).run(); }); }); req.end(); From a107739c84e2462496fcabfcc085a79c444115f4 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 20 Dec 2012 21:35:22 -0800 Subject: [PATCH 025/450] Rip out first layer of old way to run package tests --- lib/meteor.js | 53 +++++++++++++++++++++------------------------------ 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/lib/meteor.js b/lib/meteor.js index cf8b721fe7..597ff3c80f 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -39,34 +39,30 @@ Fiber(function () { process.exit(1); }; - var require_project = function (cmd, accept_package) { + // Looks up the directory tree from the current working directory + // to find an app directory. If not found, print an error message. + // + // @param cmd {String} The command that was run. Used when printing + // error message. + var require_project = function (cmd) { var app_dir = files.find_upwards(files.is_app_dir); - if (app_dir) + if (app_dir) { return app_dir; - - var package_dir = files.find_upwards(function (p) { - return files.is_package_dir(p) || files.is_package_collection_dir(p); - }); - if (package_dir) { - if (accept_package) - return package_dir; - - process.stdout.write(cmd + ": Only works on applications, not packages\n"); + } else { + // This is where you end up if you type 'meteor' with no + // args. Be gentle to the noobs.. + process.stdout.write( + cmd + ": You're not in a Meteor project directory.\n" + + "\n" + + "To create a new Meteor project:\n" + + " meteor create \n" + + "For example:\n" + + " meteor create myapp\n" + + "\n" + + "For more help, see 'meteor --help'.\n"); process.exit(1); + return false; // no need for this since we exit(), but makes jslint happy } - - // This is where you end up if you type 'meteor' with no - // args. Be gentle to the noobs.. - process.stdout.write( - cmd + ": You're not in a Meteor project directory.\n" + - "\n" + - "To create a new Meteor project:\n" + - " meteor create \n" + - "For example:\n" + - " meteor create myapp\n" + - "\n" + - "For more help, see 'meteor --help'.\n"); - process.exit(1); }; var find_mongo_port = function (cmd, callback) { @@ -128,8 +124,7 @@ Fiber(function () { process.exit(1); } - var app_dir = path.resolve(require_project("run", true)); // app or package - + var app_dir = path.resolve(require_project("run")); var bundle_opts = { no_minify: !new_argv.production, symlink_dev_bundle: true }; runner.run(app_dir, bundle_opts, new_argv.port, new_argv.once, new_argv.settings); } @@ -517,12 +512,8 @@ Fiber(function () { .describe('delete', "permanently delete this deployment") .boolean('debug') .describe('debug', 'deploy in debug mode (don\'t minify, etc)') - .boolean('tests') .describe('settings', 'set optional data for Meteor.settings') - // .describe('tests', 'deploy the tests instead of the actual application') .usage( - // XXX document --tests in the future, once we publicly - // support tests "Usage: meteor deploy [--password] [--settings settings.json] [--debug] [--delete]\n" + "\n" + "Deploys the project in your current directory to Meteor's servers.\n" + @@ -563,7 +554,7 @@ Fiber(function () { if (new_argv.settings) settings = runner.getSettings(new_argv.settings); // accept packages iff we're deploying tests - var project_dir = path.resolve(require_project("bundle", new_argv.tests)); + var project_dir = path.resolve(require_project("bundle")); deploy.deploy_app(new_argv._[1], project_dir, new_argv.debug, new_argv.tests, new_argv.password, settings); } From d01c00657274ca6ae2b35188477d8adc91a22514 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 20 Dec 2012 21:55:59 -0800 Subject: [PATCH 026/450] Strip some unused code that was preparation for app tests --- lib/run.js | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/run.js b/lib/run.js index 1143ccf3f4..2e3c8ffd3c 100644 --- a/lib/run.js +++ b/lib/run.js @@ -540,27 +540,14 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { var outer_port = port || 3000; var inner_port = outer_port + 1; var mongo_port = outer_port + 2; - var test_port = outer_port + 3; var bundle_path = path.join(app_dir, '.meteor', 'local', 'build'); - var test_bundle_path = path.join(app_dir, '.meteor', 'local', 'build_test'); // Allow override and use of external mongo. Matches code in launch_mongo. var mongo_url = process.env.MONGO_URL || ("mongodb://127.0.0.1:" + mongo_port + "/meteor"); - var test_mongo_url = "mongodb://127.0.0.1:" + mongo_port + "/meteor_test"; - var test_bundle_opts; - - if (files.is_app_dir(app_dir)) { - // If we're an app, make separate test_bundle_opts to trigger a - // separate runner. - - // XXX test_bundle_opts = _.extend({include_tests: true}, bundle_opts); - // Disable app dir testing for now! It is not fully developed and we - // don't want to burden users yet. - } else { - // Otherwise we're running in a package directory, run the tests as - // the main app (so we get reload watching and such). - bundle_opts = _.extend({include_tests: true}, bundle_opts); + if (!files.is_app_dir(app_dir)) { + process.stdout.write("\nPlease fix the problem and restart.\n"); + process.exit(1); } var deps_info = null; @@ -670,7 +657,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { settingsFile: settingsFile }); - // launch test bundle and server if needed. if (test_bundle_opts) { var errors = @@ -696,7 +682,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { }; }); - var mongo_err_count = 0; var mongo_err_timer; var mongo_startup_print_timer; From 1de7926e1d182bd71fe0a2d6c238218b209aa24a Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 20 Dec 2012 22:09:12 -0800 Subject: [PATCH 027/450] Strip some unused code --- lib/run.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/run.js b/lib/run.js index 2e3c8ffd3c..e9b6c78577 100644 --- a/lib/run.js +++ b/lib/run.js @@ -554,7 +554,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { var warned_about_no_deps_info = false; var server_handle; - var test_server_handle; var watcher; if (once) { @@ -588,8 +587,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { Status.listening = false; if (server_handle) kill_server(server_handle); - if (test_server_handle) - kill_server(test_server_handle); server_log = []; From 6e25474b31d483064fe6758ebef0debae55df62e Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 20 Dec 2012 22:19:20 -0800 Subject: [PATCH 028/450] Fix error messages when running meteor run --- lib/meteor.js | 2 +- tools/cli-test.sh | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/meteor.js b/lib/meteor.js index 597ff3c80f..7c36de02e5 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -554,7 +554,7 @@ Fiber(function () { if (new_argv.settings) settings = runner.getSettings(new_argv.settings); // accept packages iff we're deploying tests - var project_dir = path.resolve(require_project("bundle")); + var project_dir = path.resolve(require_project("deploy")); deploy.deploy_app(new_argv._[1], project_dir, new_argv.debug, new_argv.tests, new_argv.password, settings); } diff --git a/tools/cli-test.sh b/tools/cli-test.sh index c76f446b09..28e1472c79 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -42,15 +42,15 @@ $METEOR reset --help | grep "Reset the current" > /dev/null echo "... not in dir" -$METEOR | grep "You're not in" > /dev/null -$METEOR run | grep "You're not in" > /dev/null -$METEOR add foo | grep "You're not in" > /dev/null -$METEOR remove foo | grep "You're not in" > /dev/null -$METEOR list --using | grep "You're not in" > /dev/null -$METEOR bundle foo.tar.gz | grep "You're not in" > /dev/null -$METEOR mongo | grep "You're not in" > /dev/null -$METEOR deploy automated-test | grep "You're not in" > /dev/null -$METEOR reset | grep "You're not in" > /dev/null +$METEOR | grep "run: You're not in" > /dev/null +$METEOR run | grep "run: You're not in" > /dev/null +$METEOR add foo | grep "add: You're not in" > /dev/null +$METEOR remove foo | grep "remove: You're not in" > /dev/null +$METEOR list --using | grep "list --using: You're not in" > /dev/null +$METEOR bundle foo.tar.gz | grep "bundle: You're not in" > /dev/null +$METEOR mongo | grep "mongo: You're not in" > /dev/null +$METEOR deploy automated-test | grep "deploy: You're not in" > /dev/null +$METEOR reset | grep "reset: You're not in" > /dev/null echo "... create" From 528f156ea88f51346bf9047b287b49171e0fd3e3 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 20 Dec 2012 22:35:39 -0800 Subject: [PATCH 029/450] Strip support for package testing from meteor deploy I believe that at this point all legacy package testing code is localized to the bundler --- lib/deploy.js | 7 ++----- lib/meteor.js | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/deploy.js b/lib/deploy.js index 9ced23df1d..c13fc756b1 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -56,7 +56,7 @@ var meteor_rpc = function (rpc_name, method, site, query_params, callback) { return r; }; -var deploy_app = function (url, app_dir, opt_debug, opt_tests, +var deploy_app = function (url, app_dir, opt_debug, opt_set_password, settings) { var parsed_url = parse_url(url); @@ -68,7 +68,6 @@ var deploy_app = function (url, app_dir, opt_debug, opt_tests, site: parsed_url.hostname, appDir: app_dir, debug: opt_debug, - tests: opt_tests, password: password, settings: settings }; @@ -86,14 +85,12 @@ var bundle_and_deploy = function (options) { var site = options.site; var app_dir = options.appDir; var opt_debug = options.debug; - var opt_tests = options.tests; var password = options.password; var set_password = options.setPassword; var settings = options.settings; var build_dir = path.join(app_dir, '.meteor', 'local', 'build_tar'); var bundle_path = path.join(build_dir, 'bundle'); - var bundle_opts = { skip_dev_bundle: true, no_minify: !!opt_debug, - include_tests: opt_tests }; + var bundle_opts = { skip_dev_bundle: true, no_minify: !!opt_debug }; process.stdout.write('Deploying to ' + site + '. Bundling ... '); var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); diff --git a/lib/meteor.js b/lib/meteor.js index 7c36de02e5..dee2dfeaa6 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -556,7 +556,7 @@ Fiber(function () { // accept packages iff we're deploying tests var project_dir = path.resolve(require_project("deploy")); deploy.deploy_app(new_argv._[1], project_dir, new_argv.debug, - new_argv.tests, new_argv.password, settings); + new_argv.password, settings); } } }); From 3e0534a9dfb8daa582f9dd29866fa96ca23d4502 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 21 Dec 2012 21:39:11 -0800 Subject: [PATCH 030/450] Wrote a first bundler unit test, and related refactoring. - No more global manifest. Instead, all functions requiring a manifest have it passed in, either by argument or as a attribute on an appropriate object (such as BundleInstance) - Some initial progress towards renaming project to app in the codebase - Eliminate references to collections of packages as a type of "Package" - Made meteor commands run correctly from a subdirectory of a meteor app, and improved tests to have caught this. - node.sh now returns the return value from the node executable --- lib/bundler.js | 33 ++++-- lib/deploy.js | 11 ++ lib/meteor.js | 10 +- lib/packages.js | 110 +++++------------- lib/project.js | 6 +- lib/run.js | 34 ++---- .../empty-versioned-app/.meteor/.gitignore | 1 + .../empty-versioned-app/.meteor/packages | 1 + lib/tests/empty-versioned-app/.meteor/version | 1 + lib/tests/run-unit-tests.sh | 12 ++ lib/tests/test_bundler.js | 44 +++++++ tools/cli-test.sh | 8 +- 12 files changed, 146 insertions(+), 125 deletions(-) create mode 100644 lib/tests/empty-versioned-app/.meteor/.gitignore create mode 100644 lib/tests/empty-versioned-app/.meteor/packages create mode 100644 lib/tests/empty-versioned-app/.meteor/version create mode 100755 lib/tests/run-unit-tests.sh create mode 100644 lib/tests/test_bundler.js diff --git a/lib/bundler.js b/lib/bundler.js index a30533d27b..8249207a5f 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -92,7 +92,10 @@ var PackageInstance = function (pkg, bundle) { names = names ? [names] : []; _.each(names, function (name) { - var pkg = packages.get(name); + var pkg = packages.get(self.bundle.manifest, name); + if (!pkg) + throw new Error("Package not found: " + name + + ". Manifest is " + self.bundle.manifest); self.bundle.use(pkg, where, self); }); }, @@ -131,7 +134,7 @@ var PackageInstance = function (pkg, bundle) { names = [names]; _.each(names, function (name) { - var pkg = packages.get(name); + var pkg = packages.get(bundle.manifest, name); self.bundle.include_tests(pkg); }); }, @@ -691,10 +694,9 @@ _.extend(Bundle.prototype, { /////////////////////////////////////////////////////////////////////////////// /** - * Take the Meteor project (app or package) in project_dir, and compile it - * into a bundle at output_path. output_path will be created if it - * doesn't exist (it will be a directory), and removed if it does - * exist. + * Take the Meteor app in project_dir, and compile it into a bundle at + * output_path. output_path will be created if it doesn't exist (it + * will be a directory), and removed if it does exist. * * Returns undefined on success. On failure, returns an array of * strings, the error messages. On failure, a bundle will still be @@ -713,21 +715,28 @@ _.extend(Bundle.prototype, { * used by meteor run). * - include_tests : include tests for the project */ -exports.bundle = function (project_dir, output_path, options) { +exports.bundle = function (app_dir, output_path, options) { options = options || {}; try { // Create a bundle, add the project packages.flush(); + var bundle = new Bundle; - var pkg = packages.get_for_dir(project_dir, ignore_files); // `package` is a reserved keyword - bundle.use(pkg); + var manifest = packages.manifestForProject(app_dir); + if (!manifest) + console.log("Couldn't find .meteor/version -- only searching local packages."); + bundle.manifest = manifest; + + // our manifest is set, let's now load the app + var app = packages.get_for_app(app_dir, ignore_files); + bundle.use(app); // Include tests if requested if (options.include_tests) { // in the future, let use specify the driver, instead of hardcoding? - bundle.use(packages.get('test-in-browser')); - bundle.include_tests(project); + bundle.use(packages.get(manifest, 'test-in-browser')); + bundle.include_tests(app); } // Minify, if requested @@ -738,7 +747,7 @@ exports.bundle = function (project_dir, output_path, options) { var dev_bundle_mode = options.skip_dev_bundle ? "skip" : ( options.symlink_dev_bundle ? "symlink" : "copy"); - bundle.write_to_directory(output_path, project_dir, dev_bundle_mode); + bundle.write_to_directory(output_path, app_dir, dev_bundle_mode); if (bundle.errors.length) return bundle.errors; diff --git a/lib/deploy.js b/lib/deploy.js index c13fc756b1..c168da83c3 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -15,6 +15,14 @@ var keypress = require('keypress'); var child_process = require('child_process'); var inFiber = require(path.join(__dirname, '..', 'lib', 'fiber-helpers.js')).inFiber; +// XXX we duplicate this code in run.js. Maybe this should +// go in some fiber-helpers.js? +var inFiber = function(func) { + return function() { + new Fiber(func).run(); + }; +}; + // // configuration // @@ -349,6 +357,9 @@ var read_password = function (callback) { var with_password = function (site, callback) { var check_url = "https://" + DEPLOY_HOSTNAME + "/has_password/" + site; + // XXX we've been using `inFiber` as needed, but I'd really love to + // just have all calls from the event loop get wrapped in a + // fiber. -Avital callback = inFiber(callback); request(check_url, function (error, response, body) { diff --git a/lib/meteor.js b/lib/meteor.js index dee2dfeaa6..6a5b79d885 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -1,3 +1,4 @@ +Error.stackTraceLimit = Infinity; // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi var Fiber = require('fibers'); Fiber(function () { @@ -277,7 +278,8 @@ Fiber(function () { var app_dir = require_project('add'); var packages = require(path.join(__dirname, 'packages.js')); var project = require(path.join(__dirname, 'project.js')); - var all = packages.list(); + var manifest = packages.manifestForProject(app_dir); + var all = packages.list(manifest); var using = {}; _.each(project.get_packages(app_dir), function (name) { using[name] = true; @@ -362,9 +364,13 @@ Fiber(function () { " meteor list\n"); } return; + } else { + var app_dir = require_project('list'); } - var list = require(path.join(__dirname, 'packages.js')).list(); + var packages = require(path.join(__dirname, 'packages.js')); + var manifest = packages.manifestForProject(app_dir); + var list = packages.list(manifest); var names = _.keys(list); names.sort(); var pkgs = []; diff --git a/lib/packages.js b/lib/packages.js index e86869504c..5c676d74fa 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -19,10 +19,6 @@ var PACKAGES_URLBASE = 'https://packages.meteor.com'; // To create a package object from an app directory: // var pkg = new Package; // pkg.init_from_app_dir(app_dir); -// -// Or from a collection (a directory whose subdirs are packages): -// var pkg = new Package; -// pkg.init_from_collection(collection_dir); var next_package_id = 1; var Package = function () { @@ -247,21 +243,6 @@ _.extend(Package.prototype, { // bundle-time. and to pull in, eg, 'fs' and 'path' to access // the file system func(self.declarationFuncs, require); - }, - - init_from_collection: function (collection_dir) { - var self = this; - self.name = null; - self.source_root = null; - self.serve_root = null; - - self.declarationFuncs.on_test(function (api) { - _.each(fs.readdirSync(collection_dir), function (name) { - // only take things that are actually packages - if (files.is_package_dir(path.join(collection_dir, name))) - api.include_tests(name); - }); - }); } }); @@ -270,36 +251,21 @@ var package_cache = {}; var packages = module.exports = { - // @param manifest {Object} parsed manifest file for appropriate meteor release - // - // XXX we should probably instead refactor packages to be a class - // rather than a global object, in which case this would be passed - // into the constructor and ensureManifestSet() would no longer be - // needed. - setManifest: function (manifest) { - if (this.manifest) - throw new Error("Can't set manifest twice"); - this.manifest = manifest; - }, - - ensureManifestSet: function() { - if (!this.manifest) - throw new Error( - "No manifest set. You need to call setManifest before taking this action."); - }, - // get a package by name. also maps package objects to themselves. - get: function (name) { + get: function (manifest, name) { var self = this; if (name instanceof Package) return name; if (!(name in package_cache)) { var pkg = new Package; - if (!pkg.initFromLocalPackageSets(name)) { - self.ensureManifestSet(); - pkg.initFromPackageCache(name, self.manifest.packages[name]); + if (pkg.initFromLocalPackageSets(name)) { + package_cache[name] = pkg; + } else { + if (manifest) { + pkg.initFromPackageCache(name, manifest.packages[name]); + package_cache[name] = pkg; + } } - package_cache[name] = pkg; } return package_cache[name]; @@ -314,28 +280,6 @@ var packages = module.exports = { return pkg; }, - get_for_collection: function (collection_dir) { - var pkg = new Package; - pkg.init_from_collection(collection_dir); - return pkg; - }, - - // get a package that represents a particular directory on disk, - // which might be an app, a package, or even a collection of - // packages. - get_for_dir: function (project_dir) { - if (files.is_app_dir(project_dir)) - return packages.get_for_app(project_dir); - else if (files.is_package_dir(project_dir)) - // this will need to change when packages are stored in more - // than one place - return packages.get(path.basename(project_dir)); - else if (files.is_package_collection_dir(project_dir)) - return packages.get_for_collection(project_dir); - else - throw new Error("Unknown project directory type"); - }, - // force reload of all packages flush: function () { package_cache = {}; @@ -344,7 +288,7 @@ var packages = module.exports = { // get all packages available, in both local package sets and in the package // cache (in case we are in an app directory) // returns {Object} maps name to Package - list: function () { + list: function (manifest) { var self = this; var list = {}; @@ -357,7 +301,7 @@ var packages = module.exports = { var packageNames = fs.readdirSync(path.join(setsDir, set)); _.each(packageNames, function (name) { if (!list[name]) - list[name] = packages.get(name); + list[name] = packages.get(null, name); // empty manifest, we're loading from local packages else throw new Error("Found package " + name + " more than once in local package sets."); }); @@ -365,14 +309,14 @@ var packages = module.exports = { }); } - if (self.manifest) { - _.each(self.manifest.packages, function(version, name) { + if (manifest) { + _.each(manifest.packages, function(version, name) { // don't even look for packages if they've already been // overridden (though this `if` isn't necessary for // correctness, since `packages.get` looks for packages in the // override directories first anyways) if (!list[name]) - list[name] = packages.get(name); + list[name] = packages.get(manifest, name); }); } @@ -423,7 +367,7 @@ var packages = module.exports = { populateCacheForReleaseVersion: function(releaseVersion) { var self = this; var future = new Future; - var manifestDir = path.join(__dirname, '..', 'cache'); + var manifestDir = path.join(__dirname, '..', 'cache', 'manifest'); files.mkdir_p(manifestDir, 0755); var manifestPath = path.join(manifestDir, releaseVersion + '.json'); @@ -440,31 +384,31 @@ var packages = module.exports = { } }, - // Look for ./.meteor/version. If it exists, presumable since we're - // inside a project load the manifest corresponding to that meteor - // release version - loadManifestForProjectIfExists: function () { + // Look for .meteor/version relative to the appDir argument. If it + // exists, presumable since we're inside a project load the manifest + // corresponding to that meteor release version, return the parsed + // manifest after having loaded all the relevant packages into the + // cache. If no .meteor/version is found, return null. + manifestForProject: function (appDir) { var self = this; var project = require(path.join(__dirname, 'project.js')); - if (fs.existsSync('.meteor/version')) { - var releaseVersion = project.getMeteorReleaseVersion(); + var releaseVersion = project.getMeteorReleaseVersion(appDir); + + if (!releaseVersion) { + return null; // no manifest found + } else { var manifestPath = path.join( __dirname, '..', 'cache', 'manifest', releaseVersion + '.json'); var manifest; if (fs.existsSync(manifestPath)) { - // read from cache manifest = JSON.parse(fs.readFileSync(manifestPath)); - packages.setManifest(manifest); } else { - // grow cache with new manifest and packages manifest = packages.populateCacheForReleaseVersion(releaseVersion); - packages.setManifest(manifest); } - var exec = require('child_process').exec; var Future = require('fibers/future'); var futures = []; _.each(manifest.packages, function (version, name) { @@ -491,8 +435,8 @@ var packages = module.exports = { files.mkdir_p(result.packageDir); files.extractTarGz(result.buffer, result.packageDir); }); + + return manifest; } } }; - -packages.loadManifestForProjectIfExists(); diff --git a/lib/project.js b/lib/project.js index 19594ea8b9..403d4c5588 100644 --- a/lib/project.js +++ b/lib/project.js @@ -50,7 +50,11 @@ var project = module.exports = { }, getMeteorReleaseVersion: function (appDir) { - return project._trim_line(project._get_lines(path.join(appDir, '.meteor', 'version'))[0]); + var versionPath = path.join(appDir, '.meteor', 'version'); + if (fs.existsSync(versionPath)) + return project._trim_line(project._get_lines(versionPath)[0]); + else + return undefined; }, add_package: function (app_dir, name) { diff --git a/lib/run.js b/lib/run.js index e9b6c78577..265aae64ff 100644 --- a/lib/run.js +++ b/lib/run.js @@ -21,6 +21,14 @@ var _ = require('underscore'); // list of log objects from the child process. var server_log = []; +// XXX we duplicate this code in deploy.js. Maybe this should +// go in some fiber-helpers.js? +var inFiber = function(func) { + return function() { + new Fiber(func).run(); + }; +}; + var Status = { running: false, // is server running now? crashing: false, // does server crash whenever we start it? @@ -582,6 +590,8 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { } }; + // Using `inFiber` since bundling can yield when loading a manifest + // file from packages.meteor.com. var restart_server = inFiber(function () { Status.running = false; Status.listening = false; @@ -653,30 +663,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { nodeOptions: getNodeOptionsFromEnvironment(), settingsFile: settingsFile }); - - // launch test bundle and server if needed. - if (test_bundle_opts) { - var errors = - bundler.bundle(app_dir, test_bundle_path, test_bundle_opts); - if (errors) { - log_to_clients({system: "Errors prevented test server from starting:"}); - _.each(errors, function (e) { - log_to_clients({system: e}); - }); - files.rm_recursive(test_bundle_path); - } else { - test_server_handle = start_server({ - bundlePath: test_bundle_path, - outerPort: test_port, - innerPort: test_port, - mongoURL: test_mongo_url, - onExit: function (code) { - // No restarting or crash loop prevention on the test server - // for now. We'll see how annoying it is. - log_to_clients({'system': "Test server crashed."}); - }}); - } - }; }); var mongo_err_count = 0; diff --git a/lib/tests/empty-versioned-app/.meteor/.gitignore b/lib/tests/empty-versioned-app/.meteor/.gitignore new file mode 100644 index 0000000000..4083037423 --- /dev/null +++ b/lib/tests/empty-versioned-app/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/lib/tests/empty-versioned-app/.meteor/packages b/lib/tests/empty-versioned-app/.meteor/packages new file mode 100644 index 0000000000..9c8d080e06 --- /dev/null +++ b/lib/tests/empty-versioned-app/.meteor/packages @@ -0,0 +1 @@ +# no packages \ No newline at end of file diff --git a/lib/tests/empty-versioned-app/.meteor/version b/lib/tests/empty-versioned-app/.meteor/version new file mode 100644 index 0000000000..49d59571fb --- /dev/null +++ b/lib/tests/empty-versioned-app/.meteor/version @@ -0,0 +1 @@ +0.1 diff --git a/lib/tests/run-unit-tests.sh b/lib/tests/run-unit-tests.sh new file mode 100755 index 0000000000..d49e963097 --- /dev/null +++ b/lib/tests/run-unit-tests.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# stop on any non-zero return value from test_bundler.js, and print "FAILED" +set -e +trap 'echo FAILED' EXIT + +# run tests +../../tools/node.sh test_bundler.js + +# cleanup trap, and print "SUCCESS" +trap - EXIT +echo PASSED diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js new file mode 100644 index 0000000000..91778ce9a0 --- /dev/null +++ b/lib/tests/test_bundler.js @@ -0,0 +1,44 @@ +var path = require('path'); +var assert = require('assert'); +var fs = require('fs'); +var files = require(path.join(__dirname, '..', 'files.js')); +var bundler = require(path.join(__dirname, '..', 'bundler.js')); + +/// SETUP +// print stack track and exit with error code if an assertion fails +process.on('uncaughtException', function (err) { + console.log(err.stack); + process.exit(1); +}); + +/// UTILITIES +var tmpDir = function () { + return files.mkdtemp('test_bundler'); +}; + +var inFiber = function (func) { + return function () { + Fiber(func).run(); + }; +}; + +/// TEST APPS +// an empty app with a .meteor/version file whose contents are "0.1" +var versionedAppDir = path.join(__dirname, 'empty-versioned-app'); + +/// TESTS +// versioned app, no options +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir); + assert.strictEqual(errors, undefined); + + // XXX leaving this here for now since it'll be helpful for + // writing more tests + console.log('bundle successfully created at ' + tmpOutputDir); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); +})); + diff --git a/tools/cli-test.sh b/tools/cli-test.sh index 28e1472c79..6839e6a2be 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -61,14 +61,16 @@ test -f "$DIR/$DIR.js" ## Tests in a meteor project cd "$DIR" +# run in a subdirectory, just to make sure this also works +cd .meteor echo "... add/remove/list" $METEOR list | grep "backbone" > /dev/null ! $METEOR list --using 2>&1 | grep "backbone" > /dev/null -$METEOR add backbone 2>&1 | grep "backbone:" > /dev/null +$METEOR add backbone 2>&1 | grep "backbone:" | grep -v "no such package" | > /dev/null $METEOR list --using | grep "backbone" > /dev/null -grep backbone .meteor/packages > /dev/null +grep backbone packages > /dev/null # remember, we are already in .meteor $METEOR remove backbone 2>&1 | grep "backbone: removed" > /dev/null ! $METEOR list --using 2>&1 | grep "backbone" > /dev/null @@ -77,7 +79,7 @@ echo "... bundle" $METEOR bundle foo.tar.gz test -f foo.tar.gz - +cd .. # we're now back to $DIR echo "... run" MONGOMARK='--bind_ip 127.0.0.1 --smallfiles --port 9102' From a7ab8da940af4f54e81f011d276f23fade6f9845 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 27 Dec 2012 01:24:26 -0800 Subject: [PATCH 031/450] Shorten error --- lib/bundler.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/bundler.js b/lib/bundler.js index 8249207a5f..57a1420887 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -94,8 +94,7 @@ var PackageInstance = function (pkg, bundle) { _.each(names, function (name) { var pkg = packages.get(self.bundle.manifest, name); if (!pkg) - throw new Error("Package not found: " + name + - ". Manifest is " + self.bundle.manifest); + throw new Error("Package not found: " + name); self.bundle.use(pkg, where, self); }); }, From 62b5e87cce084977882804a7337793f4fa3e410f Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 27 Dec 2012 01:54:04 -0800 Subject: [PATCH 032/450] bundler: add a versionOverride option --- lib/bundler.js | 4 +- lib/packages.js | 7 +++- .../empty-unversioned-app/.meteor/.gitignore | 1 + .../empty-unversioned-app/.meteor/packages | 1 + lib/tests/test_bundler.js | 38 +++++++++++++++++++ 5 files changed, 48 insertions(+), 3 deletions(-) create mode 100644 lib/tests/empty-unversioned-app/.meteor/.gitignore create mode 100644 lib/tests/empty-unversioned-app/.meteor/packages diff --git a/lib/bundler.js b/lib/bundler.js index 57a1420887..894f0dd63d 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -713,6 +713,8 @@ _.extend(Bundle.prototype, { * local installation (to save startup time when running locally, * used by meteor run). * - include_tests : include tests for the project + * - versionOverride : (for tests) a meteor release version to use + * instead of reading from .meteor/version */ exports.bundle = function (app_dir, output_path, options) { options = options || {}; @@ -722,7 +724,7 @@ exports.bundle = function (app_dir, output_path, options) { packages.flush(); var bundle = new Bundle; - var manifest = packages.manifestForProject(app_dir); + var manifest = packages.manifestForProject(app_dir, {versionOverride: options.versionOverride}); if (!manifest) console.log("Couldn't find .meteor/version -- only searching local packages."); bundle.manifest = manifest; diff --git a/lib/packages.js b/lib/packages.js index 5c676d74fa..29a02d09aa 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -389,10 +389,13 @@ var packages = module.exports = { // corresponding to that meteor release version, return the parsed // manifest after having loaded all the relevant packages into the // cache. If no .meteor/version is found, return null. - manifestForProject: function (appDir) { + // + // Options: + // - versionOverride (String). Used instead of reading .meteor/version + manifestForProject: function (appDir, options) { var self = this; var project = require(path.join(__dirname, 'project.js')); - var releaseVersion = project.getMeteorReleaseVersion(appDir); + var releaseVersion = options.versionOverride || project.getMeteorReleaseVersion(appDir); if (!releaseVersion) { return null; // no manifest found diff --git a/lib/tests/empty-unversioned-app/.meteor/.gitignore b/lib/tests/empty-unversioned-app/.meteor/.gitignore new file mode 100644 index 0000000000..4083037423 --- /dev/null +++ b/lib/tests/empty-unversioned-app/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/lib/tests/empty-unversioned-app/.meteor/packages b/lib/tests/empty-unversioned-app/.meteor/packages new file mode 100644 index 0000000000..9c8d080e06 --- /dev/null +++ b/lib/tests/empty-unversioned-app/.meteor/packages @@ -0,0 +1 @@ +# no packages \ No newline at end of file diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 91778ce9a0..5824532d79 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -4,14 +4,20 @@ var fs = require('fs'); var files = require(path.join(__dirname, '..', 'files.js')); var bundler = require(path.join(__dirname, '..', 'bundler.js')); +/// /// SETUP +/// + // print stack track and exit with error code if an assertion fails process.on('uncaughtException', function (err) { console.log(err.stack); process.exit(1); }); +/// /// UTILITIES +/// + var tmpDir = function () { return files.mkdtemp('test_bundler'); }; @@ -22,11 +28,19 @@ var inFiber = function (func) { }; }; +/// /// TEST APPS +/// + // an empty app with a .meteor/version file whose contents are "0.1" var versionedAppDir = path.join(__dirname, 'empty-versioned-app'); +// an empty app with no .meteor/version file +var unversionedAppDir = path.join(__dirname, 'empty-unversioned-app'); +/// /// TESTS +/// + // versioned app, no options assert.doesNotThrow(inFiber(function () { var tmpOutputDir = tmpDir(); @@ -42,3 +56,27 @@ assert.doesNotThrow(inFiber(function () { "require(require('path').join(__dirname, 'server', 'server.js'));"); })); +// unversioned app, no options +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir); + assert.notEqual(errors.length, 0); + assert.notEqual(errors[0].indexOf('Exception while bundling'), -1); + assert.notEqual(errors[0].indexOf('Package not found: meteor'), -1); +})); + +// unversioned app, using `versionOverride` +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.1'}); + assert.strictEqual(errors, undefined); + + // XXX leaving this here for now since it'll be helpful for + // writing more tests + console.log('bundle successfully created at ' + tmpOutputDir); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); +})); + From b25934b71b663f6e195f35a69cc93156a2f5e6a2 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 27 Dec 2012 17:45:39 -0800 Subject: [PATCH 033/450] Strip unnecessary check --- lib/run.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/run.js b/lib/run.js index 265aae64ff..da6f0fcc1e 100644 --- a/lib/run.js +++ b/lib/run.js @@ -553,11 +553,6 @@ exports.run = function (app_dir, bundle_opts, port, once, settingsFile) { var mongo_url = process.env.MONGO_URL || ("mongodb://127.0.0.1:" + mongo_port + "/meteor"); - if (!files.is_app_dir(app_dir)) { - process.stdout.write("\nPlease fix the problem and restart.\n"); - process.exit(1); - } - var deps_info = null; var warned_about_no_deps_info = false; From b812d6e9ec5882c4296e475b746e1233ac9d0aca Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Fri, 28 Dec 2012 15:27:06 -0800 Subject: [PATCH 034/450] Introduce fiber-helpers.js and some minor comment improvements --- lib/bundler.js | 6 +++++- lib/deploy.js | 10 +--------- lib/fiber-helpers.js | 19 +++++++++++++++++++ lib/meteor.js | 1 - lib/packages.js | 14 +++++++++----- lib/run.js | 10 +--------- lib/tests/test_bundler.js | 7 +------ 7 files changed, 36 insertions(+), 31 deletions(-) create mode 100644 lib/fiber-helpers.js diff --git a/lib/bundler.js b/lib/bundler.js index 894f0dd63d..a7ecef7b0b 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -725,8 +725,12 @@ exports.bundle = function (app_dir, output_path, options) { var bundle = new Bundle; var manifest = packages.manifestForProject(app_dir, {versionOverride: options.versionOverride}); - if (!manifest) + if (!manifest) { + // XXX We should instead use the latest version installed, and + // notify the user. + // https://app.asana.com/0/2604247562419/2765125200674 console.log("Couldn't find .meteor/version -- only searching local packages."); + } bundle.manifest = manifest; // our manifest is set, let's now load the app diff --git a/lib/deploy.js b/lib/deploy.js index c168da83c3..63f6b8124e 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -13,15 +13,7 @@ var files = require(path.join(__dirname, '..', 'lib', 'files.js')); var _ = require('underscore'); var keypress = require('keypress'); var child_process = require('child_process'); -var inFiber = require(path.join(__dirname, '..', 'lib', 'fiber-helpers.js')).inFiber; - -// XXX we duplicate this code in run.js. Maybe this should -// go in some fiber-helpers.js? -var inFiber = function(func) { - return function() { - new Fiber(func).run(); - }; -}; +var inFiber = require(path.join(__dirname, 'fiber-helpers.js')).inFiber; // // configuration diff --git a/lib/fiber-helpers.js b/lib/fiber-helpers.js new file mode 100644 index 0000000000..7674a776e5 --- /dev/null +++ b/lib/fiber-helpers.js @@ -0,0 +1,19 @@ +// node (v8) defaults to only recording 10 lines of stack trace. +// this becomes especially bad when using fibers, since you get deeper +// stack traces that would have been split up between different callbacks +// +// this only affects the `meteor` executable, not server code on +// meteor apps. +Error.stackTraceLimit = Infinity; // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi + +// runs a function within a fiber. we wrap the entry point into meteor.js into a fiber +// but if you use callbacks that call synchronous code you need to wrap those as well. +// +// NOTE: It's probably better to not use callbacks. Instead you can +// use Futures to generate synchronous equivalents. +exports.inFiber = function(func) { + return function() { + new Fiber(func).run(); + }; +}; + diff --git a/lib/meteor.js b/lib/meteor.js index 6a5b79d885..be599c2e84 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -1,4 +1,3 @@ -Error.stackTraceLimit = Infinity; // http://code.google.com/p/v8/wiki/JavaScriptStackTraceApi var Fiber = require('fibers'); Fiber(function () { diff --git a/lib/packages.js b/lib/packages.js index 29a02d09aa..680b17922b 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -384,11 +384,15 @@ var packages = module.exports = { } }, - // Look for .meteor/version relative to the appDir argument. If it - // exists, presumable since we're inside a project load the manifest - // corresponding to that meteor release version, return the parsed - // manifest after having loaded all the relevant packages into the - // cache. If no .meteor/version is found, return null. + // Load and return a manifest for an app, based on the + // .meteor/version file + // + // If .meteor/version exists, load the manifest corresponding to + // that meteor release. Load from packages.meteor.com and cache on + // disk. Parse and ensure that all used package versions are cached. + // Return parsed manifest. + // + // If .meteor/version does not exist, return null. // // Options: // - versionOverride (String). Used instead of reading .meteor/version diff --git a/lib/run.js b/lib/run.js index da6f0fcc1e..10b5be3bc2 100644 --- a/lib/run.js +++ b/lib/run.js @@ -11,9 +11,9 @@ var updater = require(path.join(__dirname, '..', 'lib', 'updater.js')); var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); var mongo_runner = require(path.join(__dirname, '..', 'lib', 'mongo_runner.js')); var mongoExitCodes = require(path.join(__dirname, '..', 'lib', 'mongo_exit_codes.js')); -var inFiber = require(path.join(__dirname, '..', 'lib', 'fiber-helpers.js')).inFiber; var _ = require('underscore'); +var inFiber = require(path.join(__dirname, 'fiber-helpers.js')).inFiber; ////////// Globals ////////// //XXX: Refactor to not have globals anymore? @@ -21,14 +21,6 @@ var _ = require('underscore'); // list of log objects from the child process. var server_log = []; -// XXX we duplicate this code in deploy.js. Maybe this should -// go in some fiber-helpers.js? -var inFiber = function(func) { - return function() { - new Fiber(func).run(); - }; -}; - var Status = { running: false, // is server running now? crashing: false, // does server crash whenever we start it? diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 5824532d79..7550e869af 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -3,6 +3,7 @@ var assert = require('assert'); var fs = require('fs'); var files = require(path.join(__dirname, '..', 'files.js')); var bundler = require(path.join(__dirname, '..', 'bundler.js')); +var inFiber = require(path.join('..', 'fiber-helpers.js')).inFiber; /// /// SETUP @@ -22,12 +23,6 @@ var tmpDir = function () { return files.mkdtemp('test_bundler'); }; -var inFiber = function (func) { - return function () { - Fiber(func).run(); - }; -}; - /// /// TEST APPS /// From 72a1bbb101c80450e6712e78624e75b965021732 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 2 Jan 2013 15:05:24 -0800 Subject: [PATCH 035/450] small fix --- lib/packages.js | 5 ++++- lib/tests/test_bundler.js | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/packages.js b/lib/packages.js index 680b17922b..54fb6f0da1 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -397,8 +397,11 @@ var packages = module.exports = { // Options: // - versionOverride (String). Used instead of reading .meteor/version manifestForProject: function (appDir, options) { - var self = this; var project = require(path.join(__dirname, 'project.js')); + + var self = this; + options = options || {}; + var releaseVersion = options.versionOverride || project.getMeteorReleaseVersion(appDir); if (!releaseVersion) { diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 7550e869af..b73cfd3041 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -3,7 +3,7 @@ var assert = require('assert'); var fs = require('fs'); var files = require(path.join(__dirname, '..', 'files.js')); var bundler = require(path.join(__dirname, '..', 'bundler.js')); -var inFiber = require(path.join('..', 'fiber-helpers.js')).inFiber; +var inFiber = require(path.join(__dirname, '..', 'fiber-helpers.js')).inFiber; /// /// SETUP From b2bd0eb9b08f24b4d805c5a4f622da31f11124d5 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 2 Jan 2013 16:09:05 -0800 Subject: [PATCH 036/450] Bundler options change: Introduce nodeModulesMode This replaces the old `symlink_dev_bundle` and `skip_dev_bundle` options. Also adds tests for these --- lib/bundler.js | 24 ++++++++++++-------- lib/deploy.js | 2 +- lib/meteor.js | 4 ++-- lib/tests/test_bundler.js | 48 +++++++++++++++++++++++++++++---------- 4 files changed, 54 insertions(+), 24 deletions(-) diff --git a/lib/bundler.js b/lib/bundler.js index a7ecef7b0b..474491ed26 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -708,16 +708,25 @@ _.extend(Bundle.prototype, { * * options include: * - no_minify : don't minify the assets - * - skip_dev_bundle : don't put any node_modules in the bundle. - * - symlink_dev_bundle : symlink bundle's node_modules to prebuilt - * local installation (to save startup time when running locally, - * used by meteor run). + * + * - nodeModulesMode : decide on how to create the bundle's + * node_modules directory. one of: + * 'skip' : don't create node_modules. used by `meteor deploy`, since + * our production servers already have all of the node modules + * 'copy' : copy from a prebuilt local installation. used by + * `meteor bundle` + * 'symlink' : symlink from a prebuild local installation. used + * by `meteor run` + * * - include_tests : include tests for the project + * * - versionOverride : (for tests) a meteor release version to use * instead of reading from .meteor/version */ exports.bundle = function (app_dir, output_path, options) { - options = options || {}; + if (!options || !options.nodeModulesMode) { + throw new Error("Must pass options.nodeModulesMode"); + } try { // Create a bundle, add the project @@ -749,10 +758,7 @@ exports.bundle = function (app_dir, output_path, options) { bundle.minify(); // Write to disk - var dev_bundle_mode = - options.skip_dev_bundle ? "skip" : ( - options.symlink_dev_bundle ? "symlink" : "copy"); - bundle.write_to_directory(output_path, app_dir, dev_bundle_mode); + bundle.write_to_directory(output_path, app_dir, options.nodeModulesMode); if (bundle.errors.length) return bundle.errors; diff --git a/lib/deploy.js b/lib/deploy.js index 63f6b8124e..2b78f03085 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -90,7 +90,7 @@ var bundle_and_deploy = function (options) { var settings = options.settings; var build_dir = path.join(app_dir, '.meteor', 'local', 'build_tar'); var bundle_path = path.join(build_dir, 'bundle'); - var bundle_opts = { skip_dev_bundle: true, no_minify: !!opt_debug }; + var bundle_opts = { nodeModulesMode: 'skip', no_minify: !!opt_debug }; process.stdout.write('Deploying to ' + site + '. Bundling ... '); var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); diff --git a/lib/meteor.js b/lib/meteor.js index be599c2e84..339c64410a 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -125,7 +125,7 @@ Fiber(function () { } var app_dir = path.resolve(require_project("run")); - var bundle_opts = { no_minify: !new_argv.production, symlink_dev_bundle: true }; + var bundle_opts = { no_minify: !new_argv.production, nodeModulesMode: 'symlink' }; runner.run(app_dir, bundle_opts, new_argv.port, new_argv.once, new_argv.settings); } }); @@ -412,7 +412,7 @@ Fiber(function () { var output_path = path.resolve(argv._[0]); // get absolute path var bundler = require(path.join(__dirname, 'bundler.js')); - var errors = bundler.bundle(app_dir, bundle_path); + var errors = bundler.bundle(app_dir, bundle_path, {nodeModulesMode: 'copy'}); if (errors) { process.stdout.write("Errors prevented bundling:\n"); _.each(errors, function (e) { diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index b73cfd3041..38bb104dea 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -36,25 +36,53 @@ var unversionedAppDir = path.join(__dirname, 'empty-unversioned-app'); /// TESTS /// -// versioned app, no options +// versioned app, nodeModules: 'skip' assert.doesNotThrow(inFiber(function () { var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(versionedAppDir, tmpOutputDir); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); assert.strictEqual(errors, undefined); - // XXX leaving this here for now since it'll be helpful for - // writing more tests - console.log('bundle successfully created at ' + tmpOutputDir); - // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), "require(require('path').join(__dirname, 'server', 'server.js'));"); + // no node_modules directory + assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules"))); +})); + +// versioned app, nodeModules: 'copy' +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'copy'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // node_modules directory exists and is not a symlink + assert(!fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + // node_modules contains fibers + assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); +})); + +// versioned app, nodeModules: 'symlink' +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'symlink'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // node_modules directory exists and is not a symlink + assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + // node_modules contains fibers + assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); })); // unversioned app, no options assert.doesNotThrow(inFiber(function () { var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(unversionedAppDir, tmpOutputDir); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); assert.notEqual(errors.length, 0); assert.notEqual(errors[0].indexOf('Exception while bundling'), -1); assert.notEqual(errors[0].indexOf('Package not found: meteor'), -1); @@ -63,13 +91,9 @@ assert.doesNotThrow(inFiber(function () { // unversioned app, using `versionOverride` assert.doesNotThrow(inFiber(function () { var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.1'}); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.1', nodeModulesMode: 'skip'}); assert.strictEqual(errors, undefined); - // XXX leaving this here for now since it'll be helpful for - // writing more tests - console.log('bundle successfully created at ' + tmpOutputDir); - // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), "require(require('path').join(__dirname, 'server', 'server.js'));"); From 87024ac5fec5e35cf1a91e049ea434144f5561e0 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 2 Jan 2013 16:27:31 -0800 Subject: [PATCH 037/450] Bundler options change: no_minify -> noMinify Also, wrote tests --- lib/bundler.js | 4 ++-- lib/deploy.js | 2 +- lib/meteor.js | 2 +- lib/tests/test_bundler.js | 20 ++++++++++++++++++++ 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/lib/bundler.js b/lib/bundler.js index 474491ed26..28403f2a27 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -707,7 +707,7 @@ _.extend(Bundle.prototype, { * without including its tests, but it's well-defined. * * options include: - * - no_minify : don't minify the assets + * - noMinify : don't minify the assets * * - nodeModulesMode : decide on how to create the bundle's * node_modules directory. one of: @@ -754,7 +754,7 @@ exports.bundle = function (app_dir, output_path, options) { } // Minify, if requested - if (!options.no_minify) + if (!options.noMinify) bundle.minify(); // Write to disk diff --git a/lib/deploy.js b/lib/deploy.js index 2b78f03085..1f57855e36 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -90,7 +90,7 @@ var bundle_and_deploy = function (options) { var settings = options.settings; var build_dir = path.join(app_dir, '.meteor', 'local', 'build_tar'); var bundle_path = path.join(build_dir, 'bundle'); - var bundle_opts = { nodeModulesMode: 'skip', no_minify: !!opt_debug }; + var bundle_opts = { nodeModulesMode: 'skip', noMinify: !!opt_debug }; process.stdout.write('Deploying to ' + site + '. Bundling ... '); var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); diff --git a/lib/meteor.js b/lib/meteor.js index 339c64410a..713b16c54c 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -125,7 +125,7 @@ Fiber(function () { } var app_dir = path.resolve(require_project("run")); - var bundle_opts = { no_minify: !new_argv.production, nodeModulesMode: 'symlink' }; + var bundle_opts = { noMinify: !new_argv.production, nodeModulesMode: 'symlink' }; runner.run(app_dir, bundle_opts, new_argv.port, new_argv.once, new_argv.settings); } }); diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 38bb104dea..4bbdbb4611 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -47,6 +47,26 @@ assert.doesNotThrow(inFiber(function () { "require(require('path').join(__dirname, 'server', 'server.js'));"); // no node_modules directory assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules"))); + // verify that contents are minified + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml)); + assert(!(/src=\"\/packages/.test(appHtml))); +})); + +// versioned app, nodeModules: 'skip', noMinify +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // verify that contents are not minified + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml))); + assert(/src=\"\/packages\/meteor/.test(appHtml)); + assert(/src=\"\/packages\/deps/.test(appHtml)); })); // versioned app, nodeModules: 'copy' From 99f80949ac0ae2a4673f9432cc015f0deeb2ae99 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 2 Jan 2013 16:54:45 -0800 Subject: [PATCH 038/450] Bundler option change: testPackages instead of include_tests Also, eliminated some dead code. This should be the last bundler change necessary for implementing `meteor test-packages` --- lib/bundler.js | 30 +++++++++++------------------- lib/tests/test_bundler.js | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/lib/bundler.js b/lib/bundler.js index 28403f2a27..9095df837c 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -125,19 +125,6 @@ var PackageInstance = function (pkg, bundle) { return _.map(ret, function (x) {return "." + x;}); }, - // Add the tests for another package. Mostly for internal - // use. Like use in that it can take either the package name or a - // package object, and can take an array. - include_tests: function (names) { - if (!(names instanceof Array)) - names = [names]; - - _.each(names, function (name) { - var pkg = packages.get(bundle.manifest, name); - self.bundle.include_tests(pkg); - }); - }, - // Report an error. It should be a single human-readable // string. If any errors are reported, the bundling is considered // to have failed. @@ -228,6 +215,9 @@ var Bundle = function () { // Packages that have had tests included. Map from package id to instance self.tests_included = {}; + // release manifest + self.manifest = null; + // map from environment, to list of filenames self.js = {client: [], server: []}; @@ -394,8 +384,9 @@ _.extend(Bundle.prototype, { pkg.on_use_handler(inst.api, where); }, - include_tests: function (pkg) { + includeTests: function (packageName) { var self = this; + var pkg = packages.get(self.manifest, packageName); if (self.tests_included[pkg.id]) return; self.tests_included[pkg.id] = true; @@ -718,7 +709,8 @@ _.extend(Bundle.prototype, { * 'symlink' : symlink from a prebuild local installation. used * by `meteor run` * - * - include_tests : include tests for the project + * - testPackages : array of package names whose tests should be included + * in this bundle * * - versionOverride : (for tests) a meteor release version to use * instead of reading from .meteor/version @@ -747,10 +739,10 @@ exports.bundle = function (app_dir, output_path, options) { bundle.use(app); // Include tests if requested - if (options.include_tests) { - // in the future, let use specify the driver, instead of hardcoding? - bundle.use(packages.get(manifest, 'test-in-browser')); - bundle.include_tests(app); + if (options.testPackages) { + _.each(options.testPackages, function(packageName) { + bundle.includeTests(packageName); + }); } // Minify, if requested diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 4bbdbb4611..5245b4b868 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -67,6 +67,23 @@ assert.doesNotThrow(inFiber(function () { assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml))); assert(/src=\"\/packages\/meteor/.test(appHtml)); assert(/src=\"\/packages\/deps/.test(appHtml)); + // verify that tests aren't included + assert(!(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml))); +})); + +// versioned app, nodeModules: 'skip', noMinify, testPackages: ['meteor'] +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle( + versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true, testPackages: ['meteor']}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // verify that tests for the meteor package are included + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml)); })); // versioned app, nodeModules: 'copy' From 36ab0f7ba183f77749de1e7da503fe59f44d4dc9 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 2 Jan 2013 18:16:07 -0800 Subject: [PATCH 039/450] First pass at `meteor test-package`. Not yet implemented: - Detect latest release if not explicitly passed (currently hard-coded to 0.1) - --deploy - Better help text --- lib/bundler.js | 7 +- lib/meteor.js | 50 ++++++++++++- lib/packages.js | 97 +++++++++++++------------- lib/test-runner-app/.meteor/.gitignore | 1 + lib/test-runner-app/.meteor/packages | 1 + tools/cli-test.sh | 1 + 6 files changed, 107 insertions(+), 50 deletions(-) create mode 100644 lib/test-runner-app/.meteor/.gitignore create mode 100644 lib/test-runner-app/.meteor/packages diff --git a/lib/bundler.js b/lib/bundler.js index 9095df837c..632edfaddc 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -725,7 +725,12 @@ exports.bundle = function (app_dir, output_path, options) { packages.flush(); var bundle = new Bundle; - var manifest = packages.manifestForProject(app_dir, {versionOverride: options.versionOverride}); + var manifest; + if (options.versionOverride) + manifest = packages.manifestForReleaseVersion(options.versionOverride); + else + manifest = packages.manifestForProject(app_dir); + if (!manifest) { // XXX We should instead use the latest version installed, and // notify the user. diff --git a/lib/meteor.js b/lib/meteor.js index 713b16c54c..a4289b53ea 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -117,7 +117,6 @@ Fiber(function () { "the .meteor directory in the root of the project.\n"); var new_argv = opt.argv; - var settings = ""; if (argv.help) { process.stdout.write(opt.help()); @@ -130,6 +129,54 @@ Fiber(function () { } }); + Commands.push({ + name: "test-package", + help: "Test one or more packages", + func: function (argv) { + // reparse args + // This help logic should probably move to run.js eventually + var opt = require('optimist') + .alias('port', 'p').default('port', 3000) + .describe('port', 'Port to listen on. NOTE: Also uses port N+1 and N+2.') + .describe('version', 'Meteor release version to test.') + .describe('deploy', 'Optionally, specify a domain to deploy to instead of running locally.') + .usage( + "Usage: meteor test-package [options] [comma delimited packages]\n" + + "XXX\n"); + + var new_argv = opt.argv; + + if (argv.help) { + process.stdout.write(opt.help()); + process.exit(1); + } + + // XXX should find latest release if not specified + var releaseVersion = new_argv['release-version'] || '0.1'; + + var testPackages; + if (new_argv._[1]) { + testPackages = new_argv._[1].split(','); + } else { + var packages = require(path.join(__dirname, 'packages.js')); + var manifest = packages.manifestForReleaseVersion(releaseVersion); + testPackages = _.keys(packages.list(manifest)); + } + + // XXX implement --deploy. Need to refactor some of deploy.js to make this possible + // since the bundler options are created within bundle_and_deploy + + var bundle_opts = { + nodeModulesMode: 'symlink', + testPackages: testPackages, + versionOverride: releaseVersion, + noMinify: true // curiously, if this isn't set we get "RangeError: Maximum call stack size exceeded" during minification + }; + app_dir = path.join(__dirname, 'test-runner-app'); + runner.run(app_dir, bundle_opts, new_argv.port); + } + }); + Commands.push({ name: "help", func: function (argv) { @@ -558,7 +605,6 @@ Fiber(function () { var settings = undefined; if (new_argv.settings) settings = runner.getSettings(new_argv.settings); - // accept packages iff we're deploying tests var project_dir = path.resolve(require_project("deploy")); deploy.deploy_app(new_argv._[1], project_dir, new_argv.debug, new_argv.password, settings); diff --git a/lib/packages.js b/lib/packages.js index 54fb6f0da1..d4f3f9077a 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -384,6 +384,53 @@ var packages = module.exports = { } }, + // Load the manifest corresponding to a given meteor release from + // packages.meteor.com and cache on disk. Parse and ensure that all + // used package versions are cached. Return parsed manifest. + manifestForReleaseVersion: function(releaseVersion) { + var self = this; + var manifestPath = path.join( + __dirname, '..', 'cache', 'manifest', releaseVersion + '.json'); + + var manifest; + if (fs.existsSync(manifestPath)) { + // read from cache + manifest = JSON.parse(fs.readFileSync(manifestPath)); + } else { + // grow cache with new manifest and packages + manifest = packages.populateCacheForReleaseVersion(releaseVersion); + } + + var Future = require('fibers/future'); + var futures = []; + _.each(manifest.packages, function (version, name) { + if (!self.existsInPackageCache(name, version)) { + var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version); + var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + + version + ".tar.gz"; + + console.log("Fetching " + packageUrl + "..."); + futures.push(Future.wrap(function (cb) { + files.getUrl({url: packageUrl, encoding: null}, function (error, result) { + if (! error && result) + result = { buffer: result, packageDir: packageDir }; + cb(error, result); + }); + })()); + } + }); + + Future.wait(futures); + + _.each(futures, function (f) { + var result = f.get(); + files.mkdir_p(result.packageDir); + files.extractTarGz(result.buffer, result.packageDir); + }); + + return manifest; + }, + // Load and return a manifest for an app, based on the // .meteor/version file // @@ -393,60 +440,16 @@ var packages = module.exports = { // Return parsed manifest. // // If .meteor/version does not exist, return null. - // - // Options: - // - versionOverride (String). Used instead of reading .meteor/version - manifestForProject: function (appDir, options) { + manifestForProject: function (appDir) { var project = require(path.join(__dirname, 'project.js')); var self = this; - options = options || {}; - - var releaseVersion = options.versionOverride || project.getMeteorReleaseVersion(appDir); + var releaseVersion = project.getMeteorReleaseVersion(appDir); if (!releaseVersion) { return null; // no manifest found } else { - var manifestPath = path.join( - __dirname, '..', 'cache', 'manifest', releaseVersion + '.json'); - - var manifest; - if (fs.existsSync(manifestPath)) { - // read from cache - manifest = JSON.parse(fs.readFileSync(manifestPath)); - } else { - // grow cache with new manifest and packages - manifest = packages.populateCacheForReleaseVersion(releaseVersion); - } - - var Future = require('fibers/future'); - var futures = []; - _.each(manifest.packages, function (version, name) { - if (!self.existsInPackageCache(name, version)) { - var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version); - var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + - version + ".tar.gz"; - - console.log("Fetching " + packageUrl + "..."); - futures.push(Future.wrap(function (cb) { - files.getUrl({url: packageUrl, encoding: null}, function (error, result) { - if (! error && result) - result = { buffer: result, packageDir: packageDir }; - cb(error, result); - }); - })()); - } - }); - - Future.wait(futures); - - _.each(futures, function (f) { - var result = f.get(); - files.mkdir_p(result.packageDir); - files.extractTarGz(result.buffer, result.packageDir); - }); - - return manifest; + return this.manifestForReleaseVersion(releaseVersion); } } }; diff --git a/lib/test-runner-app/.meteor/.gitignore b/lib/test-runner-app/.meteor/.gitignore new file mode 100644 index 0000000000..4083037423 --- /dev/null +++ b/lib/test-runner-app/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/lib/test-runner-app/.meteor/packages b/lib/test-runner-app/.meteor/packages new file mode 100644 index 0000000000..b63d829fb4 --- /dev/null +++ b/lib/test-runner-app/.meteor/packages @@ -0,0 +1 @@ +test-in-browser diff --git a/tools/cli-test.sh b/tools/cli-test.sh index 6839e6a2be..81cd1785b9 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -29,6 +29,7 @@ echo "... --help" $METEOR --help | grep "List available" > /dev/null $METEOR run --help | grep "Port to listen" > /dev/null +$METEOR test-package --help | grep "Port to listen" > /dev/null $METEOR create --help | grep "Make a subdirectory" > /dev/null $METEOR update --help | grep "Checks to see" > /dev/null $METEOR add --help | grep "Adds packages" > /dev/null From 63d69a4a848098f54ba21088f83d09a7083e1825 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 3 Jan 2013 13:37:16 -0800 Subject: [PATCH 040/450] Support `meteor test-package --deploy` --- lib/deploy.js | 29 +++++++++++++++-------------- lib/meteor.js | 17 +++++++++++------ 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/lib/deploy.js b/lib/deploy.js index 1f57855e36..6c5cc289a1 100644 --- a/lib/deploy.js +++ b/lib/deploy.js @@ -62,39 +62,39 @@ var deploy_app = function (url, app_dir, opt_debug, // a bit contorted here to make sure we ask for the password before // launching the slow bundle process. - with_password(parsed_url.hostname, function (password) { var deployOptions = { site: parsed_url.hostname, - appDir: app_dir, - debug: opt_debug, password: password, settings: settings }; + var bundleOptions = { + nodeModulesMode: 'skip', + noMinify: !!opt_debug + }; if (opt_set_password) get_new_password(function (set_password) { deployOptions.setPassword = set_password; - bundle_and_deploy(deployOptions); + bundle_and_deploy(app_dir, bundleOptions, deployOptions); }); else - bundle_and_deploy(deployOptions); + bundle_and_deploy(app_dir, bundleOptions, deployOptions); }); }; -var bundle_and_deploy = function (options) { - var site = options.site; - var app_dir = options.appDir; - var opt_debug = options.debug; - var password = options.password; - var set_password = options.setPassword; - var settings = options.settings; +var bundle_and_deploy = function (app_dir, bundleOptions, deployOptions) { + // might be parsing twice if called from deploy_app but that's fine + var site = parse_url(deployOptions.site).hostname; + + var password = deployOptions.password; + var set_password = deployOptions.setPassword; + var settings = deployOptions.settings; var build_dir = path.join(app_dir, '.meteor', 'local', 'build_tar'); var bundle_path = path.join(build_dir, 'bundle'); - var bundle_opts = { nodeModulesMode: 'skip', noMinify: !!opt_debug }; process.stdout.write('Deploying to ' + site + '. Bundling ... '); var bundler = require(path.join(__dirname, '..', 'lib', 'bundler.js')); - var errors = bundler.bundle(app_dir, bundle_path, bundle_opts); + var errors = bundler.bundle(app_dir, bundle_path, bundleOptions); if (errors) { process.stdout.write("\n\nErrors prevented deploying:\n"); _.each(errors, function (e) { @@ -388,6 +388,7 @@ var get_new_password = function (callback) { }); }; +exports.bundle_and_deploy = bundle_and_deploy; exports.deploy_app = deploy_app; exports.delete_app = delete_app; exports.mongo = mongo; diff --git a/lib/meteor.js b/lib/meteor.js index a4289b53ea..53e2c46de6 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -163,17 +163,22 @@ Fiber(function () { testPackages = _.keys(packages.list(manifest)); } - // XXX implement --deploy. Need to refactor some of deploy.js to make this possible - // since the bundler options are created within bundle_and_deploy - - var bundle_opts = { - nodeModulesMode: 'symlink', + var bundleOptions = { + nodeModulesMode: new_argv.deploy ? 'skip' : 'symlink', testPackages: testPackages, versionOverride: releaseVersion, noMinify: true // curiously, if this isn't set we get "RangeError: Maximum call stack size exceeded" during minification }; app_dir = path.join(__dirname, 'test-runner-app'); - runner.run(app_dir, bundle_opts, new_argv.port); + + if (new_argv.deploy) { + var deployOptions = { + site: new_argv.deploy + }; + deploy.bundle_and_deploy(app_dir, bundleOptions, deployOptions); + } else { + runner.run(app_dir, bundle_opts, new_argv.port); + } } }); From 56e08b073ee63c4025c012da52e2ecd965d8815e Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 3 Jan 2013 14:43:45 -0800 Subject: [PATCH 041/450] `meteor test-package` finds latest release manifest --- lib/meteor.js | 4 ++-- lib/packages.js | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/meteor.js b/lib/meteor.js index 53e2c46de6..feb4681928 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -151,8 +151,8 @@ Fiber(function () { process.exit(1); } - // XXX should find latest release if not specified - var releaseVersion = new_argv['release-version'] || '0.1'; + var packages = require(path.join(__dirname, 'packages.js')); + var releaseVersion = new_argv['release-version'] || packages.latestReleaseVersion(); var testPackages; if (new_argv._[1]) { diff --git a/lib/packages.js b/lib/packages.js index d4f3f9077a..858c40d38e 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -384,6 +384,25 @@ var packages = module.exports = { } }, + // look in the manifest cache for the latest release version + latestReleaseVersion: function() { + var manifestPath = path.join(__dirname, '..', 'cache', 'manifest'); + var files = fs.readdirSync(manifestPath); + var semver = require('semver'); + + var latestReleaseVersion = null; + _.each(files, function(file) { + var match = /^(.*)\.json$/.exec(file); + if (match) { + var version = match[1]; + if (semver.valid(version) && (!latestReleaseVersion || semver.gt(version, latestReleaseVersion))) + latestReleaseVersion = version; + } + }); + + return latestReleaseVersion; + }, + // Load the manifest corresponding to a given meteor release from // packages.meteor.com and cache on disk. Parse and ensure that all // used package versions are cached. Return parsed manifest. From 40ef67fc93e55522af0456169dc6f2d640e6e83b Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 3 Jan 2013 15:26:53 -0800 Subject: [PATCH 042/450] More testing improvements for `meteor test-packages` and bundler. --- lib/meteor.js | 9 ++-- lib/tests/test_bundler.js | 12 ++++- tools/cli-test.sh | 111 ++++++++++++++++++++++---------------- 3 files changed, 80 insertions(+), 52 deletions(-) diff --git a/lib/meteor.js b/lib/meteor.js index feb4681928..0205d0cf24 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -130,7 +130,7 @@ Fiber(function () { }); Commands.push({ - name: "test-package", + name: "test-packages", help: "Test one or more packages", func: function (argv) { // reparse args @@ -141,8 +141,9 @@ Fiber(function () { .describe('version', 'Meteor release version to test.') .describe('deploy', 'Optionally, specify a domain to deploy to instead of running locally.') .usage( - "Usage: meteor test-package [options] [comma delimited packages]\n" + - "XXX\n"); + "Usage: meteor test-packages [options] [comma delimited packages]\n" + + "\n" + + "Run unit tests for packages. Point your browser to localhost:3000 to see results."); var new_argv = opt.argv; @@ -177,7 +178,7 @@ Fiber(function () { }; deploy.bundle_and_deploy(app_dir, bundleOptions, deployOptions); } else { - runner.run(app_dir, bundle_opts, new_argv.port); + runner.run(app_dir, bundleOptions, new_argv.port); } } }); diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 5245b4b868..86bcbb2b2f 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -12,6 +12,8 @@ var inFiber = require(path.join(__dirname, '..', 'fiber-helpers.js')).inFiber; // print stack track and exit with error code if an assertion fails process.on('uncaughtException', function (err) { console.log(err.stack); + console.log(); + console.log('Bundle can be found at ' + lastTmpDir); process.exit(1); }); @@ -19,8 +21,13 @@ process.on('uncaughtException', function (err) { /// UTILITIES /// +var tmpBaseDir = files.mkdtemp('test_bundler'); +var tmpCounter = 1; +var lastTmpDir; var tmpDir = function () { - return files.mkdtemp('test_bundler'); + lastTmpDir = path.join(tmpBaseDir, "" + (tmpCounter++) /* path.join likes string, not numbers */); + files.mkdir_p(lastTmpDir); + return lastTmpDir; }; /// @@ -136,3 +143,6 @@ assert.doesNotThrow(inFiber(function () { "require(require('path').join(__dirname, 'server', 'server.js'));"); })); +/// SUCCESS + +files.rm_recursive(tmpBaseDir); \ No newline at end of file diff --git a/tools/cli-test.sh b/tools/cli-test.sh index 81cd1785b9..d61c623caa 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -18,7 +18,8 @@ if [ "$1" == "--global" ]; then fi DIR=`mktemp -d -t meteor-cli-test-XXXXXXXX` -trap 'echo FAILED ; rm -rfd `find $METEOR_DIR -name __tmp`; rm -rf "$DIR" >/dev/null 2>&1' EXIT +OUTPUT="$DIR/output" +trap 'echo "[...]"; tail -25 $OUTPUT; echo FAILED ; rm -rfd `find $METEOR_DIR -name __tmp`; rm -rf "$DIR" >/dev/null 2>&1' EXIT cd "$DIR" set -e -x @@ -27,31 +28,31 @@ set -e -x echo "... --help" -$METEOR --help | grep "List available" > /dev/null -$METEOR run --help | grep "Port to listen" > /dev/null -$METEOR test-package --help | grep "Port to listen" > /dev/null -$METEOR create --help | grep "Make a subdirectory" > /dev/null -$METEOR update --help | grep "Checks to see" > /dev/null -$METEOR add --help | grep "Adds packages" > /dev/null -$METEOR remove --help | grep "Removes a package" > /dev/null -$METEOR list --help | grep "Without arguments" > /dev/null -$METEOR bundle --help | grep "Package this project" > /dev/null -$METEOR mongo --help | grep "Opens a Mongo" > /dev/null -$METEOR deploy --help | grep "Deploys the project" > /dev/null -$METEOR logs --help | grep "Retrieves the" > /dev/null -$METEOR reset --help | grep "Reset the current" > /dev/null +$METEOR --help | grep "List available" >> $OUTPUT +$METEOR run --help | grep "Port to listen" >> $OUTPUT +$METEOR test-packages --help | grep "Port to listen" >> $OUTPUT +$METEOR create --help | grep "Make a subdirectory" >> $OUTPUT +$METEOR update --help | grep "Checks to see" >> $OUTPUT +$METEOR add --help | grep "Adds packages" >> $OUTPUT +$METEOR remove --help | grep "Removes a package" >> $OUTPUT +$METEOR list --help | grep "Without arguments" >> $OUTPUT +$METEOR bundle --help | grep "Package this project" >> $OUTPUT +$METEOR mongo --help | grep "Opens a Mongo" >> $OUTPUT +$METEOR deploy --help | grep "Deploys the project" >> $OUTPUT +$METEOR logs --help | grep "Retrieves the" >> $OUTPUT +$METEOR reset --help | grep "Reset the current" >> $OUTPUT echo "... not in dir" -$METEOR | grep "run: You're not in" > /dev/null -$METEOR run | grep "run: You're not in" > /dev/null -$METEOR add foo | grep "add: You're not in" > /dev/null -$METEOR remove foo | grep "remove: You're not in" > /dev/null -$METEOR list --using | grep "list --using: You're not in" > /dev/null -$METEOR bundle foo.tar.gz | grep "bundle: You're not in" > /dev/null -$METEOR mongo | grep "mongo: You're not in" > /dev/null -$METEOR deploy automated-test | grep "deploy: You're not in" > /dev/null -$METEOR reset | grep "reset: You're not in" > /dev/null +$METEOR | grep "run: You're not in" >> $OUTPUT +$METEOR run | grep "run: You're not in" >> $OUTPUT +$METEOR add foo | grep "add: You're not in" >> $OUTPUT +$METEOR remove foo | grep "remove: You're not in" >> $OUTPUT +$METEOR list --using | grep "list --using: You're not in" >> $OUTPUT +$METEOR bundle foo.tar.gz | grep "bundle: You're not in" >> $OUTPUT +$METEOR mongo | grep "mongo: You're not in" >> $OUTPUT +$METEOR deploy automated-test | grep "deploy: You're not in" >> $OUTPUT +$METEOR reset | grep "reset: You're not in" >> $OUTPUT echo "... create" @@ -67,13 +68,13 @@ cd .meteor echo "... add/remove/list" -$METEOR list | grep "backbone" > /dev/null -! $METEOR list --using 2>&1 | grep "backbone" > /dev/null -$METEOR add backbone 2>&1 | grep "backbone:" | grep -v "no such package" | > /dev/null -$METEOR list --using | grep "backbone" > /dev/null -grep backbone packages > /dev/null # remember, we are already in .meteor -$METEOR remove backbone 2>&1 | grep "backbone: removed" > /dev/null -! $METEOR list --using 2>&1 | grep "backbone" > /dev/null +$METEOR list | grep "backbone" >> $OUTPUT +! $METEOR list --using 2>&1 | grep "backbone" >> $OUTPUT +$METEOR add backbone 2>&1 | grep "backbone:" | grep -v "no such package" | >> $OUTPUT +$METEOR list --using | grep "backbone" >> $OUTPUT +grep backbone packages >> $OUTPUT # remember, we are already in .meteor +$METEOR remove backbone 2>&1 | grep "backbone: removed" >> $OUTPUT +! $METEOR list --using 2>&1 | grep "backbone" >> $OUTPUT echo "... bundle" @@ -89,21 +90,21 @@ MONGOMARK='--bind_ip 127.0.0.1 --smallfiles --port 9102' # (the || true is needed on linux, whose xargs will invoke kill even with no args) ps ax | grep -e 'meteor.js -p 9100' | grep -v grep | awk '{print $1}' | xargs kill || true -! $METEOR mongo > /dev/null 2>&1 -$METEOR reset > /dev/null 2>&1 +! $METEOR mongo >> $OUTPUT 2>&1 +$METEOR reset >> $OUTPUT 2>&1 test ! -d .meteor/local -! ps ax | grep -e "$MONGOMARK" | grep -v grep > /dev/null +! ps ax | grep -e "$MONGOMARK" | grep -v grep >> $OUTPUT PORT=9100 -$METEOR -p $PORT > /dev/null 2>&1 & +$METEOR -p $PORT >> $OUTPUT 2>&1 & METEOR_PID=$! sleep 2 # XXX XXX lame test -d .meteor/local/db -ps ax | grep -e "$MONGOMARK" | grep -v grep > /dev/null -curl -s "http://localhost:$PORT" > /dev/null +ps ax | grep -e "$MONGOMARK" | grep -v grep >> $OUTPUT +curl -s "http://localhost:$PORT" >> $OUTPUT echo "show collections" | $METEOR mongo @@ -112,19 +113,35 @@ kill $METEOR_PID sleep 10 # XXX XXX lame. have to wait for inner app to die via keepalive! -! ps ax | grep "$METEOR_PID" | grep -v grep > /dev/null -ps ax | grep -e "$MONGOMARK" | grep -v grep > /dev/null +! ps ax | grep "$METEOR_PID" | grep -v grep >> $OUTPUT +ps ax | grep -e "$MONGOMARK" | grep -v grep >> $OUTPUT echo "... rerun" -$METEOR -p $PORT > /dev/null 2>&1 & +$METEOR -p $PORT >> $OUTPUT 2>&1 & METEOR_PID=$! sleep 2 # XXX XXX lame -ps ax | grep -e "$MONGOMARK" | grep -v grep > /dev/null -curl -s "http://localhost:$PORT" > /dev/null +ps ax | grep -e "$MONGOMARK" | grep -v grep >> $OUTPUT +curl -s "http://localhost:$PORT" >> $OUTPUT + +kill $METEOR_PID +sleep 10 # XXX XXX lame. have to wait for inner app to die via keepalive! + +ps ax | grep -e "$MONGOMARK" | grep -v grep | awk '{print $1}' | xargs kill || true +sleep 2 # need to make sure these kills take effect + +echo "... test-packages" + +$METEOR test-packages -p $PORT >> $OUTPUT 2>&1 & +METEOR_PID=$! + +sleep 2 # XXX XXX lame + +ps ax | grep -e "$MONGOMARK" | grep -v grep >> $OUTPUT +curl -s "http://localhost:$PORT" >> $OUTPUT kill $METEOR_PID sleep 10 # XXX XXX lame. have to wait for inner app to die via keepalive! @@ -142,7 +159,7 @@ sleep 1 $METEOR -p $PORT > error.txt || true -grep 'port was closed' error.txt > /dev/null +grep 'port was closed' error.txt >> $OUTPUT # Kill the server by connecting to it. $NODE -e 'require("net").connect({host:"127.0.0.1",port:'$PORT'+2},function(){process.exit(0);})' @@ -165,7 +182,7 @@ if (Meteor.isServer) { } EOF -$METEOR -p $PORT --settings='settings.json' --once > /dev/null +$METEOR -p $PORT --settings='settings.json' --once >> $OUTPUT # prepare die.js so that we have a server that loads packages and dies @@ -182,8 +199,8 @@ cat > $METEOR_DIR/local-package-sets/__tmp/a-package-named-bar/package.js < /dev/null -$METEOR -p $PORT --once | grep "loaded a-package-named-bar" > /dev/null +$METEOR add a-package-named-bar >> $OUTPUT +$METEOR -p $PORT --once | grep "loaded a-package-named-bar" >> $OUTPUT rm -rf $METEOR_DIR/local-package-sets/__tmp/ @@ -198,8 +215,8 @@ Package.describe({ EOF -$METEOR add accounts-ui 2>&1 | grep "accounts-ui - overridden" > /dev/null -$METEOR list | grep "accounts-ui - overridden" > /dev/null +$METEOR add accounts-ui 2>&1 | grep "accounts-ui - overridden" >> $OUTPUT +$METEOR list | grep "accounts-ui - overridden" >> $OUTPUT rm -rf $METEOR_DIR/local-package-sets/__tmp/ From 665f5ee9e4d8ea03598278e7fa7e87a04c228c9e Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 3 Jan 2013 15:37:26 -0800 Subject: [PATCH 043/450] fix typo --- lib/tests/test_bundler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 86bcbb2b2f..1ea2e0ef85 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -117,7 +117,7 @@ assert.doesNotThrow(inFiber(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), "require(require('path').join(__dirname, 'server', 'server.js'));"); - // node_modules directory exists and is not a symlink + // node_modules directory exists and is a symlink assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); // node_modules contains fibers assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); From 9af19b3c101cb7b5d2aa59479861071c8edcabf7 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Thu, 3 Jan 2013 17:58:24 -0800 Subject: [PATCH 044/450] Better URLS on packages.meteor.com --- lib/packages.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/packages.js b/lib/packages.js index 858c40d38e..8d0c8a8fd0 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -426,7 +426,7 @@ var packages = module.exports = { if (!self.existsInPackageCache(name, version)) { var packageDir = path.join(__dirname, '..', 'cache', 'packages', name, version); var packageUrl = PACKAGES_URLBASE + "/packages/" + name + "/" + - version + ".tar.gz"; + name + '-' + version + ".tar.gz"; console.log("Fetching " + packageUrl + "..."); futures.push(Future.wrap(function (cb) { From 9f4702a04409e147d60f2302d8be5655adb39e9a Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Mon, 7 Jan 2013 18:47:05 -0800 Subject: [PATCH 045/450] Load packages from PACKAGE_DIRS and CHECKOUT_DIR/packages Some rehaul of tests for this purpose and general cleanup, including re-enabling reloading app when packages change --- lib/files.js | 33 ++++++--- lib/packages.js | 71 +++++++------------ lib/run.js | 8 ++- lib/skel/.meteor/version | 2 +- lib/tests/empty-versioned-app/.meteor/version | 2 +- lib/tests/test_bundler.js | 28 +++++--- tools/cli-test.sh | 33 ++++----- 7 files changed, 91 insertions(+), 86 deletions(-) diff --git a/lib/files.js b/lib/files.js index 3f39129fc7..23dc8536eb 100644 --- a/lib/files.js +++ b/lib/files.js @@ -122,14 +122,26 @@ var files = module.exports = { return fs.existsSync(path.join(filepath, 'package.js')); }, - // given a path, return true if this is a collection of packages. - // This is used to run all the tests in meteor. - is_package_collection_dir: function (filepath) { - // XXX implementation is kinda specific to our code base, but this - // is better than confusing the hell out of someone who names their - // project 'packages' - return path.basename(filepath) === 'packages' && - fs.existsSync(path.join(filepath, 'meteor', 'package.js')); + localPackageDirs: function () { + var packageDirs = [path.join(__dirname, '..', 'packages')]; + if (process.env.PACKAGE_DIRS) + packageDirs = process.env.PACKAGE_DIRS.split(':').concat(packageDirs); + return packageDirs; + }, + + // for a packge that exists in localPackageDirs, find the directory + // in which it exists + localPackageDir: function(name) { + var ret; + _.find(this.localPackageDirs(), function(packageDir) { + var dir = path.join(packageDir, name); + if (fs.existsSync(path.join(dir, 'package.js'))) { + ret = dir; + return true; + } + }); + + return ret; }, // given a predicate function and a starting path, traverse upwards @@ -204,10 +216,9 @@ var files = module.exports = { return path.join(__dirname, '..'); }, - // Return the directory that contains the core tool (the top-level - // 'app' directory) + // Return the top-level directory for this meteor install or checkout get_core_dir: function () { - return path.join(__dirname, '..', '..', 'app'); + return path.join(__dirname, '..'); }, // Try to find the prettiest way to present a path to the diff --git a/lib/packages.js b/lib/packages.js index 8d0c8a8fd0..2713442b02 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -92,32 +92,18 @@ var Package = function () { }; _.extend(Package.prototype, { + // Searches: + // - $PACKAGE_DIRS (colon-separated) + // - $METEOR/packages // @returns {Boolean} was the package found in any local package sets? - initFromLocalPackageSets: function (name) { - var self = this; - var setsDir = path.join(__dirname, '..', 'local-package-sets'); - if (!fs.existsSync(setsDir)) + initFromLocalPackages: function (name) { + var packageDir = files.localPackageDir(name); + if (packageDir) { + this._initFromPackageDir(name, packageDir); + return true; + } else { return false; - - var sets = fs.readdirSync(setsDir); - var found = false; - _.each(sets, function (set) { - - if (found) - return; - - var setDir = path.join(setsDir, set); - if (fs.statSync(setDir).isDirectory()) { - var packageNames = fs.readdirSync(setDir); - if (_.contains(packageNames, name)) { - found = true; - self._initFromPackageDir(name, - path.join(setsDir, set, name)); - } - } - }); - - return found; + } }, initFromPackageCache: function (name, version) { @@ -246,8 +232,9 @@ _.extend(Package.prototype, { } }); -// in the future, this could be an on-disk cache that tracks mtimes. -var package_cache = {}; +// (OLD COMMENT?) in the future, this could be an on-disk cache that +// tracks mtimes. +var compiledPackages = {}; var packages = module.exports = { @@ -256,19 +243,19 @@ var packages = module.exports = { var self = this; if (name instanceof Package) return name; - if (!(name in package_cache)) { + if (!(name in compiledPackages)) { var pkg = new Package; - if (pkg.initFromLocalPackageSets(name)) { - package_cache[name] = pkg; + if (pkg.initFromLocalPackages(name)) { + compiledPackages[name] = pkg; } else { if (manifest) { pkg.initFromPackageCache(name, manifest.packages[name]); - package_cache[name] = pkg; + compiledPackages[name] = pkg; } } } - return package_cache[name]; + return compiledPackages[name]; }, // get a package that represents an app. (ignore_files is optional @@ -282,7 +269,7 @@ var packages = module.exports = { // force reload of all packages flush: function () { - package_cache = {}; + compiledPackages = {}; }, // get all packages available, in both local package sets and in the package @@ -292,22 +279,14 @@ var packages = module.exports = { var self = this; var list = {}; - var setsDir = path.join(__dirname, '..', 'local-package-sets'); - if (fs.existsSync(setsDir)) { - var sets = fs.readdirSync(setsDir); - _.each(sets, function (set) { - var setDir = path.join(setsDir, set); - if (fs.statSync(setDir).isDirectory()) { - var packageNames = fs.readdirSync(path.join(setsDir, set)); - _.each(packageNames, function (name) { - if (!list[name]) - list[name] = packages.get(null, name); // empty manifest, we're loading from local packages - else - throw new Error("Found package " + name + " more than once in local package sets."); - }); + _.each(files.localPackageDirs(), function (dir) { + _.each(fs.readdirSync(dir), function (name) { + if (files.is_package_dir(path.join(dir, name))) { + if (!list[name]) + list[name] = packages.get(null, name); // empty manifest, we're loading from local packages } }); - } + }); if (manifest) { _.each(manifest.packages, function(version, name) { diff --git a/lib/run.js b/lib/run.js index 10b5be3bc2..273f613966 100644 --- a/lib/run.js +++ b/lib/run.js @@ -337,8 +337,12 @@ var DependencyWatcher = function (deps, app_dir, relativeFiles, on_change) { // Additional list of specific files that are interesting. self.specific_files = {}; - // #PackageOverride use this mechanism to support local package - // override directories + for (var pkg in (deps.packages || {})) { + _.each(deps.packages[pkg], function (file) { + self.specific_files[path.join(files.localPackageDir(pkg), file)] + = true; + }); + }; _.each(relativeFiles, function (file) { self.specific_files[file] = true; diff --git a/lib/skel/.meteor/version b/lib/skel/.meteor/version index 49d59571fb..8acdd82b76 100644 --- a/lib/skel/.meteor/version +++ b/lib/skel/.meteor/version @@ -1 +1 @@ -0.1 +0.0.1 diff --git a/lib/tests/empty-versioned-app/.meteor/version b/lib/tests/empty-versioned-app/.meteor/version index 49d59571fb..8acdd82b76 100644 --- a/lib/tests/empty-versioned-app/.meteor/version +++ b/lib/tests/empty-versioned-app/.meteor/version @@ -1 +1 @@ -0.1 +0.0.1 diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index 1ea2e0ef85..d30a4b1dfb 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -123,24 +123,34 @@ assert.doesNotThrow(inFiber(function () { assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); })); -// unversioned app, no options +// unversioned app, no options -- should look in packages/ assert.doesNotThrow(inFiber(function () { var tmpOutputDir = tmpDir(); var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); - assert.notEqual(errors.length, 0); - assert.notEqual(errors[0].indexOf('Exception while bundling'), -1); - assert.notEqual(errors[0].indexOf('Package not found: meteor'), -1); -})); -// unversioned app, using `versionOverride` -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.1', nodeModulesMode: 'skip'}); assert.strictEqual(errors, undefined); // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), "require(require('path').join(__dirname, 'server', 'server.js'));"); + + // XXX actually test that we get the contents out of packages/? + + // XXX someone test that if this is an installed version (that doesn't have packages/) + // then this fails with an appropriate error ("Can't find package 'meteor'") +})); + +// unversioned app, using `versionOverride` +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.0.1', nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + + // XXX actually test that we get the contents out of the cache? })); /// SUCCESS diff --git a/tools/cli-test.sh b/tools/cli-test.sh index d61c623caa..b7c280fde9 100755 --- a/tools/cli-test.sh +++ b/tools/cli-test.sh @@ -17,11 +17,11 @@ if [ "$1" == "--global" ]; then METEOR=/usr/local/bin/meteor fi -DIR=`mktemp -d -t meteor-cli-test-XXXXXXXX` -OUTPUT="$DIR/output" -trap 'echo "[...]"; tail -25 $OUTPUT; echo FAILED ; rm -rfd `find $METEOR_DIR -name __tmp`; rm -rf "$DIR" >/dev/null 2>&1' EXIT +TMPDIR=`mktemp -d -t meteor-cli-test-XXXXXXXX` +OUTPUT="$TMPDIR/output" +trap 'echo "[...]"; tail -25 $OUTPUT; echo FAILED ; rm -rfd `find $METEOR_DIR -name __tmp`; rm -rf "$TMPDIR" >/dev/null 2>&1' EXIT -cd "$DIR" +cd "$TMPDIR" set -e -x ## Begin actual tests @@ -194,31 +194,32 @@ EOF echo "... local-package-sets -- new package" -mkdir -p $METEOR_DIR/local-package-sets/__tmp/a-package-named-bar/ -cat > $METEOR_DIR/local-package-sets/__tmp/a-package-named-bar/package.js < "$TMPDIR/local-packages/a-package-named-bar/package.js" <> $OUTPUT -$METEOR -p $PORT --once | grep "loaded a-package-named-bar" >> $OUTPUT - -rm -rf $METEOR_DIR/local-package-sets/__tmp/ +! $METEOR add a-package-named-bar >> $OUTPUT +PACKAGE_DIRS="$TMPDIR/local-packages" $METEOR add a-package-named-bar >> $OUTPUT +! $METEOR -p $PORT --once | grep "loaded a-package-named-bar" >> $OUTPUT +PACKAGE_DIRS="$TMPDIR/local-packages" $METEOR -p $PORT --once | grep "loaded a-package-named-bar" >> $OUTPUT echo "... local-package-sets -- overridden package" -mkdir -p $METEOR_DIR/local-package-sets/__tmp/accounts-ui/ -cat > $METEOR_DIR/local-package-sets/__tmp/accounts-ui/package.js < "$TMPDIR/local-packages/accounts-ui/package.js" <&1 | grep "accounts-ui - overridden" >> $OUTPUT -$METEOR list | grep "accounts-ui - overridden" >> $OUTPUT - -rm -rf $METEOR_DIR/local-package-sets/__tmp/ +! $METEOR add accounts-ui 2>&1 | grep "accounts-ui - overridden" >> $OUTPUT +$METEOR remove accounts-ui 2>&1 >> $OUTPUT +PACKAGE_DIRS="$TMPDIR/local-packages" $METEOR add accounts-ui 2>&1 | grep "accounts-ui - overridden" >> $OUTPUT +! $METEOR list | grep "accounts-ui - overridden" >> $OUTPUT +PACKAGE_DIRS="$TMPDIR/local-packages" $METEOR list | grep "accounts-ui - overridden" >> $OUTPUT # remove die.js, we're done with package tests. From ecce7f00c63ea6e279bee492a72f7d7210a759b3 Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Mon, 7 Jan 2013 19:43:30 -0800 Subject: [PATCH 046/450] meteor test-packages --verison ==> --release also actually make it work. :/ --- lib/meteor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/meteor.js b/lib/meteor.js index 0205d0cf24..ccc87b24f6 100644 --- a/lib/meteor.js +++ b/lib/meteor.js @@ -138,7 +138,7 @@ Fiber(function () { var opt = require('optimist') .alias('port', 'p').default('port', 3000) .describe('port', 'Port to listen on. NOTE: Also uses port N+1 and N+2.') - .describe('version', 'Meteor release version to test.') + .describe('release', 'Meteor release version to test.') .describe('deploy', 'Optionally, specify a domain to deploy to instead of running locally.') .usage( "Usage: meteor test-packages [options] [comma delimited packages]\n" + @@ -153,7 +153,7 @@ Fiber(function () { } var packages = require(path.join(__dirname, 'packages.js')); - var releaseVersion = new_argv['release-version'] || packages.latestReleaseVersion(); + var releaseVersion = new_argv['release'] || packages.latestReleaseVersion(); var testPackages; if (new_argv._[1]) { From 5352db933a06f65339bd559a330251f0cafd1cce Mon Sep 17 00:00:00 2001 From: Avital Oliver Date: Wed, 16 Jan 2013 12:45:49 -0800 Subject: [PATCH 047/450] Implement a `useNpm` directive for packages --- .gitignore | 1 + lib/bundler.js | 64 +++++- lib/files.js | 3 +- lib/meteor_npm.js | 210 ++++++++++++++++++ lib/packages.js | 2 + lib/tests/test_bundler.js | 169 ++------------ lib/tests/test_bundler_npm.js | 190 ++++++++++++++++ lib/tests/test_bundler_options.js | 123 ++++++++++ .../.meteor/.gitignore | 1 + .../.meteor/packages | 1 + packages/templating/html_scanner.js | 2 +- packages/test-helpers/async_multi.js | 2 +- packages/test-helpers/seeded_random.js | 2 +- server/server.js | 22 +- tools/cli-test.sh | 11 + 15 files changed, 641 insertions(+), 162 deletions(-) create mode 100644 lib/meteor_npm.js create mode 100644 lib/tests/test_bundler_npm.js create mode 100644 lib/tests/test_bundler_options.js create mode 100644 lib/tests/unversioned-app-with-package/.meteor/.gitignore create mode 100644 lib/tests/unversioned-app-with-package/.meteor/packages diff --git a/.gitignore b/.gitignore index bcc5e6b119..25f24e994b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ cache TAGS *.log *.out +npm-debug.log diff --git a/lib/bundler.js b/lib/bundler.js index 632edfaddc..5b568a882e 100644 --- a/lib/bundler.js +++ b/lib/bundler.js @@ -99,6 +99,38 @@ var PackageInstance = function (pkg, bundle) { }); }, + // Called when this package wants to depend on node + // modules for server code. + // + // @param npmDependencies {Object} eg {gcd: "0.0.0", tar: "0.1.14"} + useNpm: function (npmDependencies) { + var meteorNpm = require(path.join(__dirname, 'meteor_npm.js')); + + if (this.npmDependencies) + throw new Error("Can only call `useNpm` once."); + this.npmDependencies = npmDependencies; + + // don't allow npm fuzzy versions so that there is complete + // consistency when deploying a meteor app + // + // XXX use something like seal or lockdown to have *complete* confidence + // we're running the same code? + meteorNpm.ensureOnlyExactVersions(npmDependencies); + + // create a package .npm subdirectory for npm bookkeeping and node_modules + var packageNpmDir = meteorNpm.ensurePackageNpmDir(self.pkg.source_root); + + // go through a specialized npm dependencies update process, ensuring + // we don't get new versions of any (sub)dependencies + meteorNpm.updateDependencies(packageNpmDir, npmDependencies); + + // map the generated node_modules directory to the package directory + // within the bundle + var nodeModulesPath = path.join(packageNpmDir, 'node_modules'); + var relNodeModulesPath = ['packages', self.pkg.name, 'node_modules'].join('/'); + self.bundle.serverExternalDirs[relNodeModulesPath] = nodeModulesPath; + }, + add_files: function (paths, where) { if (!(paths instanceof Array)) paths = paths ? [paths] : []; @@ -215,8 +247,8 @@ var Bundle = function () { // Packages that have had tests included. Map from package id to instance self.tests_included = {}; - // release manifest - self.manifest = null; + // meteor release manifest + self.releaseManifest = null; // map from environment, to list of filenames self.js = {client: [], server: []}; @@ -239,6 +271,11 @@ var Bundle = function () { // the individual input files that were combined. self.manifest = []; + // these directories are copied (cp -r) without any intervention, eg + // for node modules. maps target path (server relative) to + // source directory on disk + self.serverExternalDirs = {}; + // list of segments of additional HTML for / self.head = []; self.body = []; @@ -386,7 +423,7 @@ _.extend(Bundle.prototype, { includeTests: function (packageName) { var self = this; - var pkg = packages.get(self.manifest, packageName); + var pkg = packages.get(self.releaseManifest, packageName); if (self.tests_included[pkg.id]) return; self.tests_included[pkg.id] = true; @@ -613,9 +650,16 @@ _.extend(Bundle.prototype, { for (var rel_path in self.files.server) { var path_in_bundle = path.join('app', rel_path); var full_path = path.join(build_path, path_in_bundle); - app_json.load.push(path_in_bundle); files.mkdir_p(path.dirname(full_path), 0755); fs.writeFileSync(full_path, self.files.server[rel_path]); + app_json.load.push(path_in_bundle); + } + + // `node_modules` directories for packages + for (var rel_path in self.serverExternalDirs) { + var path_in_bundle = path.join('app', rel_path); + var full_path = path.join(build_path, path_in_bundle); + files.cp_r(self.serverExternalDirs[rel_path], full_path); } var app_html = self._generate_app_html(); @@ -725,21 +769,21 @@ exports.bundle = function (app_dir, output_path, options) { packages.flush(); var bundle = new Bundle; - var manifest; + var releaseManifest; if (options.versionOverride) - manifest = packages.manifestForReleaseVersion(options.versionOverride); + releaseManifest = packages.manifestForReleaseVersion(options.versionOverride); else - manifest = packages.manifestForProject(app_dir); + releaseManifest = packages.manifestForProject(app_dir); - if (!manifest) { + if (!releaseManifest) { // XXX We should instead use the latest version installed, and // notify the user. // https://app.asana.com/0/2604247562419/2765125200674 console.log("Couldn't find .meteor/version -- only searching local packages."); } - bundle.manifest = manifest; + bundle.releaseManifest = releaseManifest; - // our manifest is set, let's now load the app + // our release manifest is set, let's now load the app var app = packages.get_for_app(app_dir, ignore_files); bundle.use(app); diff --git a/lib/files.js b/lib/files.js index 23dc8536eb..219ea01f21 100644 --- a/lib/files.js +++ b/lib/files.js @@ -100,7 +100,6 @@ var files = module.exports = { return ret; }, - // given a path, returns true if it is a meteor application (has a // .meteor directory with a 'packages' file). false otherwise. is_app_dir: function (filepath) { @@ -248,7 +247,7 @@ var files = module.exports = { file = path.join(p, file); files.rm_recursive(file); }); - fs.rmdirSync(p) + fs.rmdirSync(p); } else fs.unlinkSync(p); }, diff --git a/lib/meteor_npm.js b/lib/meteor_npm.js new file mode 100644 index 0000000000..ae4e7d9ac3 --- /dev/null +++ b/lib/meteor_npm.js @@ -0,0 +1,210 @@ +var semver = require('semver'); +var exec = require('child_process').exec; +var Future = require('fibers/future'); + +var path = require('path'); +var fs = require('fs'); +var files = require(path.join(__dirname, 'files.js')); +var _ = require('underscore'); + +var meteorNpm = module.exports = { + ensureOnlyExactVersions: function(npmDependencies) { + _.each(npmDependencies, function(version, name) { + if (!semver.valid(version)) + throw new Error( + "Must declare exact version of npm package dependency: " + name + '@' + version); + }); + }, + + // Ensure a package has a well-structured .npm subdirectory. + // + // @returns {String} path to said .npm subdirectory + ensurePackageNpmDir: function(packageDir) { + var packageNpmDir = path.join(packageDir, '.npm'); + + if (fs.existsSync(packageNpmDir)) { + // upgrade npm dependencies + if (!fs.statSync(packageNpmDir).isDirectory()) { + throw new Error("Should be a directory: " + packageNpmDir); + } + } else { + console.log('npm: creating ' + packageNpmDir); + files.mkdir_p(packageNpmDir); + + // we recreate package.json each time we bundle, based on the + // arguments to useNpm. similarly, we recreate + // npm-shrinkwrap.json from meteor-npm-shrinkwrap.json, with + // some modifications. at the end of the bundling process we + // remove thes files but in case we crashed mid-way we make + // sure they're gitignored. + // + // node_modules shouldn't be in git since we recreate it as + // needed by using `npm install`. since we use `npm + // shrinkwrap` we're guarenteed to have the same version + // installed each time. + fs.writeFileSync(path.join(packageNpmDir, '.gitignore'), + ['package.json', 'npm-shrinkwrap.json', 'node_modules'].join('\n')); + } + + return packageNpmDir; + }, + + // @param npmDependencies {Object} dependencies that should be installed, + // eg {tar: '0.1.6', gcd: '0.0.0'} + updateDependencies: function(packageNpmDir, npmDependencies) { + var self = this; + + // prepare .npm dir from which we'll be calling out to npm, and + // compute dependencies that need to be updated + var dependenciesToUpdate = this._prepareForUpdate(packageNpmDir, npmDependencies); + + // if we have a shrinkwrap file, call `npm install`. + if (fs.existsSync(path.join(packageNpmDir, 'npm-shrinkwrap.json'))) { + if (!fs.existsSync(path.join(packageNpmDir, 'node_modules'))) { + console.log('installing shrinkwrapped npm dependencies into ' + packageNpmDir); + } else { + // just calling `npm install` to make sure we have all of the + // node_modules we should. eg if you ran this package before + // new dependencies were added and then you took a new + // version. + } + self._installFromShrinkwrap(packageNpmDir); + } + + // install modified dependencies + if (!_.isEmpty(dependenciesToUpdate)) { + process.stdout.write( + 'installing npm dependencies ' + this._depsToString(dependenciesToUpdate) + + ' into ' + packageNpmDir + '... '); + + _.each(dependenciesToUpdate, function(version, name) { + self._installNpmModule(name, version, packageNpmDir); + }); + + // before shrinkwrapping we need to delete unused `node_modules` directories + _.each(fs.readdirSync(path.join(packageNpmDir, 'node_modules')), function(installedModule) { + if (!npmDependencies[installedModule]) { + files.rm_recursive(path.join(packageNpmDir, 'node_modules', installedModule)); + } + }); + + // shrinkwrap + self._shrinkwrap(packageNpmDir); + process.stdout.write("DONE\n"); + } + + this._updateComplete(packageNpmDir); + }, + + // Prepare a package .npm directory for installing new packages and/or + // new versions of packages: + // + // - Copies meteor-npm-shrinkwrap.json into npm-shrinkwrap.json + // while removing the parts related to packages being upgraded. + // + // - Creates a package.json file corresponding to the packages that + // - are to be installed (needed for both `npm install` and `npm shrinkwrap`) + // + // XXX doesn't support uninstalling packages + // + // @param npmDependencies {Object} dependencies that should be installed, + // eg {tar: '0.1.6', gcd: '0.0.0'} + // @returns {Object} dependencies to update, eg {tar: '0.1.6'} + _prepareForUpdate: function(packageNpmDir, npmDependencies) { + // + // construct package.json + // + var packageJsonContents = JSON.stringify({ + // name and version are unimportant but required for `npm install` + name: 'packages', + version: '0.0.0', + dependencies: npmDependencies + }); + var packageJsonPath = path.join(packageNpmDir, 'package.json'); + // this file will be removed in `_updateComplete`, but it's also .gitignored + fs.writeFileSync(packageJsonPath, packageJsonContents); + + + // + // meteor-npm-shrinkwrap.json -> npm-shrinkwrap.json, compute dependenciesToUpdate + // + var meteorShrinkwrapJsonPath = path.join(packageNpmDir, 'meteor-npm-shrinkwrap.json'); + if (fs.existsSync(meteorShrinkwrapJsonPath)) { + var shrinkwrap = JSON.parse(fs.readFileSync(meteorShrinkwrapJsonPath)); + dependenciesToUpdate = {}; + _.each(npmDependencies, function(version, name) { + if (!shrinkwrap.dependencies[name] || shrinkwrap.dependencies[name].version !== version) { + dependenciesToUpdate[name] = version; + delete shrinkwrap.dependencies[name]; + } + }); + + // this file will be removed in `_shrinkwrap` or `_updateComplete`, but it's also .gitignored + fs.writeFileSync(path.join(packageNpmDir, 'npm-shrinkwrap.json'), + JSON.stringify(shrinkwrap)); + } else { + dependenciesToUpdate = npmDependencies; + } + + return dependenciesToUpdate; + }, + + _updateComplete: function(packageNpmDir) { + var npmShrinkwrapJsonPath = path.join(packageNpmDir, 'npm-shrinkwrap.json'); + if (fs.existsSync(npmShrinkwrapJsonPath)) // if we didn't update any dependencies + fs.unlinkSync(npmShrinkwrapJsonPath); + + var packageJsonPath = path.join(packageNpmDir, 'package.json'); + fs.unlinkSync(packageJsonPath); + }, + + _execSync: function(cmd, opts) { + return Future.wrap(function(cb) { + exec(cmd, opts, function (err, stdout, stderr) { + var result = {stdout: stdout, stderr: stderr}; + if (err) + _.extend(err, result); + cb(err, result); + }); + })().wait(); + }, + + _installNpmModule: function(name, version, dir) { + // We don't use npm.commands.install since we couldn't + // figure out how to silence all output (specifically the + // installed tree which is printed out with `console.log`) + this._execSync(path.join(files.get_dev_bundle(), "bin", "npm") + " install " + + name + "@" + version, + {cwd: dir}); + }, + + _installFromShrinkwrap: function(dir) { + if (!fs.existsSync(path.join(dir, "npm-shrinkwrap.json"))) + throw new Error("Can't call `npm install` without a npm-shrinkwrap.json file present"); + // `npm install`, which reads npm-shrinkwrap.json + this._execSync(path.join(files.get_dev_bundle(), "bin", "npm") + " install", + {cwd: dir}); + }, + + // shrinkwraps into meteor-npm-shrinkwrap.json + _shrinkwrap: function(dir) { + // We don't use npm.commands.shrinkwrap for two reasons: + // 1. As far as we could tell there's no way to completely silence the output + // (the `silent` flag isn't piped in to the call to npm.commands.ls) + // 2. In various (non-deterministic?) cases we observed the + // npm-shrinkwrap.json file not being updated + this._execSync(path.join(files.get_dev_bundle(), "bin", "npm") + " shrinkwrap", + {cwd: dir}); + + var meteorShrinkwrapJsonPath = path.join(dir, 'meteor-npm-shrinkwrap.json'); + var npmShrinkwrapJsonPath = path.join(dir, 'npm-shrinkwrap.json'); + fs.renameSync(npmShrinkwrapJsonPath, meteorShrinkwrapJsonPath); + }, + + _depsToString: function(dependenciesToUpdate) { + return _.map(dependenciesToUpdate, function(version, name) { + return name + '@' + version; + }).join(', '); + } +}; + diff --git a/lib/packages.js b/lib/packages.js index 2713442b02..6d762ef265 100644 --- a/lib/packages.js +++ b/lib/packages.js @@ -218,6 +218,8 @@ _.extend(Package.prototype, { var code = fs.readFileSync(fullpath).toString(); // \n is necessary in case final line is a //-comment var wrapped = "(function(Package,require){" + code + "\n})"; + // See #runInThisContext + // // XXX it'd be nice to runInNewContext so that the package // setup code can't mess with our globals, but objects that // come out of runInNewContext have bizarro antimatter diff --git a/lib/tests/test_bundler.js b/lib/tests/test_bundler.js index d30a4b1dfb..84fa60a497 100644 --- a/lib/tests/test_bundler.js +++ b/lib/tests/test_bundler.js @@ -1,158 +1,35 @@ -var path = require('path'); -var assert = require('assert'); -var fs = require('fs'); -var files = require(path.join(__dirname, '..', 'files.js')); -var bundler = require(path.join(__dirname, '..', 'bundler.js')); -var inFiber = require(path.join(__dirname, '..', 'fiber-helpers.js')).inFiber; - -/// -/// SETUP -/// - -// print stack track and exit with error code if an assertion fails -process.on('uncaughtException', function (err) { - console.log(err.stack); - console.log(); - console.log('Bundle can be found at ' + lastTmpDir); - process.exit(1); -}); - -/// -/// UTILITIES -/// +/*global*/ path = require('path'); +/*global*/ fs = require('fs'); +/*global*/ files = require(path.join(__dirname, '..', 'files.js')); +/*global*/ bundler = require(path.join(__dirname, '..', 'bundler.js')); +/*global*/ inFiber = require(path.join(__dirname, '..', 'fiber-helpers.js')).inFiber; +/*global*/ _ = require('underscore'); +/*global*/ assert = require('assert'); var tmpBaseDir = files.mkdtemp('test_bundler'); var tmpCounter = 1; var lastTmpDir; -var tmpDir = function () { +/*global*/ tmpDir = function () { lastTmpDir = path.join(tmpBaseDir, "" + (tmpCounter++) /* path.join likes string, not numbers */); files.mkdir_p(lastTmpDir); return lastTmpDir; }; -/// -/// TEST APPS -/// +Fiber(function () { + try { + /// RUN TESTS +//xcxc require(path.join(__dirname, 'test_bundler_options.js')); + require(path.join(__dirname, 'test_bundler_npm.js')); -// an empty app with a .meteor/version file whose contents are "0.1" -var versionedAppDir = path.join(__dirname, 'empty-versioned-app'); -// an empty app with no .meteor/version file -var unversionedAppDir = path.join(__dirname, 'empty-unversioned-app'); + /// SUCCESS + files.rm_recursive(tmpBaseDir); + } catch (err) { + // print stack track and exit with error code if an assertion fails + console.log(err.stack); + console.log(); + console.log('Bundle can be found at ' + lastTmpDir); + process.exit(1); + }; +}).run(); -/// -/// TESTS -/// -// versioned app, nodeModules: 'skip' -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - // no node_modules directory - assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules"))); - // verify that contents are minified - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); - assert(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml)); - assert(!(/src=\"\/packages/.test(appHtml))); -})); - -// versioned app, nodeModules: 'skip', noMinify -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - // verify that contents are not minified - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); - assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml))); - assert(/src=\"\/packages\/meteor/.test(appHtml)); - assert(/src=\"\/packages\/deps/.test(appHtml)); - // verify that tests aren't included - assert(!(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml))); -})); - -// versioned app, nodeModules: 'skip', noMinify, testPackages: ['meteor'] -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle( - versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true, testPackages: ['meteor']}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - // verify that tests for the meteor package are included - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); - assert(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml)); -})); - -// versioned app, nodeModules: 'copy' -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'copy'}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - // node_modules directory exists and is not a symlink - assert(!fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); - // node_modules contains fibers - assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); -})); - -// versioned app, nodeModules: 'symlink' -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'symlink'}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - // node_modules directory exists and is a symlink - assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); - // node_modules contains fibers - assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); -})); - -// unversioned app, no options -- should look in packages/ -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); - - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - - // XXX actually test that we get the contents out of packages/? - - // XXX someone test that if this is an installed version (that doesn't have packages/) - // then this fails with an appropriate error ("Can't find package 'meteor'") -})); - -// unversioned app, using `versionOverride` -assert.doesNotThrow(inFiber(function () { - var tmpOutputDir = tmpDir(); - var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.0.1', nodeModulesMode: 'skip'}); - assert.strictEqual(errors, undefined); - - // sanity check -- main.js has expected contents. - assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require(require('path').join(__dirname, 'server', 'server.js'));"); - - // XXX actually test that we get the contents out of the cache? -})); - -/// SUCCESS - -files.rm_recursive(tmpBaseDir); \ No newline at end of file diff --git a/lib/tests/test_bundler_npm.js b/lib/tests/test_bundler_npm.js new file mode 100644 index 0000000000..8d2ef81ea1 --- /dev/null +++ b/lib/tests/test_bundler_npm.js @@ -0,0 +1,190 @@ +var meteorNpm = require(path.join(__dirname, '..', 'meteor_npm.js')); + +/// +/// TEST PACKAGE DIR +/// +var tmpPackageDirContainer = tmpDir(); +var testPackageDir = path.join(tmpPackageDirContainer, 'test-package'); + +fs.mkdirSync(testPackageDir); +fs.writeFileSync(path.join(testPackageDir, 'package.js'), + "Package.describe({summary: 'a package that uses npm modules'});\n" + + "\n" + + "Package.on_use(function(api, where) { api.useNpm({gcd: '0.0.0'}); });"); +process.env.PACKAGE_DIRS = tmpPackageDirContainer; + + +/// +/// TEST APP USING TEST PACKAGE DIR +/// +var appWithPackageDir = path.join(__dirname, 'unversioned-app-with-package'); + +/// +/// HELPERS +/// + +var _assertCorrectPackageNpmDir = function(deps, nodeModulesShouldHaveBeenDeleted) { + // test-package/.npm was generated + + // sort of a weird way to do it, but i don't want to have to look up all subdependencies + // to write these tests, so just transplant that information + var actualMeteorNpmShrinkwrapDependencies = JSON.parse(fs.readFileSync(path.join(testPackageDir, ".npm", "meteor-npm-shrinkwrap.json"), 'utf8')).dependencies; + var expectedMeteorNpmShrinkwrapDependencies = _.object(_.map(deps, function(version, name) { + var val = {version: version}; + if (actualMeteorNpmShrinkwrapDependencies[name].dependencies) + val.dependencies = actualMeteorNpmShrinkwrapDependencies[name].dependencies; + return [name, val]; + })); + + assert.equal( + fs.readFileSync(path.join(testPackageDir, ".npm", "meteor-npm-shrinkwrap.json"), 'utf8'), + JSON.stringify({ + name: "packages", + version: "0.0.0", + dependencies: expectedMeteorNpmShrinkwrapDependencies}, null, /*indentation, the way npm uses it*/2) + '\n'); + + + if (nodeModulesShouldHaveBeenDeleted) { + assert(!fs.exists(path.join(testPackageDir, '.npm', 'node_modules'))); + } else { + // verify the contents of the `node_modules` dir + _.each(deps, function(version, name) { + // presumably if this file is here we have correctly installed the package + assert(fs.existsSync(path.join(testPackageDir, '.npm', 'node_modules', name, 'LICENSE'))); + + assert.equal(JSON.parse( + fs.readFileSync( + path.join(testPackageDir, ".npm", "node_modules", name, "package.json"), + 'utf8')).version, + version); + }); + } +}; + +var _assertCorrectBundleNpmContents = function(bundleDir, deps) { + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(bundleDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + + var bundledPackageNodeModulesDir = path.join(bundleDir, 'app', 'packages', 'test-package', 'node_modules'); + + // bundle actually has the npm modules + _.each(deps, function(version, name) { + // presumably if this file is here we have correctly installed the package + assert(fs.existsSync(path.join(bundledPackageNodeModulesDir, name, 'LICENSE'))); + + assert.equal(JSON.parse( + fs.readFileSync(path.join(bundledPackageNodeModulesDir, name, 'package.json'), 'utf8')) + .version, + version); + }); +}; + +/// +/// TESTS +/// + +console.log("app that uses gcd - clean run"); +assert.doesNotThrow(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + _assertCorrectPackageNpmDir({gcd: '0.0.0'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0'}); +}); + +console.log("app that uses gcd - no changes, running again"); +assert.doesNotThrow(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + _assertCorrectPackageNpmDir({gcd: '0.0.0'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0'}); +}); + +console.log("app that uses gcd - simulate failure, or as would be in a 3rd party repository (no .npm/node_modules)"); +assert.doesNotThrow(function () { + var tmpOutputDir = tmpDir(); + meteorNpm._execSync("rm -rf .npm/node_modules", {cwd: testPackageDir}); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + _assertCorrectPackageNpmDir({gcd: '0.0.0'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0'}); +}); + + +console.log("app that uses gcd - add mime and semver"); +assert.doesNotThrow(function () { + fs.writeFileSync(path.join(testPackageDir, 'package.js'), + "Package.describe({summary: 'a package that uses npm modules'});\n" + + "\n" + + "Package.on_use(function(api, where) {\n" + + " api.useNpm({gcd: '0.0.0', mime: '1.2.7', semver: '1.1.0'});\n" + + "});"); + + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + _assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.7', semver: '1.1.0'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0', mime: '1.2.7', semver: '1.1.0'}); +}); + +console.log("app that uses gcd - upgrade mime, remove semver"); +assert.doesNotThrow(function () { + fs.writeFileSync(path.join(testPackageDir, 'package.js'), + "Package.describe({summary: 'a package that uses npm modules'});\n" + + "\n" + + "Package.on_use(function(api, where) {\n" + + " api.useNpm({gcd: '0.0.0', mime: '1.2.8'});\n" + + "});"); + + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + _assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.8'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0', mime: '1.2.8'}); +}); + +console.log("app that uses gcd - try downgrading mime to non-existant version"); +assert.doesNotThrow(function () { + fs.writeFileSync(path.join(testPackageDir, 'package.js'), + "Package.describe({summary: 'a package that uses npm modules'});\n" + + "\n" + + "Package.on_use(function(api, where) {\n" + + " api.useNpm({gcd: '0.0.0', mime: '0.1.2'});\n" + + "});"); + + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors.length, 1); + assert(/version not found/.test(errors[0])); + + // make sure we delete `node_modules` in case some packages got + // installed and others haven't. the next test will make sure we + // recover from this state. + _assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.8'}/*shouldn't've changed*/, + /*nodeModulesShouldHaveBeenDeleted*/true); +}); + +console.log("app that uses gcd - downgrade mime to an existant version"); +assert.doesNotThrow(function () { + fs.writeFileSync(path.join(testPackageDir, 'package.js'), + "Package.describe({summary: 'a package that uses npm modules'});\n" + + "\n" + + "Package.on_use(function(api, where) {\n" + + " api.useNpm({gcd: '0.0.0', mime: '1.2.7'});\n" + + "});"); + + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined, errors && errors[0]); + + _assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.7'}); + _assertCorrectBundleNpmContents(tmpOutputDir, {gcd: '0.0.0', mime: '1.2.7'}); +}); + + +/// +/// SUCCESS +/// +delete process.env.PACKAGE_DIRS; \ No newline at end of file diff --git a/lib/tests/test_bundler_options.js b/lib/tests/test_bundler_options.js new file mode 100644 index 0000000000..0b1b88b1f8 --- /dev/null +++ b/lib/tests/test_bundler_options.js @@ -0,0 +1,123 @@ +/// +/// TEST APPS +/// + +// an empty app with a .meteor/version file whose contents are "0.1" +var versionedAppDir = path.join(__dirname, 'empty-versioned-app'); +// an empty app with no .meteor/version file +var unversionedAppDir = path.join(__dirname, 'empty-unversioned-app'); + + +/// +/// TESTS +/// + +console.log("versioned app, nodeModules: 'skip'"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // no node_modules directory + assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules"))); + // verify that contents are minified + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml)); + assert(!(/src=\"\/packages/.test(appHtml))); +})); + +console.log("versioned app, nodeModules: 'skip', noMinify"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // verify that contents are not minified + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml))); + assert(/src=\"\/packages\/meteor/.test(appHtml)); + assert(/src=\"\/packages\/deps/.test(appHtml)); + // verify that tests aren't included + assert(!(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml))); +})); + +console.log("versioned app, nodeModules: 'skip', noMinify, testPackages: ['meteor']"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle( + versionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip', noMinify: true, testPackages: ['meteor']}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // verify that tests for the meteor package are included + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + assert(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml)); +})); + +console.log("versioned app, nodeModules: 'copy'"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'copy'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // node_modules directory exists and is not a symlink + assert(!fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + // node_modules contains fibers + assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); +})); + +console.log("versioned app, nodeModules: 'symlink'"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(versionedAppDir, tmpOutputDir, {nodeModulesMode: 'symlink'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + // node_modules directory exists and is a symlink + assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + // node_modules contains fibers + assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); +})); + +console.log("unversioned app, no options -- should look in packages/"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {nodeModulesMode: 'skip'}); + + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + + // XXX actually test that we get the contents out of packages/? + + // XXX someone test that if this is an installed version (that doesn't have packages/) + // then this fails with an appropriate error ("Can't find package 'meteor'") +})); + +console.log("unversioned app, using `versionOverride`"); +assert.doesNotThrow(inFiber(function () { + var tmpOutputDir = tmpDir(); + var errors = bundler.bundle(unversionedAppDir, tmpOutputDir, {versionOverride: '0.0.1', nodeModulesMode: 'skip'}); + assert.strictEqual(errors, undefined); + + // sanity check -- main.js has expected contents. + assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), + "require(require('path').join(__dirname, 'server', 'server.js'));"); + + // XXX actually test that we get the contents out of the cache? +})); diff --git a/lib/tests/unversioned-app-with-package/.meteor/.gitignore b/lib/tests/unversioned-app-with-package/.meteor/.gitignore new file mode 100644 index 0000000000..4083037423 --- /dev/null +++ b/lib/tests/unversioned-app-with-package/.meteor/.gitignore @@ -0,0 +1 @@ +local diff --git a/lib/tests/unversioned-app-with-package/.meteor/packages b/lib/tests/unversioned-app-with-package/.meteor/packages new file mode 100644 index 0000000000..8eca235559 --- /dev/null +++ b/lib/tests/unversioned-app-with-package/.meteor/packages @@ -0,0 +1 @@ +test-package \ No newline at end of file diff --git a/packages/templating/html_scanner.js b/packages/templating/html_scanner.js index d5ac1954ae..0256339763 100644 --- a/packages/templating/html_scanner.js +++ b/packages/templating/html_scanner.js @@ -1,5 +1,5 @@ -var html_scanner = { +/*global*/ html_scanner = { // Scan a template file for , , and + +