diff --git a/tools/bundler.js b/tools/bundler.js index 8d175cfc5f..c08f9b4ba7 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1,29 +1,109 @@ -// Bundle contents: -// main.js [run to start the server] -// /static [served by node for now] -// /static_cacheable [cache-forever files, served by node for now] -// /server [XXX split out into a package] -// server.js, .... [contents of tools/server] -// node_modules [for now, contents of (dev_bundle)/lib/node_modules] -// /app.html -// /app [user code] -// /app.json: [data for server.js] -// - load [list of files to load, relative to root, presumably under /app] -// - manifest [list of resources in load order, each consists of an object]: -// { -// "path": relative path of file in the bundle, normalized to use forward slashes -// "where": "client", "internal" [could also be "server" in future] -// "type": "js", "css", or "static" -// "cacheable": (client) boolean, is it safe to ask the browser to cache this file -// "url": (client) relative url to download the resource, includes cache -// busting param if used -// "size": size in bytes -// "hash": sha1 hash of the contents -// } +// == Site Archive (*.star) file layout (subject to rapid change) == // -// The application launcher is expected to execute /main.js with node, setting -// various environment variables (such as PORT and MONGO_URL). The enclosed node -// application is expected to do the rest, including serving /static. +// /star.json +// +// - builtBy: human readable banner (eg, "Meteor 0.6.0") +// +// - programs: array of programs in the star, each an object: +// - name: short name for program +// - arch: architecture that this program targets. Currently it is +// "client" or "server" but in the future this will change +// to something like "browser.w3c" or "darwin.x86_64". +// - path: directory (relative to star.json) containing this program +// +// /README: human readable instructions +// +// /main.js: script that can be run in node.js to start the site +// running in standalone mode (after setting appropriate environment +// variables as documented in README) +// +// /server/.bundle_version.txt: contains the dev_bundle version that +// legacy (read: current) Galaxy version read in order to set +// NODE_PATH to point to arch-specific builds of binary node modules +// (primarily this is for node-fibers) +// +// +// Conventionally programs will be located at /programs/, but +// really the build tool can lay out the star however it wants. +// +// +// == Format of a program when arch is "client" == +// +// Standard: +// +// /app.json +// +// - page: path to the template for the HTML to serve when a browser +// loads a page that is part of the application. In the file +// ##HTML_ATTRIBUTES## and ##RUNTIME_CONFIG## will be replaced with +// appropriate values at runtime. +// +// - manifest: array of resources to serve with HTTP, each an object: +// - path: path of file relative to app.json +// - where: "client" +// - type: "js", "css", or "static" +// - cacheable: is it safe to ask the browser to cache this file (boolean) +// - url: relative url to download the resource, includes cache busting +// parameter when used +// - size: size of file in bytes +// - hash: sha1 hash of the file contents +// Additionally there will be an entry with where equal to +// "internal", path equal to page (above), and hash equal to the +// sha1 of page (before replacements.) Currently this is used to +// trigger HTML5 appcache reloads at the right time (if the +// 'appcache' package is being used.) +// +// - static: a path, relative to app.json, to a directory. If the +// server is too dumb to read 'manifest', it can just serve all of +// the files in this directory (with a relatively short cache +// expiry time.) +// XXX do not use this. It will go away soon. +// +// - static_cacheable: just like 'static' but resources that can be +// cached aggressively (cacheable: true in the manifest) +// XXX do not use this. It will go away soon. +// +// Convention: +// +// page is 'app.html', static is 'static', and staticCacheable is +// 'static_cacheable'. +// +// +// == Format of a program when arch is "server" == +// +// Standard: +// +// /app.json: +// +// - load: array with each item describing a JS file to load at startup: +// - path: path of file, relative to app.json +// - node_modules: if Npm.require is called from this file, this is +// the path (relative to app.json) of the directory that should +// be search for npm modules +// +// - client: the client program that should be served up by HTTP, +// expressed as a path (relative to app.json) to the *client's* +// app.json. +// +// - config: additional framework-specific configuration. currently: +// - meteorRelease: the value to use for Meteor.release, if any +// +// /server.js: script to run inside node.js to start the program +// XXX Subject to change! This will likely go away in favor of a +// shell script (allowing us to represent types of programs that +// don't use or depend on node) +// +// Convention: +// +// /app/*: source code of the (server part of the) app +// /packages/foo.js: the (linked) source code for package foo +// /package-tests/foo.js: the (linked) source code for foo's tests +// /npm/foo: node_modules for package foo. may be symlinked if +// developing locally. +// +// /node_modules: node_modules needed for server.js. omitted if +// deploying (see .bundle_version.txt above), copied if bundling, +// symlinked if developing locally. var path = require('path'); var files = require(path.join(__dirname, 'files.js')); @@ -50,6 +130,13 @@ var sha1 = exports.sha1 = function (contents) { return hash.digest('hex'); }; +// http://davidshariff.com/blog/javascript-inheritance-patterns/ +var inherits = function (child, parent) { + var tmp = function () {}; + tmp.prototype = parent.prototype; + child.prototype = new tmp; + child.prototype.constructor = child; +}; /////////////////////////////////////////////////////////////////////////////// // Slice @@ -72,97 +159,189 @@ var Slice = function (pkg, role, where) { }; /////////////////////////////////////////////////////////////////////////////// -// Bundle +// File /////////////////////////////////////////////////////////////////////////////// -// options to include: -// -// - releaseStamp: the Meteor release name to write into the bundle metadata -// - library: tells you how to find packages -var Bundle = function (options) { +// Allowed options: +// - data: contents of the file as a Buffer +// - cacheable +var File = function (options) { var self = this; + if (options.data && ! (options.data instanceof Buffer)) { + throw new Error('File contents must be provided as a Buffer'); + } + + // The absolute path in the filesystem from which we loaded (or will + // load) this file (null if the file does not correspond to one on + // disk.) + self.sourcePath = null; + + // Where this file is intended to reside within the target's + // filesystem. + self.targetPath = null; + + // The URL at which this file is intended to be served, relative to + // the base URL at which the target is being served (ignored if this + // file is not intended to be served over HTTP.) + self.url = null + + // Is this file guaranteed to never change, so that we can let it be + // cached forever? Only makes sense of self.url is set. + self.cacheable = options.cacheable || false; + + // The node_modules directory that Npm.require() should search when + // called from inside this file, given as a path in the target's + // filesystem. Only works in the "server" architecture. + self.nodeModulesTargetPath = null; + + self._contents = options.data || null; // contents, if known, as a Buffer + self._hash = null; // hash, if known, as a hex string +}; + +_.extend(File.prototype, { + hash: function () { + var self = this; + if (! self._hash) + self._hash = sha1(self.contents()); + return self._hash; + }, + + // Omit encoding to get a buffer, or provide something like 'utf8' + // to get a string + contents: function (encoding) { + var self = this; + if (! self._contents) { + if (! self.sourcePath) + throw new Error("Have neither contents nor sourcePath for file"); + } + + return encoding ? self._contents : self._contents.toString(encoding); + }, + + size: function () { + var self = this; + return self.contents().length; + }, + + // Set the URL of this file to "/". suffix will + // typically be used to pick a reasonable extension. Also set + // cacheable to true, since the file's name is now derived from its + // contents. + setUrlToHash: function (suffix) { + var self = this; + self.url = "/" + self.hash() + suffix; + self.cacheable = true; + }, + + // Append "?" to the URL and mark the file as cacheable. + addCacheBuster: function () { + var self = this; + if (! self.url) + throw new Error("File must have a URL"); + if (self.cacheable) + return; // eg, already got setUrlToHash + if (/\?/.test(self.url)) + throw new Error("URL already has a query string"); + self.url += "?" + self.hash(); + self.cacheable = true; + }, + + // Make `bundle` depend on this file for the purpose of automatic + // reload. + addDependencyToBundle: function (bundle) { + var self = this; + if (! self.sourcePath) + throw new Error("Don't know sourcePath"); + self.bundle._addDependency(self.sourcePath, self.hash()); + }, + + // Given a relative path like 'a/b/c' (where '/' is this system's + // path component separator), produce a URL that always starts with + // a forward slash and that uses a literal forward slash as the + // component separator. + setUrlFromRelPath: function (relPath) { + var self = this + var url = relPath.split(path.sep).join('/'); + + if (url.charAt(0) !== '/') + url = '/' + url; + + self.url = url; + } +}); + +/////////////////////////////////////////////////////////////////////////////// +// Target +/////////////////////////////////////////////////////////////////////////////// + +var Target = function (name, bundle, options) { + var self = this; + + // Bundle that contains this target. + self.bundle = bundle; + + // A name for this target. + self.name = name; + + // Path of this target in the bundle, relative to the root of the bundle. + self.pathInBundle = path.join('programs', self.name); + + // Should be "client" or "server" for now, but soon we will get rid + // of that and instead have something more like "browser.w3c" or + // "nodejs.linux.i686". + self.arch = options.arch; + // All of the Slices that are to go into this bundle, in the order // that they are to be loaded. self.slices = []; - // meteor release version - self.releaseStamp = options.releaseStamp; - - // search configuration for package.get() - self.library = options.library; - - // map from environment, to list of filenames - self.js = {client: [], server: []}; - - // list of filenames - self.css = []; - - // images and other static files added from packages - // map from environment, to list of filenames - self.static = {client: [], server: []}; - - // Map from environment, to path name (server relative), to contents - // of file as buffer. - self.files = {client: {}, clientCacheable: {}, server: {}}; - - // See description of the manifest at the top. - // Note that in contrast to self.js etc., the manifest only includes - // files which are in the final bundler output: for example, if code - // is minified, the manifest includes the minify output file but not - // the individual input files that were combined. - self.manifest = []; - - // these directories are copied (cp -r) or symlinked into the - // bundle. maps target path (server relative) to source directory on - // disk - self.nodeModulesDirs = {}; - - // list of segments of additional HTML for / - self.head = []; - self.body = []; - - // Files and paths used by the bundle, in the format returned by - // bundle().dependencyInfo. - self.dependencyInfo = null; + // JavaScript files. List of File. They will be loaded at startup in + // the order given. + self.js = []; }; -_.extend(Bundle.prototype, { +_.extend(Target.prototype, { // Determine the packages to load, create Slices for // them, put them in load order, save in slices. // - // contents is a map from role ('use' or 'test') to environment - // ('client' or 'server') to an array of either package names or - // actual Package objects. + // contents is a map from role ('use' or 'test') to an array of + // either package names or actual Package objects. determineLoadOrder: function (contents) { var self = this; // Package slices being used. Map from a role string (eg, "use" or - // "test") to "client" or "server" to a package id to a Slice. - var sliceIndex = {use: {client: {}, server: {}}, - test: {client: {}, server: {}}}; + // "test") to a package id to a Slice. + var sliceIndex = {use: {}, test: {}}; var slicesUnordered = []; + var get = function (packageOrPackageName) { + var pkg = self.bundle.library.get(packageOrPackageName); + if (! pkg) { + console.error("Package not found: " + packageOrPackageName); + process.exit(1); + } + return pkg; + }; + // Ensure that slices exist for a package and its dependencies. - var add = function (pkg, role, where) { - if (sliceIndex[role][where][pkg.id]) + var add = function (pkg, role) { + if (sliceIndex[role][pkg.id]) return; - var slice = new Slice(pkg, role, where); - sliceIndex[role][where][pkg.id] = slice; + var slice = new Slice(pkg, role, self.arch); + sliceIndex[role][pkg.id] = slice; slicesUnordered.push(slice); - _.each(pkg.uses[role][where], function (usedPkgName) { - var usedPkg = self.getPackage(usedPkgName); - add(usedPkg, "use", where); + _.each(pkg.uses[role][self.arch], function (usedPkgName) { + var usedPkg = get(usedPkgName); + add(usedPkg, "use"); }); }; // Add the provided roots and all of their dependencies. - _.each(contents, function (whereToArray, role) { - _.each(whereToArray, function (packageList, where) { - _.each(packageList, function (packageOrPackageName) { - var pkg = self.getPackage(packageOrPackageName); - add(pkg, role, where); - }); + _.each(contents, function (packageList, role) { + _.each(packageList, function (packageOrPackageName) { + var pkg = get(packageOrPackageName); + add(pkg, role); }); }); @@ -172,7 +351,7 @@ _.extend(Bundle.prototype, { // before X in the ordering. Raises an exception iff there is no // such ordering (due to circular dependency.) var id = function (slice) { - return slice.role + ":" + slice.where + ":" + slice.pkg.id; + return slice.role + ":" + slice.pkg.id; }; var done = {}; @@ -197,11 +376,11 @@ _.extend(Bundle.prototype, { if (done[id(slice)]) return; - _.each(slice.pkg.uses[slice.role][slice.where], function (usedPkgName) { + _.each(slice.pkg.uses[slice.role][self.arch], function (usedPkgName) { if (slice.pkg.name && slice.pkg.unordered[usedPkgName]) return; - var usedPkg = self.getPackage(usedPkgName); - var usedSlice = sliceIndex.use[slice.where][usedPkg.id]; + var usedPkg = get(usedPkgName); + var usedSlice = sliceIndex.use[usedPkg.id]; if (! usedSlice) throw new Error("Missing slice?"); if (onStack[id(usedSlice)]) { @@ -221,37 +400,6 @@ _.extend(Bundle.prototype, { } }, - prepNodeModules: function () { - var self = this; - var seen = {}; - _.each(self.slices, function (slice) { - // Bring npm dependencies up to date. One day this will probably - // grow into a full-fledged package build step. - if (slice.pkg.npmDependencies && ! seen[slice.pkg.id]) { - seen[slice.pkg.id] = true; - slice.pkg.installNpmDependencies(); - self.bundleNodeModules(slice.pkg); - } - }); - }, - - getPackage: function (packageOrPackageName) { - var self = this; - var pkg = self.library.get(packageOrPackageName); - if (! pkg) { - console.error("Package not found: " + packageOrPackageName); - process.exit(1); - } - return pkg; - }, - - // map a package's generated node_modules directory to the package - // directory within the bundle - bundleNodeModules: function (pkg) { - var nodeModulesPath = path.join(pkg.npmDir(), 'node_modules'); - this.nodeModulesDirs[pkg.name] = nodeModulesPath; - }, - // Sort the packages in dependency order, then, package by package, // write their resources into the bundle (which includes running the // JavaScript linker.) @@ -265,7 +413,7 @@ _.extend(Bundle.prototype, { // ** linker. slice.pkg.ensureCompiled(); - var resources = _.clone(slice.pkg.resources[slice.role][slice.where]); + var resources = _.clone(slice.pkg.resources[slice.role][self.arch]); var isApp = ! slice.pkg.name; // Compute imports by merging the exports of all of the @@ -277,12 +425,12 @@ _.extend(Bundle.prototype, { // in JavaScript.) Note that in the case of conflicting // symbols, later packages get precedence. var imports = {}; // map from symbol to supplying package name - _.each(_.values(slice.pkg.uses[slice.role][slice.where]), function (otherPkgName){ - var otherPkg = self.getPackage(otherPkgName); + _.each(_.values(slice.pkg.uses[slice.role][self.arch]), function (otherPkgName){ + var otherPkg = self.bundle.library.get(otherPkgName); if (otherPkg.name && ! slice.pkg.unordered[otherPkg.name]) { // make sure otherPkg.exports is valid otherPkg.ensureCompiled(); - _.each(otherPkg.exports.use[slice.where], function (symbol) { + _.each(otherPkg.exports.use[self.arch], function (symbol) { imports[symbol] = otherPkg.name; }); } @@ -292,8 +440,8 @@ _.extend(Bundle.prototype, { var files = linker.link({ imports: imports, useGlobalNamespace: isApp, - prelinkFiles: slice.pkg.prelinkFiles[slice.role][slice.where], - boundary: slice.pkg.boundary[slice.role][slice.where] + prelinkFiles: slice.pkg.prelinkFiles[slice.role][self.arch], + boundary: slice.pkg.boundary[slice.role][self.arch] }); // Add each output as a resource @@ -305,13 +453,10 @@ _.extend(Bundle.prototype, { }); }); - // ** Emit the resources + // Emit the resources _.each(resources, function (resource) { - if (resource.type === "js") { - self.files[slice.where][resource.servePath] = resource.data; - self.js[slice.where].push(resource.servePath); - } else if (resource.type === "css") { - if (slice.where !== "client") + if (_.contains(["js", "css", "static"], resource.type)) { + if (resource.type === "css" && self.arch !== "client") // XXX might be nice to throw an error here, but then we'd // have to make it so that packages.js ignores css files // that appear in the server directories in an app tree @@ -320,119 +465,396 @@ _.extend(Bundle.prototype, { // meteor.js? return; - self.files[slice.where][resource.servePath] = resource.data; - self.css.push(resource.servePath); - } else if (resource.type === "static") { - self.files[slice.where][resource.servePath] = resource.data; - self.static[slice.where].push(resource.servePath); - } else if (resource.type === "head" || resource.type === "body") { - if (slice.where !== "client") + var f = new File({ + data: resource.data, + cacheable: false + }); + + if (self.arch === "client") + f.setUrlFromRelPath(resource.servePath); + else { + // XXX hack + if (resource.servePath.match(/^\/packages\//) || + resource.servePath.match(/^\/package-tests\//)) + f.targetPath = resource.servePath; + else + f.targetPath = path.join('/app', resource.servePath); + } + + if (self.arch === "server" && resource.type === "js" && ! isApp) + f.nodeModulesTargetPath = path.join('/npm', slice.pkg.name); + + self[resource.type].push(f); + return; + } + + if (_.contains(["head", "body"], resource.type)) { + if (self.arch !== "client") throw new Error("HTML segments can only go to the client"); self[resource.type].push(resource.data); - } else { - throw new Error("Unknown type " + resource.type); + return; + } + + throw new Error("Unknown type " + resource.type); + }); + + // Depend on the source files that produced these + // resources. (Since the dependencyInfo.directories should be + // disjoint, it should be OK to merge them this way.) + _.extend(self.bundle.dependencyInfo.files, + slice.pkg.dependencyInfo.files); + _.extend(self.bundle.dependencyInfo.directories, + slice.pkg.dependencyInfo.directories); + }); + }, + + // Minify the JS in this target + minifyJs: function () { + var self = this; + + var allJs = _.map(self.js, function (file) { + return file.contents('utf8'); + }).join('\n;\n'); + + allJs = uglify.minify(allJs, { + fromString: true, + compress: {drop_debugger: false} + }).code; + + self.js = [new File({ data: new Buffer(allJs) })]; + self.js[0].setUrlToHash(".js"); + }, + + // For each resource of the given type, make it cacheable by adding + // a query string to the URL based on its hash. + addCacheBusters: function (type) { + var self = this; + _.each(self[type], function (file) { + file.addCacheBuster(); + }); + } +}); + + +//////////////////// ClientTarget //////////////////// + +var ClientTarget = function (name, bundle, options) { + var self = this; + Target.apply(this, arguments); + + // CSS files. List of File. Applicable only on 'client' + // architecture. They will be loaded at page load in the order + // given. + self.css = []; + + // Static assets to serve with HTTP. List of File. Applicable only + // on 'client' architecture. + self.static = []; + + // List of segments of additional HTML for /. Only for + // "client" arch. + self.head = []; + self.body = []; +}; + +inherits(ClientTarget, Target); + +_.extend(ClientTarget.prototype, { + // Minify the JS in this target + minifyCss: function () { + var self = this; + + var allCss = _.map(self.css, function (file) { + return file.contents('utf8'); + }).join('\n'); + + allCss = cleanCSS.process(allCss); + + self.css = [new File({ data: new Buffer(allCss) })]; + self.css[0].setUrlToHash(".css"); + }, + + // Add all of the files in a directory `rootDir` (and its + // subdirectories) as static assets. `rootDir` should be an absolute + // path. Only makes sense on clients. If provided, exclude is an + // array of filename regexps to exclude. If provided, assetPath is a + // prefix to use when computing the path for each file in the + // client's asset tree. + addAssetDir: function (rootDir, exclude, assetPathPrefix) { + var self = this; + exclude = exclude || []; + + if (self.arch !== "client") + throw new Error("Only clients can have assets"); + + self.bundle.dependencyInfo.directories[dir] = { + include: [/.?/], + exclude: exclude + }; + + var walk = function (dir, assetPath) { + _.each(fs.readdirSync(dir), function (item) { + // Skip excluded files + var matchesAnExclude = _.any(exclude, function (pattern) { + return item.match(pattern); + }); + if (matchesAnExclude) + return; + + var absPath = path.resolve(dir, item); + assetPath = path.join(dir, item); + if (fs.statSync(absPath).isDirectory()) { + walk(absPath, assetPath); + return; + } + + var f = new File({ sourcePath: absPath }); + f.setUrlFromRelPath(assetPath); + f.addDependencyToBundle(self.bundle); + self.static.push(f); + }); + }; + + walk(rootDir, assetPathPrefix || ''); + }, + + assignTargetPaths: function () { + var self = this; + _.each(["js", "css", "static"], function (type) { + _.each(self[type], function (file) { + if (! file.targetPath) { + if (! file.url) + throw new Error("Client file with no URL?"); + + var parts = file.url.replace(/\?.*$/, '').split('/').slice(1); + parts.unshift(file.cacheable ? "static_cacheable" : "static"); + file.targetPath = path.sep + path.join.apply(path, parts); } }); }); }, - // Minify the bundle - minify: function () { + generateHtmlBoilerplate: function () { var self = this; - var addFile = function (type, finalCode) { - var contents = new Buffer(finalCode); - var hash = sha1(contents); - var name = '/' + hash + '.' + type; - self.files.clientCacheable[name] = contents; - self.manifest.push({ - path: 'static_cacheable' + name, - where: 'client', - type: type, - cacheable: true, - url: name, - size: contents.length, - hash: hash - }); - }; + var templatePath = path.join(__dirname, "app.html.in"); + var template = fs.readFileSync(templatePath); + self.bundle._addDependency(templatePath, sha1(template)); - /// Javascript - var codeParts = []; - _.each(self.js.client, function (jsPath) { - codeParts.push(self.files.client[jsPath].toString('utf8')); - - delete self.files.client[jsPath]; - }); - self.js.client = []; - - var combinedCode = codeParts.join('\n;\n'); - var finalCode = uglify.minify( - combinedCode, {fromString: true, compress: {drop_debugger: false}}).code; - - addFile('js', finalCode); - - /// CSS - var cssConcat = ""; - _.each(self.css, function (cssPath) { - var cssData = self.files.client[cssPath]; - cssConcat = cssConcat + "\n" + cssData.toString('utf8'); - - delete self.files.client[cssPath]; - }); - self.css = []; - - var finalCss = cleanCSS.process(cssConcat); - - addFile('css', finalCss); - }, - - _clientUrlsFor: function (type) { - var self = this; - return _.pluck( - _.filter(self.manifest, function (resource) { - return resource.where === 'client' && resource.type === type; - }), - 'url' - ); - }, - - _generateAppHtml: function () { - var self = this; - - // XXX we don't do content-based dependency watching for this file - var template = fs.readFileSync(path.join(__dirname, "app.html.in")); var f = require('handlebars').compile(template.toString()); return f({ - scripts: self._clientUrlsFor('js'), + scripts: _.pluck(self.js, 'url'), + stylesheets: _.pluck(self.css, 'url'), head_extra: self.head.join('\n'), - body_extra: self.body.join('\n'), - stylesheets: self._clientUrlsFor('css') + body_extra: self.body.join('\n') }); }, - // The extensions registered by the application package, if - // any. Kind of a hack. - _appExtensions: function () { + // Output the finished target to disk + write: function (outputPath) { var self = this; - var ret = []; + var manifest = []; + + // Resources served via HTTP + _.each(["js", "css", "static"], function (type) { + _.each(self[type], function (file) { + + if (! file.targetPath) + throw new Error("No targetPath?"); + + var writePath = path.join(outputPath, file.targetPath); + files.mkdir_p(path.dirname(writePath), 0755); + fs.writeFileSync(writePath, file.contents()); + + manifest.push({ + path: file.targetPath, + where: "client", + type: type, + cacheable: file.cacheable, + url: file.url, + size: file.size(), + hash: file.hash() + }); + }); + }); + + // HTML boilerplate (the HTML served to make the client load the + // JS and CSS files and start the app) + var htmlBoilerplate = self.generateHtmlBoilerplate(); + fs.writeFileSync(path.join(outputPath, 'app.html'), htmlBoilerplate); + manifest.push({ + path: 'app.html', + where: 'internal', + hash: sha1(htmlBoilerplate) + }); + + // Control file + var json = { + manifest: manifest, + page: 'app.html', + + // XXX the following are for use by 'legacy' (read: current) + // server.js implementations which aren't smart enough to read + // the manifest and instead want all of the resources in a + // directory together so they can just point gzippo at it. we + // should remove this and make the server work from the + // manifest. + static: 'static', + staticCacheable: 'static_cacheable' + }; + fs.writeFileSync(path.join(outputPath, 'app.json'), + JSON.stringify(json, null, 2)); + } +}); + + +//////////////////// ServerTarget //////////////////// + +var ServerTarget = function (name, bundle, options) { + var self = this; + Target.apply(this, arguments); + + // These directories are copied (cp -r) or symlinked into the + // bundle. Map from targetPath (path in the Target's filesystem) to + // sourcePath (absolute path in the local filesystem.) + self.nodeModulesDirs = {}; + + // The ClientTarget that we will serve. + self.clientTarget = options.clientTarget; +}; + +inherits(ServerTarget, Target); + +_.extend(ServerTarget.prototype, { + // Output the finished target to disk + write: function (outputPath, nodeModulesMode) { + var self = this; + + // JavaScript sources + _.each(self.js, function (file) { + if (! file.targetPath) + throw new Error("No targetPath?"); + + var writePath = path.join(outputPath, file.targetPath); + files.mkdir_p(path.dirname(writePath), 0755); + fs.writeFileSync(writePath, file.contents()); + }); + + // Server driver + var serverPath = path.join(__dirname, 'server'); + var copied = files.cp_r(serverPath, outputPath, {ignore: ignoreFiles}); + _.each(copied, function (relPath) { + self.bundle._addDependency(path.join(serverPath, relPath)); + }); + self.bundle.dependencyInfo.directories[serverPath] = { + include: [/.?/], + exclude: ignoreFiles + }; + + // Main, architecture-dependent node_modules from the dependency + // kit. This one is copied in 'meteor bundle', symlinked in + // 'meteor run', and omitted by 'meteor deploy' (Galaxy provides a + // version that's appropriate for the server architecture.) + + var devBundleNodeModules = + path.join(files.get_dev_bundle(), 'lib', 'node_modules'); + if (nodeModulesMode === "symlink") + fs.symlinkSync(devBundleNodeModules, + path.join(outputPath, 'node_modules')); + else if (nodeModulesMode === "copy") + files.cp_r(devBundleNodeModules, + path.join(outputPath, 'node_modules'), + {ignore: ignoreFiles}); + else + /* nodeModulesMode === "skip" */; + + // Extra user-defined arch-independent node_module. 'meteor + // bundle' and 'meteor deploy' copy them, and 'meteor run' + // symlinks them. (XXX Note that this doesn't work for + // arch-specific packages. They'll just break if you deploy to a + // different arch than you built on. We'll get to that Soon + // Enough!) + + // XXX we should consider supporting bundle time-only npm + // dependencies which don't need to be pushed to the server. _.each(self.slices, function (slice) { - if (! slice.pkg.name) { - var exts = slice.pkg.registeredExtensions(slice.role, slice.where); - ret = _.union(ret, exts); + if (slice.pkg.npmDependencies) { + // Make sure the right stuff is installed. This is slow and + // should move to a separate package build step. However, the + // Package object has code that will make sure we at least + // only do it once per package. + slice.pkg.installNpmDependencies(); + + // Copy the package's npm dependencies into the bundle. + var sourcePath = path.join(slice.pkg.npmDir(), 'node_modules'); + var targetPath = path.join(outputPath, 'npm', slice.pkg.name); + if (fs.existsSync(targetPath)) + // We already did this package (eg, we've used the package + // in both a "use" and a "test" role) + return; + + files.mkdir_p(path.dirname(targetPath)); + if (nodeModulesMode === "symlink") + fs.symlinkSync(sourcePath, targetPath); + else + files.cp_r(sourcePath, targetPath); } }); - return ret; - }, + // Control file + var release = + self.bundle.releaseStamp && self.bundle.releaseStamp !== "none" ? + self.bundle.releaseStamp : undefined; + var json = { + load: _.map(self.js, function (file) { + return { + path: file.targetPath, + node_modules: file.nodeModulesTargetPath || undefined + }; + }), + client: path.join(path.relative(self.pathInBundle, + self.clientTarget.pathInBundle), + 'app.json'), + config: { + meteorRelease: release + } + }; + fs.writeFileSync(path.join(outputPath, 'app.json'), + JSON.stringify(json, null, 2)); + } +}); - _addDependency: function (filePath, hash, onlyIfExists) { + +/////////////////////////////////////////////////////////////////////////////// +// Bundle +/////////////////////////////////////////////////////////////////////////////// + +// options to include: +// +// - releaseStamp: the Meteor release name to write into the bundle metadata +// - library: tells you how to find packages +var Bundle = function (options) { + var self = this; + + // meteor release version + self.releaseStamp = options.releaseStamp; + + // search configuration for package.get() + self.library = options.library; + + // all of the targets that are part of this bundle + self.targets = []; + + // Files and paths used by the bundle, in the format returned by + // bundle().dependencyInfo. + self.dependencyInfo = {files: {}, directories: {}}; +}; + +_.extend(Bundle.prototype, { + _addDependency: function (filePath, hash) { var self = this; filePath = path.resolve(filePath); - - if (onlyIfExists && ! fs.existsSync(filePath)) - return; - if (! hash) hash = sha1(fs.readFileSync(filePath)); self.dependencyInfo.files[filePath] = hash; @@ -440,20 +862,11 @@ _.extend(Bundle.prototype, { // nodeModulesMode should be "skip", "symlink", or "copy" // computes self.dependencyInfo - writeToDirectory: function (outputPath, projectDir, nodeModulesMode) { + write: function (outputPath, projectDir, nodeModulesMode) { var self = this; var appJson = {}; var isApp = files.is_app_dir(projectDir); - self.dependencyInfo = {files: {}, directories: {}}; - - if (isApp) { - self._addDependency(path.join(projectDir, '.meteor', 'packages')); - // Not sure why 'release' doesn't exist in a test run, but roll with it - self._addDependency(path.join(projectDir, '.meteor', 'release'), null, - true); - } - // --- Set up build area --- // foo/bar => foo/.build.bar @@ -465,165 +878,28 @@ _.extend(Bundle.prototype, { files.rm_recursive(buildPath); files.mkdir_p(buildPath, 0755); - // --- Core runner code --- - - var serverPath = path.join(__dirname, 'server'); - var copied = files.cp_r(serverPath, path.join(buildPath, 'server'), - {ignore: ignoreFiles}); - _.each(copied, function (relPath) { - self._addDependency(path.join(serverPath, relPath)); + // Write out each target + _.each(self.targets, function (target) { + target.write(path.join(buildPath, target.pathInBundle), + nodeModulesMode); }); - self.dependencyInfo.directories[serverPath] = { - include: [/.?/], - exclude: ignoreFiles - }; - - // --- Third party dependencies --- - - if (nodeModulesMode === "symlink") - fs.symlinkSync(path.join(files.get_dev_bundle(), 'lib', 'node_modules'), - path.join(buildPath, 'server', 'node_modules')); - else if (nodeModulesMode === "copy") - files.cp_r(path.join(files.get_dev_bundle(), 'lib', 'node_modules'), - path.join(buildPath, 'server', 'node_modules'), - {ignore: ignoreFiles}); - else - /* nodeModulesMode === "skip" */; + // Tell Galaxy what version of the dependency kit we're using, so + // it can load the right modules. (Include this even if we copied + // or symlinked a node_modules, since that's probably enough for + // it to work in spite of the presence of node_modules for the + // wrong arch.) The place we stash this is grody for temporary + // reasons of backwards compatibility. + files.mkdir_p(path.join(buildPath, "server", 0755)); fs.writeFileSync( path.join(buildPath, 'server', '.bundle_version.txt'), fs.readFileSync( path.join(files.get_dev_bundle(), '.bundle_version.txt'))); - // --- Static assets --- - - var addClientFileToManifest = function (filepath, contents, type, cacheable, url, hash) { - if (! contents instanceof Buffer) - throw new Error('contents must be a Buffer'); - var normalized = filepath.split(path.sep).join('/'); - if (normalized.charAt(0) === '/') - normalized = normalized.substr(1); - self.manifest.push({ - // path is normalized to use forward slashes - path: (cacheable ? 'static_cacheable' : 'static') + '/' + normalized, - where: 'client', - type: type, - cacheable: cacheable, - url: url || '/' + normalized, - // contents is a Buffer and so correctly gives us the size in bytes - size: contents.length, - hash: hash || sha1(contents) - }); - }; - - if (isApp) { - var publicDir = path.join(projectDir, 'public'); - - if (fs.existsSync(publicDir)) { - var copied = - files.cp_r(publicDir, path.join(buildPath, 'static'), - {ignore: ignoreFiles}); - - _.each(copied, function (relPath) { - var filepath = path.join(publicDir, relPath); - var contents = fs.readFileSync(filepath); - var hash = sha1(contents); - - self._addDependency(filepath, hash); - addClientFileToManifest(relPath, contents, 'static', false, - undefined, hash); - }); - } - - self.dependencyInfo.directories[publicDir] = { - include: [/.?/], - exclude: ignoreFiles - }; - } - - // Add cache busting query param if needed, and - // add to manifest. - var processClientCode = function (type, file) { - var contents, url, hash; - if (file in self.files.clientCacheable) { - contents = self.files.clientCacheable[file]; - url = file; - } - else if (file in self.files.client) { - // Client css and js becomes cacheable with the addition of the - // cache busting query parameter. - contents = self.files.client[file]; - delete self.files.client[file]; - self.files.clientCacheable[file] = contents; - hash = sha1(contents); - url = file + '?' + hash; - } - else - throw new Error('unable to find file: ' + file); - - addClientFileToManifest(file, contents, type, true, url, hash); - }; - - _.each(self.js.client, function (file) { processClientCode('js', file); }); - _.each(self.css, function (file) { processClientCode('css', file); }); - - // -- Client code -- - for (var relPath in self.files.client) { - var fullPath = path.join(buildPath, 'static', relPath); - files.mkdir_p(path.dirname(fullPath), 0755); - fs.writeFileSync(fullPath, self.files.client[relPath]); - addClientFileToManifest(relPath, self.files.client[relPath], 'static', false); - } - - // -- Client cache forever code -- - for (var relPath in self.files.clientCacheable) { - var fullPath = path.join(buildPath, 'static_cacheable', relPath); - files.mkdir_p(path.dirname(fullPath), 0755); - fs.writeFileSync(fullPath, self.files.clientCacheable[relPath]); - } - - appJson.load = []; - files.mkdir_p(path.join(buildPath, 'app'), 0755); - for (var relPath in self.files.server) { - var pathInBundle = path.join('app', relPath); - var fullPath = path.join(buildPath, pathInBundle); - files.mkdir_p(path.dirname(fullPath), 0755); - fs.writeFileSync(fullPath, self.files.server[relPath]); - appJson.load.push(pathInBundle); - } - - // `node_modules` directories for packages - _.each(self.nodeModulesDirs, function (sourceNodeModulesDir, packageName) { - files.mkdir_p(path.join(buildPath, 'npm')); - var buildModulesPath = path.join(buildPath, 'npm', packageName); - // XXX we should consider supporting bundle time-only npm dependencies - // which don't need to be pushed to the server. - if (nodeModulesMode === 'symlink') { - // if we symlink the dev_bundle, also symlink individual package - // node_modules. - fs.symlinkSync(sourceNodeModulesDir, buildModulesPath); - } else { - // otherwise, copy them. if we're skipping the dev_bundle - // modules (eg for deploy) we still need the per-package - // modules. - // XXX this breaks arch-specific modules. oh well. - files.cp_r(sourceNodeModulesDir, buildModulesPath); - } - }); - - var appHtml = self._generateAppHtml(); - fs.writeFileSync(path.join(buildPath, 'app.html'), appHtml); - self.manifest.push({ - path: 'app.html', - where: 'internal', - hash: sha1(appHtml) - }); - self._addDependency(path.join(__dirname, 'app.html.in')); - - // --- Documentation, and running from the command line --- + // Affordances for standalone use fs.writeFileSync(path.join(buildPath, 'main.js'), -"require('./server/server.js');\n"); +"require('./programs/server/server.js');\n"); fs.writeFileSync(path.join(buildPath, 'README'), "This is a Meteor application bundle. It has only one dependency,\n" + @@ -641,48 +917,29 @@ _.extend(Bundle.prototype, { "\n" + "Find out more about Meteor at meteor.com.\n"); - // --- Source file dependencies --- - - _.each(_.values(self.slices), function (slice) { - _.extend(self.dependencyInfo.files, slice.pkg.dependencyFileShas); - }); - - if (isApp) { - // Include files in the app that match any file extension - // handled by any package that the app uses - self.dependencyInfo.directories[path.resolve(projectDir)] = { - include: _.map(self._appExtensions(), function (ext) { - return new RegExp('\\.' + ext.slice(1) + "$"); - }), - exclude: ignoreFiles - }; - // Exclude the packages directory in an app, since packages - // explicitly call out the files they use - self.dependencyInfo.directories[path.resolve(projectDir, 'packages')] = { - exclude: [/.?/] - }; - // Exclude .meteor/local and everything under it - self.dependencyInfo.directories[ - path.resolve(projectDir, '.meteor', 'local')] = { exclude: [/.?/] }; - } - - // --- Metadata --- - - appJson.manifest = self.manifest; - - if (self.releaseStamp && self.releaseStamp !== 'none') - appJson.release = self.releaseStamp; - - fs.writeFileSync(path.join(buildPath, 'app.json'), - JSON.stringify(appJson, null, 2)); - - // --- Move into place --- + // Control file + // XXX not currently including the release in the bundle in a + // machine-readable format. is that a bad idea? + var json = { + version: "1", + builtBy: "Meteor" + (self.releaseStamp && self.releaseStamp !== "none" ? + " " + self.releaseStamp : ""), + programs: _.map(self.targets, function (target) { + return { + name: target.name, + arch: target.arch, + path: target.pathInBundle + } + }) + }; + fs.writeFileSync(path.join(buildPath, 'star.json'), + JSON.stringify(json, null, 2)); + // Move into place // XXX cleaner error handling (no exceptions) files.rm_recursive(outputPath); fs.renameSync(buildPath, outputPath); } - }); /////////////////////////////////////////////////////////////////////////////// @@ -750,29 +1007,46 @@ exports.bundle = function (appDir, outputPath, options) { library: library }); + // Create targets + var client = new ClientTarget("client", bundle, { arch: "client" }); + var server = new ServerTarget("server", bundle, { arch: "server", + clientTarget: client }); + bundle.targets = [client, server]; + // Create a Package object that represents the app var app = library.getForApp(appDir, ignoreFiles); // Populate the list of slices to load - bundle.determineLoadOrder({ - use: {client: [app], server: [app]}, - test: {client: options.testPackages || [], - server: options.testPackages || []} - }); - - // Process npm modules - bundle.prepNodeModules(); + client.determineLoadOrder({use: [app], test: options.testPackages || []}); + server.determineLoadOrder({use: [app], test: options.testPackages || []}); // Link JavaScript, put resources in load order, and copy them to // the bundle - bundle.emitResources(); + client.emitResources(); + server.emitResources(); - // Minify, if requested - if (options.minify) - bundle.minify(); + // Minify, if requested (only the client) + if (options.minify) { + client.minifyJs(); + client.minifyCss(); + } + + // Add assets from /public directory + // XXX this should probably be part of the appDir reader + if (files.is_app_dir(appDir)) { /* XXX what is this checking? */ + var publicDir = path.join(appDir, 'public'); + if (fs.existsSync(publicDir)) + client.addAssetDir(publicDir, ignoreFiles); + } + + // Make client-side CSS and JS assets cacheable forever, by adding + // a query string with a cache-busting hash. + client.addCacheBusters("js"); + client.addCacheBusters("css"); // Write to disk - bundle.writeToDirectory(outputPath, appDir, options.nodeModulesMode); + client.assignTargetPaths(); + bundle.write(outputPath, appDir, options.nodeModulesMode); return { errors: false, diff --git a/tools/library.js b/tools/library.js index c89d6f13ed..2a8433de72 100644 --- a/tools/library.js +++ b/tools/library.js @@ -83,8 +83,7 @@ _.extend(Library.prototype, { if (name in self.overrides) packageDir = self.overrides[name]; - // Try local directories ('packages' subdirectory in an app, - // PACKAGE_DIRS environment variable, git checkout.) + // Try localPackageDirs if (! packageDir) { for (var i = 0; i < self.localPackageDirs.length; ++i) { var packageDir = path.join(self.localPackageDirs[i], name); diff --git a/tools/meteor.js b/tools/meteor.js index fe93fa902a..773d473e58 100644 --- a/tools/meteor.js +++ b/tools/meteor.js @@ -96,7 +96,7 @@ Fiber(function () { // Let the user provide additional package directories to search // in PACKAGE_DIRS (colon-separated.) if (process.env.PACKAGE_DIRS) - localPackageDirs.push.apply(packageDirs, + localPackageDirs.push.apply(localPackageDirs, process.env.PACKAGE_DIRS.split(':')); // If we're running out of a git checkout of meteor, use the packages from diff --git a/tools/packages.js b/tools/packages.js index a960cdd7fa..330f3c716e 100644 --- a/tools/packages.js +++ b/tools/packages.js @@ -47,10 +47,10 @@ var Package = function (library) { // package name to true. self.unordered = {}; - // Files that we want to monitor for changes in development mode, such as - // source files and package.js. Maps relative paths to the SHA of file - // contents. Set only after ensureCompiled(). - self.dependencyFileShas = {}; + // Files and directories that we want to monitor for changes in + // development mode, such as source files and package.js, in the + // format accepted by watch.Watcher. + self.dependencyInfo = { files: {}, directories: {} }; // All symbols exported from the JavaScript code in this // package. Map from role to where to array of string symbol (eg @@ -104,6 +104,10 @@ var Package = function (library) { // XXX NOTE: this is set by Library reaching into us self.inWarehouse = false; + // True if we've run installNpmDependencies. (It's slow and there's + // no need to do it more than once.) + self.npmUpdated = false; + // functions that can be called when the package is scanned -- // visible as `Package` when package.js is executed self.packageFacade = { @@ -188,16 +192,20 @@ var Package = function (library) { }; _.extend(Package.prototype, { - // Add a dependency (in the sense of dependencyFileShas) on a + // Add a dependency (in the sense of dependencyInfo) on a // file. If hash is supplied it should be the sha1 of the file // contents. If omitted it will be computed. relPath will be // resolved to an absolute path (relative to self.sourceRoot.) - _addDependency: function (relPath, contents) { + _addDependency: function (relPath, contents, onlyIfExists) { var self = this; var absPath = path.resolve(self.sourceRoot, relPath); + + if (onlyIfExists && ! fs.existsSync(absPath)) + return; + if (! contents) contents = fs.readFileSync(absPath); - self.dependencyFileShas[absPath] = bundler.sha1(contents); + self.dependencyInfo.files[absPath] = bundler.sha1(contents); }, // loads a package's package.js file into memory, using @@ -414,6 +422,7 @@ _.extend(Package.prototype, { packages = _.union(packages, project.get_packages(appDir)); // XXX this read has a race with the actual read that is used self._addDependency(path.join(appDir, '.meteor', 'packages')); + self._addDependency(path.join(appDir, '.meteor', 'release'), null, true); _.each(["use", "test"], function (role) { _.each(["client", "server"], function (where) { @@ -439,6 +448,32 @@ _.extend(Package.prototype, { }; self.forceExports = {use: {client: [], server: []}, test: {client: [], server: []}}; + + // Directories to monitor for new files + var allExts = []; + _.each(["use", "test"], function (role) { + _.each(["client", "server"], function (where) { + allExts = _.union(allExts, self.registeredExtensions(role, where)); + }); + }); + + self.dependencyInfo.directories[path.resolve(appDir)] = { + include: _.map(allExts, function (ext) { + return new RegExp('\\.' + ext.slice(1) + "$"); + }), + exclude: ignoreFiles + }; + // Inside the packages directory, only look for new packages + // (which we can detect by the appearance of a package.js file.) + // Other than that, packages explicitly call out the files they + // use. + self.dependencyInfo.directories[path.resolve(appDir, 'packages')] = { + include: [ /^package\.js$/ ], + exclude: ignoreFiles + }; + // Exclude .meteor/local and everything under it. + self.dependencyInfo.directories[ + path.resolve(appDir, '.meteor', 'local')] = { exclude: [/.?/] }; }, // Process all source files through the appropriate handlers and run @@ -621,18 +656,26 @@ _.extend(Package.prototype, { // @param npmDependencies {Object} eg {gcd: "0.0.0", tar: "0.1.14"} installNpmDependencies: function(quiet) { var self = this; + // Nothing to do if there's no Npm.depends(). if (!self.npmDependencies) return; + // Warehouse packages come with their NPM dependencies and are read-only. if (self.inWarehouse) return; + + // No need to do it more than once. + if (self.npmUpdated) + return; + // go through a specialized npm dependencies update process, ensuring we // don't get new versions of any (sub)dependencies. this process also runs // mostly safely multiple times in parallel (which could happen if you have // two apps running locally using the same package) meteorNpm.updateDependencies( self.name, self.npmDir(), self.npmDependencies, quiet); + self.npmUpdated = true; }, npmDir: function () { diff --git a/tools/server/server.js b/tools/server/server.js index d970dc22b7..557360ca5e 100644 --- a/tools/server/server.js +++ b/tools/server/server.js @@ -24,18 +24,18 @@ if (require('semver').lt(process.version, MIN_NODE_VERSION)) { // Keepalives so that when the outer server dies unceremoniously and // doesn't kill us, we quit ourselves. A little gross, but better than // pidfiles. -var init_keepalive = function () { - var keepalive_count = 0; +var initKeepalive = function () { + var keepaliveCount = 0; process.stdin.on('data', function (data) { - keepalive_count = 0; + keepaliveCount = 0; }); process.stdin.resume(); setInterval(function () { - keepalive_count ++; - if (keepalive_count >= 3) { + keepaliveCount ++; + if (keepaliveCount >= 3) { console.log("Failed to receive keepalive! Exiting."); process.exit(1); } @@ -105,31 +105,14 @@ var categorizeRequest = function (req) { }; }; - - - -// add any runtime configuration options needed to app_html -var runtime_config = function (app_html) { - var insert = ''; - if (typeof __meteor_runtime_config__ === 'undefined') - return app_html; - - app_html = app_html.replace( - "// ##RUNTIME_CONFIG##", - "__meteor_runtime_config__ = " + - JSON.stringify(__meteor_runtime_config__) + ";"); - - return app_html; -}; - -var htmlAttributes = function (app_html, request) { +var htmlAttributes = function (template, request) { var attributes = ''; _.each(__meteor_bootstrap__.htmlAttributeHooks || [], function (hook) { var attribute = hook(request); if (attribute !== null && attribute !== undefined && attribute !== '') attributes += ' ' + attribute; }); - return app_html.replace('##HTML_ATTRIBUTES##', attributes); + return template.replace('##HTML_ATTRIBUTES##', attributes); }; // Serve app HTML for this URL? @@ -156,7 +139,16 @@ var appUrl = function (url) { } var run = function () { - var bundle_dir = path.join(__dirname, '..'); + var serverDir = __dirname; + + // read our control file + var serverJson = + JSON.parse(fs.readFileSync(path.join(serverDir, 'app.json'), 'utf8')); + + // read the control for the client we'll be serving up + var clientJsonPath = path.join(serverDir, serverJson.client); + var clientDir = path.dirname(clientJsonPath); + var clientJson = JSON.parse(fs.readFileSync(clientJsonPath, 'utf8')); // check environment var port = process.env.PORT ? parseInt(process.env.PORT) : 80; @@ -167,17 +159,19 @@ var run = function () { // webserver var app = connect.createServer(); - var static_cacheable_path = path.join(bundle_dir, 'static_cacheable'); - if (fs.existsSync(static_cacheable_path)) + + var staticCacheablePath = path.join(clientDir, clientJson.staticCacheable); + if (staticCacheablePath) // cacheable files are files that should never change. Typically // named by their hash (eg meteor bundled js and css files). // cache them ~forever (1yr) // // 'root' option is to work around an issue in connect/gzippo. // See https://github.com/meteor/meteor/pull/852 - app.use(gzippo.staticGzip(static_cacheable_path, + app.use(gzippo.staticGzip(staticCacheablePath, {clientMaxAge: 1000 * 60 * 60 * 24 * 365, root: '/'})); + // cache non-cacheable file anyway. This isn't really correct, as // users can change the files and changes won't propogate // immediately. However, if we don't cache them, browsers will @@ -186,22 +180,23 @@ var run = function () { // bust caches. That way we can both get good caching behavior and // allow users to change assets without delay. // https://github.com/meteor/meteor/issues/773 - app.use(gzippo.staticGzip(path.join(bundle_dir, 'static'), - {clientMaxAge: 1000 * 60 * 60 * 24, - root: '/'})); - - // read bundle config file - var info_raw = - fs.readFileSync(path.join(bundle_dir, 'app.json'), 'utf8'); - var info = JSON.parse(info_raw); - var bundle = {manifest: info.manifest, root: bundle_dir}; + var staticPath = path.join(clientDir, clientJson.static); + if (staticPath) + app.use(gzippo.staticGzip(staticPath, + {clientMaxAge: 1000 * 60 * 60 * 24, + root: '/'})); // start up app __meteor_bootstrap__ = { startup_hooks: [], app: app, // metadata about this bundle - bundle: bundle, + // XXX this could use some refactoring to better distinguish + // server and client + bundle: { + manifest: clientJson.manifest, + root: clientDir + }, // function that takes a connect `req` object and returns a summary // object with information about the request. See // #BrowserIdentifcation @@ -216,38 +211,30 @@ var run = function () { }; __meteor_runtime_config__ = {}; - if (info.release) { - __meteor_runtime_config__.meteorRelease = info.release; + if (serverJson.config && serverJson.config.release) { + __meteor_runtime_config__.meteorRelease = serverJson.config.release; } Fiber(function () { // (put in a fiber to let Meteor.db operations happen during loading) // load app code - _.each(info.load, function (filename) { - var code = fs.readFileSync(path.join(bundle_dir, filename)); + _.each(serverJson.load, function (fileInfo) { + var code = fs.readFileSync(path.join(serverDir, fileInfo.path)); - // even though the npm packages are correctly placed in - // node_modules/ relative to the package source, we can't just - // use standard `require` because packages are loaded using - // runInThisContext. see #runInThisContext var Npm = { // require an npm module used by your package, or one from the // dev bundle if you are in an app or your package isn't using // said npm module - require: function(name) { - var filePathParts = filename.split(path.sep); - // XXX it's weird that we're dependent on the dir structure - if (!(filePathParts[0] === 'app' - && (filePathParts[1] === 'packages' - || filePathParts[1] === 'package-tests'))) { - return require(name); // current no support for npm outside packages. load from dev bundle only + require: function (name) { + if (! fileInfo.node_modules) { + // current no support for npm outside packages. load from + // dev bundle only + return require(name); } - var packageName = filePathParts[2].replace(/\.js$/, ''); - var nodeModuleDir = path.join( - __dirname, - '..', // get out of server - 'npm', packageName, name); + + var nodeModuleDir = + path.join(__dirname, fileInfo.node_modules, name); if (fs.existsSync(nodeModuleDir)) { return require(nodeModuleDir); @@ -255,6 +242,11 @@ var run = function () { try { return require(name); } catch (e) { + // Try to guess the package name so we can print a nice + // error message + var filePathParts = fileInfo.path.split(path.sep); + var packageName = filePathParts[2].replace(/\.js$/, ''); + // XXX better message throw new Error( "Can't find npm module '" + name + @@ -265,8 +257,7 @@ var run = function () { }; // \n is necessary in case final line is a //-comment var wrapped = "(function(Npm){" + code + "\n})"; - // See #runInThisContext - // + // it's tempting to run the code in a new context so we can // precisely control the enviroment the user code sees. but, // this is harder than it looks. you get a situation where [] @@ -279,7 +270,7 @@ var run = function () { // runIn[Foo]Context that causes it to print out a descriptive // error message on parse error. it's what require() uses to // generate its errors. - var func = require('vm').runInThisContext(wrapped, filename, true); + var func = require('vm').runInThisContext(wrapped, fileInfo.path, true); // Setting `this` to `global` allows you to do a top-level // "this.foo = " to define global variables when using "use strict" // (http://es5.github.io/#x15.3.4.4); this is the only way to do @@ -291,9 +282,12 @@ var run = function () { // Actually serve HTML. This happens after user code, so that // packages can insert connect middlewares and update // __meteor_runtime_config__ - var app_html = fs.readFileSync(path.join(bundle_dir, 'app.html'), 'utf8'); - - app_html = runtime_config(app_html); + var boilerplateHtmlPath = path.join(clientDir, clientJson.page); + var boilerplateHtml = + fs.readFileSync(boilerplateHtmlPath, 'utf8').replace( + "// ##RUNTIME_CONFIG##", + "__meteor_runtime_config__ = " + + JSON.stringify(__meteor_runtime_config__) + ";"); app.use(function (req, res, next) { if (! appUrl(req.url)) @@ -303,7 +297,7 @@ var run = function () { res.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'}); - var requestSpecificHtml = htmlAttributes(app_html, request); + var requestSpecificHtml = htmlAttributes(boilerplateHtml, request); res.write(requestSpecificHtml); res.end(); }); @@ -326,7 +320,7 @@ var run = function () { }).run(); if (argv.keepalive) - init_keepalive(); + initKeepalive(); }; run(); diff --git a/tools/tests/test_bundler_npm.js b/tools/tests/test_bundler_npm.js index 7fae8ac5bf..c00fd67606 100644 --- a/tools/tests/test_bundler_npm.js +++ b/tools/tests/test_bundler_npm.js @@ -6,6 +6,13 @@ var meteorNpm = require(path.join(__dirname, '..', 'meteor_npm.js')); var tmpPackageDirContainer = tmpDir(); var testPackageDir = path.join(tmpPackageDirContainer, 'test-package'); +lib = new library.Library({ + localPackageDirs: [ + tmpPackageDirContainer, + path.join(files.getCurrentToolsDir(), 'packages') + ] +}); + var updateTestPackage = function(npmDependencies) { if (!fs.existsSync(testPackageDir)) fs.mkdirSync(testPackageDir); @@ -100,9 +107,9 @@ var _assertCorrectPackageNpmDir = function(deps) { var _assertCorrectBundleNpmContents = function(bundleDir, deps) { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(bundleDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); - var bundledPackageNodeModulesDir = path.join(bundleDir, 'npm', 'test-package'); + var bundledPackageNodeModulesDir = path.join(bundleDir, 'programs', 'server', 'npm', 'test-package'); // bundle actually has the npm modules _.each(deps, function(version, name) { @@ -128,8 +135,6 @@ var looksInstalled = function (nodeModulesDir, name) { /// TESTS /// -var lib = new library.Library(); - console.log("app that uses gcd - clean run"); assert.doesNotThrow(function () { updateTestPackage({gcd: '0.0.0'}); @@ -158,6 +163,7 @@ assert.doesNotThrow(function () { assert(fs.existsSync(path.join(nodeModulesDir))); files.rm_recursive(nodeModulesDir); assert(!fs.existsSync(path.join(nodeModulesDir))); + lib.refresh(); // while bundling, verify that we don't call `npm install // name@version unnecessarily` -- calling `npm install` is enough, @@ -199,6 +205,7 @@ assert.doesNotThrow(function () { files.rm_recursive(nodeModulesMimeDir); assert(!fs.existsSync(path.join(nodeModulesMimeDir))); + lib.refresh(); var result = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip', releaseStamp: 'none', library: lib}); assert.strictEqual(result.errors, false, result.errors && result.errors[0]); _assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.7', semver: '1.1.0'}); diff --git a/tools/tests/test_bundler_options.js b/tools/tests/test_bundler_options.js index 0cd1fa70b4..5484097c60 100644 --- a/tools/tests/test_bundler_options.js +++ b/tools/tests/test_bundler_options.js @@ -10,7 +10,9 @@ var emptyAppDir = path.join(__dirname, 'empty-app'); /// TESTS /// -var lib = new library.Library(); +lib = new library.Library({ + localPackageDirs: [ path.join(files.getCurrentToolsDir(), 'packages') ] +}); console.log("nodeModules: 'skip'"); assert.doesNotThrow(function () { @@ -20,16 +22,18 @@ assert.doesNotThrow(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); // no top level node_modules directory - assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules"))); + assert(!fs.existsSync(path.join(tmpOutputDir, + "programs", "server", "node_modules"))); // yes package node_modules directory assert(fs.lstatSync(path.join( - tmpOutputDir, "npm", "livedata")) + tmpOutputDir, "programs", "server", "npm", "livedata")) .isDirectory()); // verify that contents are minified - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", + "client", "app.html")); assert(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml)); assert(!(/src=\"\/packages/.test(appHtml))); }); @@ -42,14 +46,15 @@ assert.doesNotThrow(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); // verify that contents are not minified - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", + "client", "app.html")); assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml))); assert(/src=\"\/packages\/meteor/.test(appHtml)); assert(/src=\"\/packages\/deps/.test(appHtml)); // verify that tests aren't included - assert(!(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml))); + assert(!(/src=\"\/package-tests\/meteor/.test(appHtml))); }); console.log("nodeModules: 'skip', no minify, testPackages: ['meteor']"); @@ -61,9 +66,10 @@ assert.doesNotThrow(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); // verify that tests for the meteor package are included - var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html")); + var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs", + "client", "app.html")); assert(/src=\"\/package-tests\/meteor.js/.test(appHtml)); }); @@ -75,11 +81,11 @@ assert.doesNotThrow(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); // node_modules directory exists and is not a symlink - assert(!fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + assert(!fs.lstatSync(path.join(tmpOutputDir, "programs", "server", "node_modules")).isSymbolicLink()); // node_modules contains fibers - assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); + assert(fs.existsSync(path.join(tmpOutputDir, "programs", "server", "node_modules", "fibers"))); }); console.log("nodeModules: 'symlink'"); @@ -90,14 +96,14 @@ assert.doesNotThrow(function () { // sanity check -- main.js has expected contents. assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(), - "require('./server/server.js');"); + "require('./programs/server/server.js');"); // node_modules directory exists and is a symlink - assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink()); + assert(fs.lstatSync(path.join(tmpOutputDir, "programs", "server", "node_modules")).isSymbolicLink()); // node_modules contains fibers - assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers"))); + assert(fs.existsSync(path.join(tmpOutputDir, "programs", "server", "node_modules", "fibers"))); // package node_modules directory also a symlink assert(fs.lstatSync(path.join( - tmpOutputDir, "npm", "livedata")) + tmpOutputDir, "programs", "server", "npm", "livedata")) .isSymbolicLink()); });