diff --git a/tools/bundler.js b/tools/bundler.js index d9b5b55967..3a75e8e2d7 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -19,6 +19,13 @@ // be the principled mechanism by which a server program could read // a client program so it can server it) // +// - plugins: array of plugins in the star, each an object: +// - name: short, unique name for plugin, for referring to it +// programmatically +// - arch: typically 'js' (for a portable plugin) or eg +// 'js.linux.x86_64' for one that include native node_modules +// - path: directory (relative to star.json) containing this plugin +// // /README: human readable instructions // // /main.js: script that can be run in node.js to start the site @@ -122,6 +129,26 @@ // /node_modules: node_modules needed for server.js. omitted if // deploying (see .bundle_version.txt above), copied if bundling, // symlinked if developing locally. +// +// +// == Format of a program that is to be used as a plugin == +// +// /program.json: +// - load: array with each item describing a JS file to load, in load order: +// - path: path of file, relative to program.json +// - node_modules: if Npm.require is called from this file, this is +// the path (relative to program.json) of the directory that should +// be search for npm modules +// +// Note that while the spec for "native.*" is going to change to +// represent an arbitrary POSIX (or Windows) process rather than +// assuming a nodejs host, these plugins will always refer to +// JavaScript code (that potentially might be a plugin to be loaded +// into an existing JS VM). But this seems to be a concern that is +// somewhat orthogonal to arch (these plugins can still use packages +// of arch "native.*".) There is probably a missing abstraction here +// somewhere (decoupling target type from architecture) but it can +// wait until later. var path = require('path'); var files = require(path.join(__dirname, 'files.js')); @@ -161,8 +188,8 @@ var inherits = function (child, parent) { var NodeModulesDirectory = function (options) { var self = this; - // The absolute path (on local disk at build time) to a directory - // that contains the built node_modules to use. + // The absolute path (on local disk) to a directory that contains + // the built node_modules to use. self.sourcePath = options.sourcePath; // The path (relative to the bundle root) where we would preferably @@ -282,21 +309,16 @@ _.extend(File.prototype, { /////////////////////////////////////////////////////////////////////////////// // options: +// - library: package library to use for resolving package dependenices // - arch: the architecture to build // // see subclasses for additional options -var Target = function (name, options) { +var Target = function (options) { var self = this; // Package library to use for resolving package dependenices. self.library = options.library; - // A name for this target. - self.name = name; - - // Path of this target in the bundle, relative to the root of the bundle. - self.pathInBundle = path.join('programs', self.name); - // Something like "browser.w3c" or "native" or "native.osx.x86_64" self.arch = options.arch; @@ -530,7 +552,7 @@ _.extend(Target.prototype, { //////////////////// ClientTarget //////////////////// -var ClientTarget = function (name, options) { +var ClientTarget = function (options) { var self = this; Target.apply(this, arguments); @@ -706,7 +728,7 @@ _.extend(ClientTarget.prototype, { // options: // - clientTarget: the ClientTarget to serve up over HTTP as our client // - releaseStamp: the Meteor release name (for retrieval at runtime) -var ServerTarget = function (name, options) { +var ServerTarget = function (options) { var self = this; Target.apply(this, arguments); @@ -719,42 +741,90 @@ var ServerTarget = function (name, options) { inherits(ServerTarget, Target); +// Code factored out of ServerTarget.write and Plugin.write +// Options: +// - load: array of objects with keys targetPath, data, nodeModulesDirectory +// - nodeModulesDirectories: array of NodeModulesDirectory referenced +// - extraControlInfo: extra keys for program.json +var writeServerTargetOrPlugin = function (builder, options) { + // Finalize choice of paths for node_modules directories -- These + // paths are no longer just "preferred"; they are the final paths + // that we will use + var nodeModulesDirectories = []; + _.each(options.nodeModulesDirectories || [], function (nmd) { + nodeModulesDirectories.push(new NodeModulesDirectory({ + sourcePath: nmd.sourcePath, + preferredBundlePath: builder.generateFilename(nmd.preferredBundlePath, + { directory: true }) + })); + }); + + // JavaScript sources + var load = []; + _.each(options.load || [], function (item) { + if (! item.targetPath) + throw new Error("No targetPath?"); + + builder.write(item.targetPath, { data: item.data }); + load.push({ + path: item.targetPath, + node_modules: item.nodeModulesDirectory ? + item.nodeModulesDirectory.preferredBundlePath : undefined + }); + }); + + // node_modules resources from the packages. Due to appropriate + // builder configuration, 'meteor bundle' and 'meteor deploy' copy + // them, and 'meteor run' symlinks them. If these contain + // arch-specific code then the target will end up having an + // appropriately specific arch. + _.each(nodeModulesDirectories, function (nmd) { + builder.copyDirectory({ + from: nmd.sourcePath, + to: nmd.preferredBundlePath, + depend: false + }); + }); + + // Control file + var json = _.extend({ + load: load + }, options.extraControlInfo || {}); + builder.writeJson('program.json', json); +}; + _.extend(ServerTarget.prototype, { // Output the finished target to disk - write: function (builder, nodeModulesMode) { + // options: + // - omitDependencyKit: if true, don't copy node_modules from dev_bundle + // - getRelativeTargetPath: a function that takes {forTarget: + // Target, relativeTo: Target} and return the path of one target + // in the bundle relative to another. hack to get the path of the + // client target.. we'll find a better solution here eventually + write: function (builder, options) { var self = this; - var json = { - load: [], - client: path.join(path.relative(self.pathInBundle, - self.clientTarget.pathInBundle), - 'program.json'), - config: { - meteorRelease: self.releaseStamp && self.releaseStamp !== "none" ? - self.releaseStamp : undefined + if (! options.omitDependencyKit) + builder.reserve("node_modules", { directory: true }); + + writeServerTargetOrPlugin(builder, { + load: _.map(self.js, function (file) { + return { + targetPath: file.targetPath, + data: file.contents(), + nodeModulesDirectory: file.nodeModulesDirectory + }; + }), + nodeModulesDirectories: self.nodeModulesDirectories, + extraControlInfo: { + client: path.join(options.getRelativeTargetPath({ + forTarget: self.clientTarget, relativeTo: self}), + 'program.json'), + config: { + meteorRelease: self.releaseStamp && self.releaseStamp !== "none" ? + self.releaseStamp : undefined + } } - }; - - // Finalize choice of paths for node_modules directories -- These - // paths are no longer just "preferred"; they are the final paths - // that we will use - _.each(self.nodeModulesDirectories, function (nmd) { - nmd.preferredBundlePath = - builder.generateFilename(nmd.preferredBundlePath, { directory: true }); - }); - - // JavaScript sources - _.each(self.js, function (file) { - if (! file.targetPath) - throw new Error("No targetPath?"); - - builder.write(file.targetPath, { data: file.contents() }); - - json.load.push({ - path: file.targetPath, - node_modules: file.nodeModulesDirectory ? - file.nodeModulesDirectory.preferredBundlePath : undefined - }); }); // Server driver @@ -769,7 +839,7 @@ _.extend(ServerTarget.prototype, { // kit. This one is copied in 'meteor bundle', symlinked in // 'meteor run', and omitted by 'meteor deploy' (Galaxy provides a // version that's appropriate for the server architecture.) - if (nodeModulesMode !== "skip") { + if (! options.omitDependencyKit) { builder.copyDirectory({ from: path.join(files.get_dev_bundle(), 'lib', 'node_modules'), to: 'node_modules', @@ -777,73 +847,162 @@ _.extend(ServerTarget.prototype, { depend: false }); } - - // Extra node_modules resources from the packages. 'meteor bundle' - // and 'meteor deploy' copy them, and 'meteor run' symlinks - // them. If these contain arch-specific code then the target will - // end up having an appropriately specific arch. - _.each(self.nodeModulesDirectories, function (nmd) { - builder.copyDirectory({ - from: nmd.sourcePath, - to: nmd.preferredBundlePath, - depend: false - }); - }); - - // Control file - builder.writeJson('program.json', json); } }); -//////////////////// InProcessTarget //////////////////// +//////////////////// PluginTarget and Plugin //////////////////// -var InProcessTarget = function (name, options) { +var Plugin = function () { + var self = this; + + // Array of objects with keys: + // - targetPath: relative path to use if saved to disk (or for stack traces) + // - source: JS source code to load, as a string + // - nodeModulesDirectory: a NodeModulesDirectory indicating which + // directory should be searched by Npm.require() + // note: this can't be called `load` at it would shadow `load()` + self.jsToLoad = []; + + // node_modules directories that we need to copy into the target (or + // otherwise make available at runtime.) A map from an absolute path + // on disk (NodeModulesDirectory.sourcePath) to a + // NodeModulesDirectory object that we have created to represent it. + self.nodeModulesDirectories = {}; +}; + +_.extend(Plugin.prototype, { + // Load the plugin into the current process. It gets its own unique + // Package object containing its own private copy of every + // unipackage that it uses. This Package object is returned. + // + // If `bindings` is provided, it is a object containing a set of + // variables to set in the global environment of the executed + // code. The keys are the variable names and the values are their + // values. In addition to the contents of `bindings`, Package and + // Npm will be provided. + // + // XXX throw an error if the plugin includes any "app-style" code + // that is built to put symbols in the global namespace rather than + // in a compartment of Package + load: function (bindings) { + var self = this; + var ret = {}; + + // Eval each JavaScript file, providing a 'Npm' symbol in the same + // way that the server environment would, and a 'Package' symbol + // so the plugin has its own private universe of loaded packages + _.each(self.jsToLoad, function (item) { + var env = _.extend({ + Package: ret, + Npm: { + require: function (name) { + if (! item.nodeModulesDirectory) { + // No Npm.depends associated with this package + return require(name); + } + + var nodeModuleDir = + path.join(item.nodeModulesDirectory.sourcePath, name); + + if (fs.existsSync(nodeModuleDir)) { + return require(nodeModuleDir); + } + try { + return require(name); + } catch (e) { + throw new Error("Can't load npm module '" + name + + "' while loading " + item.targetPath + + ". Check your Npm.depends().'"); + } + } + } + }, bindings || {}); + + // \n is necessary in case final line is a //-comment + var wrapped = "(function(" + _.keys(env).join(",") + "){" + + item.source + "\n})"; + var func = require('vm').runInThisContext(wrapped, item.targetPath, true); + func.apply({}, _.values(env)); + }); + + return ret; + }, + + // Write this plugin out to disk + write: function (builder) { + var self = this; + + writeServerTargetOrPlugin(builder, { + load: _.map(self.jsToLoad, function (item) { + return { + targetPath: item.targetPath, + source: new Buffer(item.source, 'utf8'), + nodeModulesDirectory: file.nodeModulesDirectory + }; + }), + nodeModulesDirectories: self.nodeModulesDirectories + }); + }, + + // `dir` is a directory on disk that contains a program with arch + // matching js.* (eg, a plugin previously written out with write()) + initFromDisk: function (dir) { + var self = this; + var json = + JSON.parse(fs.readFileSync(path.join(dir, 'program.json'))); + + _.each(json.load, function (item) { + if (item.path.match(/\.\./)) + throw new Error("bad path in plugin bundle"); + + var nmd = undefined; + if (item.node_modules) { + var node_modules = path.join(dir, item.node_modules); + if (! node_modules in self.nodeModulesDirectories) + self.nodeModulesDirectories[node_modules] = + new NodeModulesDirectory({ + sourcePath: node_modules, + preferredBundlePath: item.node_modules + }); + nmd = self.nodeModulesDirectories[node_modules]; + } + + self.jsToLoad.push({ + targetPath: item.path, + source: fs.readFileSync(path.join(dir, item.path)), + nodeModulesDirectory: nmd + }); + }); + } +}); + +var PluginTarget = function (options) { var self = this; Target.apply(this, arguments); if (! archinfo.matches(self.arch, "native")) - throw new Error("InProcessTarget targeting something incompatible?"); + throw new Error("PluginTarget targeting something incompatible?"); }; -inherits(InProcessTarget, Target); +inherits(PluginTarget, Target); -_.extend(InProcessTarget.prototype, { - // Load all of the JavaScript in this target into the current process - load: function (builder, nodeModulesMode) { +_.extend(PluginTarget.prototype, { + toPlugin: function () { var self = this; + var ret = new Plugin; - // Eval each JavaScript file, providing a 'Npm' symbol in the same - // way that the server environment would _.each(self.js, function (file) { - var Npm = { - require: function (name) { - if (! file.nodeModulesDirectory) { - // No Npm.depends associated with this package - return require(name); - } - - var nodeModuleDir = - path.join(file.nodeModulesDirectory.sourcePath, name); - - if (fs.existsSync(nodeModuleDir)) { - return require(nodeModuleDir); - } - try { - return require(name); - } catch (e) { - throw new Error("Can't load npm module '" + name + - "' while loading " + file.targetPath + - ". Check your Npm.depends().'"); - } - } - }; - // \n is necessary in case final line is a //-comment - var wrapped = "(function(Npm){" + - file.contents().toString('utf8') + "\n})"; - var func = require('vm').runInThisContext(wrapped, file.targetPath, true); - func(Npm); + ret.jsToLoad.push({ + targetPath: file.targetPath, + source: file.contents().toString('utf8'), + nodeModulesDirectory: file.nodeModulesDirectory + }); }); + + ret.nodeModulesDirectories = self.nodeModulesDirectories; + + return ret; } }); @@ -851,9 +1010,10 @@ _.extend(InProcessTarget.prototype, { // writeSiteArchive /////////////////////////////////////////////////////////////////////////////// -// targets is an array of Targets to include in the bundle. outputPath -// is the path of a directory that should be created to contain the -// generated site archive. +// targets is a set of Targets to include in the bundle, as a map from +// target name (to use in the bundle) to a Target. outputPath is the +// path of a directory that should be created to contain the generated +// site archive. // // Returns dependencyInfo (in the format expected by watch.Watcher) // for all files and directories that ultimately went into the bundle. @@ -874,14 +1034,47 @@ var writeSiteArchive = function (targets, outputPath, options) { programs: [] }; + // Pick a path in the bundle for each target + var paths = {}; + _.each(targets, function (target, name) { + var p = path.join('programs', name); + builder.reserve(p, { directory: true }); + paths[name] = p; + }); + + // Hack to let servers find relative paths to clients. Should find + // another solution eventually (probably some kind of mount + // directive that mounts the client bundle in the server at runtime) + var getRelativeTargetPath = function (options) { + var pathForTarget = function (target) { + var name; + _.each(targets, function (t, n) { + if (t === target) + name = n; + }); + if (! name) + throw new Error("missing target?"); + + if (! (name in paths)) + throw new Error("missing target path?"); + + return paths[name]; + }; + + return path.relative(pathForTarget(options.relativeTo), + pathForTarget(options.forTarget)); + }; + // Write out each target - _.each(targets, function (target) { - target.pathInBundle = path.join('programs', target.name); - target.write(builder.enter(target.pathInBundle), options.nodeModulesMode); + _.each(targets, function (target, name) { + target.pathInBundle = path.join('programs', name); + target.write(builder.enter(paths[name]), + { omitDependencyKit: options.nodeModulesMode === "skip", + getRelativeTargetPath: getRelativeTargetPath }); json.programs.push({ - name: target.name, + name: name, arch: target.mostCompatibleArch(), - path: target.pathInBundle + path: paths[name] }); }); @@ -923,7 +1116,7 @@ var writeSiteArchive = function (targets, outputPath, options) { // bundle. A naive merge like this doesn't work in general but // should work in this case. var fileDeps = {}, directoryDeps = {}; - var dependencySources = targets.concat([builder]); + var dependencySources = [builder].concat(_.values(targets)); _.each(dependencySources, function (s) { var info = s.getDependencyInfo(); _.extend(fileDeps, info.files); @@ -1008,17 +1201,20 @@ exports.bundle = function (appDir, outputPath, options) { try { // Create targets - var client = new ClientTarget("client", { + var client = new ClientTarget({ library: library, arch: "browser" }); - var server = new ServerTarget("server", { + var server = new ServerTarget({ library: library, arch: archinfo.host(), clientTarget: client, releaseStamp: options.releaseStamp }); - var targets = [client, server]; + var targets = { + client: client, + server: server + }; // Create a Package object that represents the app var app = library.getForApp(appDir, ignoreFiles); @@ -1075,17 +1271,43 @@ exports.bundle = function (appDir, outputPath, options) { } }; -// Public entry point is unipackage.load, but for now it shares a lot -// of innards with the bundler (this possibly will get factored out at -// some point) so the implementation is here -exports._load = function (library, packages) { - var target = new InProcessTarget("load", { - library: library, +// Return a Plugin object. It can either be loaded into memory with +// load(), which returns the `Package` object inside the plugin's +// namespace, or saved to disk with write(builder). +// +// options: +// - library: required. the Library for resolving package dependencies +// - use: list of packages to use in the plugin, as strings (foo or foo.bar) +// - sources: source files to use (paths on local disk) +// - npmDependencies: map from npm package name to required version +// - npmDir: where to keep the npm cache and npm version shrinkwrap +// info. required if npmDependencies present. +exports.buildPlugin = function (options) { + if (options.npmDependencies && ! options.npmDir) + throw new Error("Must indicate .npm directory to use"); + + var pkg = new packages.Package(options.library); + + // It would be nice to have a way to say "make this package + // anonymous" without also saying "make its namespace the same as + // the global namespace." Though it would be an easy refactor, we + // don't have that yet, so just make up a random name. + var pkgName = "plugin" + Math.floor(Math.random() * 100000); + + pkg.initFromOptions(pkgName, { + sliceName: "plugin", + use: options.use || [], + sources: options.sources || [], + npmDependencies: options.npmDependencies, + npmDir: options.npmDir + }); + + var target = new PluginTarget({ + library: options.library, arch: archinfo.host() }); - target.determineLoadOrder({ packages: packages }); + target.determineLoadOrder({ packages: [pkg] }); target.emitResources(); - // I love it when a plan comes together - target.load(); -}; \ No newline at end of file + return target.toPlugin(); +}; diff --git a/tools/meteor.js b/tools/meteor.js index e014db9e5a..41b2c5ab54 100644 --- a/tools/meteor.js +++ b/tools/meteor.js @@ -1062,13 +1062,6 @@ Fiber(function () { if (!files.in_checkout() && !process.env.METEOR_TEST_NO_SPRINGBOARD) toolsSpringboard(); - // Load any needed unipackages - require('./unipackage.js').load({ - library: context.library, - packages: [], // add any unipackages here - release: context.releaseVersion - }); - if (argv['get-ready']) { getReady(); return; diff --git a/tools/packages.js b/tools/packages.js index da760c5e48..27d6d62bf3 100644 --- a/tools/packages.js +++ b/tools/packages.js @@ -404,16 +404,17 @@ var nextPackageId = 1; var Package = function (library) { var self = this; - // Fields set by init_*: - // name: package name, or null for an app pseudo-package or collection - // sourceRoot: base directory for resolving source files, null for collection - // serveRoot: base directory for serving files, null for collection - // A unique ID (guaranteed to not be reused in this process -- if // the package is reloaded, it will get a different id the second // time) self.id = nextPackageId++; + // The name of the package, or null for an app pseudo-package or + // collection. The package's exports will reside in Package.. + // When it is null it is linked like an application instead of like + // a package. + self.name = null; + // The path from which this package was loaded. null if loaded from // unipackage. self.sourceRoot = null; @@ -514,6 +515,45 @@ _.extend(Package.prototype, { preheat: function () { }, + // Programmatically create a package from scratch. For now, cannot + // create browser packages. + // + // Options: + // - sliceName + // - use + // - sources + // - npmDependencies + // - npmDir + initFromOptions: function (name, options) { + var self = this; + self.name = name; + + var isPortable = true; + var nodeModulesPath = null; + if (options.npmDependencies) { + meteorNpm.ensureOnlyExactVersions(option.npmDependencies); + meteorNpm.updateDependencies(name, options.npmDir, + options.npmDependencies); + if (! meteorNpm.dependenciesArePortable(options.npmDir)) + isPortable = false; + nodeModulesPath = path.join(options.npmDir, 'node_modules'); + } + + var arch = isPortable ? "native" : archinfo.host(); + var slice = new Slice(self, { + name: options.sliceName, + arch: arch, + uses: _.map(["meteor"].concat(options.use || []), function (spec) { + return { spec: spec } + }), + sources: options.sources || [], + nodeModulesPath: nodeModulesPath + }); + self.slices.push(slice); + + self.defaultSlices = {'native': [options.sliceName]}; + }, + // loads a package's package.js file into memory, using // runInThisContext. Wraps the contents of package.js in a closure, // supplying pseudo-globals 'Package' and 'Npm'. diff --git a/tools/unipackage.js b/tools/unipackage.js index 12870562ee..cd98205a62 100644 --- a/tools/unipackage.js +++ b/tools/unipackage.js @@ -4,20 +4,12 @@ var bundler = require('./bundler.js'); // Load unipackages into the currently running node.js process. Use // this to use unipackages (such as the DDP client) from command-line -// tools (such as 'meteor'.) The package's exports will be available -// as usual in Package.packagename. They will not be copied into your -// scope. -// -// Currently this may only be called once. This is because in the -// future we want to support packages that have portions that are -// conditionally included (whether slices like 'ddp.server', or units -// like an individual function in DomUtils) and the only way to add -// symbols to package's namespace once it's been initially set up is -// to use eval. We're not quite ready to sign up for eval because we'd -// first want to see how much that usage of it frustrates the -// JIT. (It's also because we currently go through the motions of -// setting up a 'proper' server environment and running any startup -// hooks -- this may or may not be the right call.) +// tools (such as 'meteor'.) The requested packages will be loaded +// together will all of their dependencies, and each time you call +// this function you load another, distinct copy of all of the +// packages. The return value is an object that maps package name to +// package exports (that is, it is the Package object from inside the +// sandbox created for the newly loaded packages.) // // Options: // - library: The Library to use to retrieve packages and their @@ -28,11 +20,18 @@ var bundler = require('./bundler.js'); // pass into the app with __meteor_runtime_config__ (essentially // this determines what Meteor.release will return within the loaded // environment) +// +// Example usage: +// var Meteor = require('./unipackage.js').load({ +// library: context.library, +// packages: ['livedata'], +// release: context.releaseVersion +// }).meteor.Meteor; +// var reverse = Meteor.connect('reverse.meteor.com'); +// console.log(reverse.call('reverse', 'hello world')); var load = function (options) { options = options || {}; - if (typeof __meteor_bootstrap__ !== "undefined") - throw new Error("unipackage.load may only be called once"); if (! (options.library instanceof library.Library)) throw new Error("unipackage.load requires a library"); @@ -40,17 +39,25 @@ var load = function (options) { // are specific to the HTTP server.) Kind of a hack. I suspect this // will get refactored before too long. Note that // __meteor_bootstrap__.require is no longer provided. - __meteor_bootstrap__ = { startup_hooks: [] }; - __meteor_runtime_config__ = { meteorRelease: options.release }; + var env = { + __meteor_bootstrap__: { startup_hooks: [] }, + __meteor_runtime_config__: { meteorRelease: options.release } + }; // Load the code - bundler._load(options.library, options.packages || []); + var plugin = bundler.buildPlugin({ + library: options.library, + use: options.packages || [] + }); + var ret = plugin.load(env); // Run any user startup hooks. - _.each(__meteor_bootstrap__.startup_hooks, function (x) { x(); }); + _.each(env.__meteor_bootstrap__.startup_hooks, function (x) { x(); }); + + return ret; }; var unipackage = exports; _.extend(exports, { load: load -}); \ No newline at end of file +});