From 2c1b297dc126f4ff9092b92fc4fa14fcb8e718d8 Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Tue, 16 Apr 2013 18:58:15 -0700 Subject: [PATCH] Facility for including unipackages from non-Meteor-built command-line nodejs programs (eg, 'meteor'). Does not fully duplicate the server environment (specifically, does not provide a HTTP server) so livedata does not yet load. --- tools/builder.js | 4 +- tools/bundler.js | 176 +++++++++++++++++++++++++++++++++----------- tools/linker.js | 7 +- tools/meteor.js | 8 ++ tools/unipackage.js | 56 ++++++++++++++ 5 files changed, 206 insertions(+), 45 deletions(-) create mode 100644 tools/unipackage.js diff --git a/tools/builder.js b/tools/builder.js index 8dcf358f7d..b5ca62b0ad 100644 --- a/tools/builder.js +++ b/tools/builder.js @@ -205,8 +205,8 @@ _.extend(Builder.prototype, { }, // Add relPath to the list of "already taken" paths in the - // bundle. This will cause writeFile, when in sanitize mode, to - // never pick this filename (and will prevent files that from being + // bundle. This will cause write, when in sanitize mode, to never + // pick this filename (and will prevent files that from being // written that would conflict with paths that we are expecting to // be directories.) Calling this twice on the same relPath will // given an exception. diff --git a/tools/bundler.js b/tools/bundler.js index ac94d5b362..0981b36b71 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -152,6 +152,24 @@ var inherits = function (child, parent) { child.prototype.constructor = child; }; +/////////////////////////////////////////////////////////////////////////////// +// NodeModulesDirectory +/////////////////////////////////////////////////////////////////////////////// + +// Represents a node_modules directory that we need to copy into the +// bundle or otherwise make available at runtime. +var NodeModulesDirectory = function (options) { + var self = this; + + // The absolute path (on local disk at build time) to a directory + // that contains the built node_modules to use. + self.sourcePath = options.sourcePath; + + // The path (relative to the bundle root) where we would preferably + // like the node_modules to be output (essentially cosmetic.) + self.preferredBundlePath = options.bundlePath; +}; + /////////////////////////////////////////////////////////////////////////////// // File /////////////////////////////////////////////////////////////////////////////// @@ -185,9 +203,10 @@ var File = function (options) { self.cacheable = options.cacheable || false; // The node_modules directory that Npm.require() should search when - // called from inside this file, given as a path in the target's - // filesystem. Only works in the "server" architecture. - self.nodeModulesTargetPath = null; + // called from inside this file, given as a NodeModulesDirectory, or + // null if Npm.depend() is not in effect for this file. Only works + // in the "server" architecture. + self.nodeModulesDirectory = null; self._contents = options.data || null; // contents, if known, as a Buffer self._hash = null; // hash, if known, as a hex string @@ -294,6 +313,12 @@ var Target = function (name, options) { // Files and paths used by this target, in the format used by // watch.Watcher. self.dependencyInfo = {files: {}, directories: {}}; + + // node_modules directories that we need to copy into the target (or + // otherwise make available at runtime.) A map from an absolute path + // on disk (NodeModulesDirectory.sourcePath) to a + // NodeModulesDirectory object that we have created to represent it. + self.nodeModulesDirectories = {}; }; _.extend(Target.prototype, { @@ -301,25 +326,17 @@ _.extend(Target.prototype, { // them, put them in load order, save in slices. // // options include: - // - packages: an array of packages whose default slices should be - // included - // - test: an array of packages whose test slices should be included - // - // In both cases you can pass either package names or Package - // objects. + // - packages: an array of packages (or, properly speaking, slices) + // to include. Each element should either be a Package object or a + // package name as a string (to include that package's default + // slices for this arch, or a string of the form 'package.slice' + // to include a particular named slice from a particular package. + // - test: an array of packages (as Package objects or as name + // strings) whose test slices should be included determineLoadOrder: function (options) { var self = this; var library = self.library; - var get = function (packageOrPackageName) { - var pkg = library.get(packageOrPackageName); - if (! pkg) { - console.error("Package not found: " + packageOrPackageName); - process.exit(1); - } - return pkg; - }; - // Each of these are map from slice.id to Slice var needed = {}; // Slices that we still need to add var done = {}; // Slices that are already in self.slices @@ -328,11 +345,15 @@ _.extend(Target.prototype, { // Find the roots var rootSlices = _.flatten([ - _.map(options.packages || [], function (pkg) { - return get(pkg).getDefaultSlices(self.arch); + _.map(options.packages || [], function (p) { + if (typeof p === "string") + return library.getSlices(p, self.arch); + else + return pkg.getDefaultSlices(self.arch); }), - _.map(options.test || [], function (pkg) { - return get(pkg).getTestSlices(self.arch); + _.map(options.test || [], function (p) { + var pkg = (p === "string" ? library.get(p) : p); + return p.getTestSlices(self.arch); }) ]); _.each(rootSlices, function (slice) { @@ -422,9 +443,18 @@ _.extend(Target.prototype, { } if (self.arch === "server" && resource.type === "js" && ! isApp && - slice.nodeModulesPath) - f.nodeModulesTargetPath = path.join('/npm', slice.pkg.name, - slice.sliceName); + slice.nodeModulesPath) { + var nmd = self.nodeModulesDirectories[slice.nodeModulesPath]; + if (! nmd) { + nmd = new NodeModulesDirectory({ + sourcePath: slice.nodeModulesPath, + preferredBundlePath: path.join('/npm', slice.pkg.name, + slice.sliceName) + }); + self.nodeModulesDirectories[slice.nodeModulesPath] = nmd; + } + f.nodeModulesDirectory = nmd; + } self[resource.type].push(f); return; @@ -671,11 +701,6 @@ var ServerTarget = function (name, options) { var self = this; Target.apply(this, arguments); - // These directories are copied (cp -r) or symlinked into the - // bundle. Map from targetPath (path in the Target's filesystem) to - // sourcePath (absolute path in the local filesystem.) - self.nodeModulesDirs = {}; - self.clientTarget = options.clientTarget; self.releaseStamp = options.releaseStamp; }; @@ -698,6 +723,14 @@ _.extend(ServerTarget.prototype, { } }; + // Finalize choice of paths for node_modules directories -- These + // paths are no longer just "preferred"; they are the final paths + // that we will use + _.each(self.nodeModulesDirectories, function (nmd) { + nmd.preferredBundlePath = + builder.generateFilename(nmd.preferredBundlePath, { directory: true }); + }); + // JavaScript sources _.each(self.js, function (file) { if (! file.targetPath) @@ -707,7 +740,8 @@ _.extend(ServerTarget.prototype, { json.load.push({ path: file.targetPath, - node_modules: file.nodeModulesTargetPath || undefined + node_modules: file.nodeModulesDirectory ? + file.nodeModulesDirectory.preferredBundlePath : undefined }); }); @@ -732,7 +766,7 @@ _.extend(ServerTarget.prototype, { }); } - // Extra user-defined arch-independent node_module. 'meteor + // Extra user-defined arch-independent node_modules. 'meteor // bundle' and 'meteor deploy' copy them, and 'meteor run' // symlinks them. (XXX Note that this doesn't work for // arch-specific packages. They'll just break if you deploy to a @@ -742,15 +776,12 @@ _.extend(ServerTarget.prototype, { // XXX we should consider supporting bundle time-only npm // dependencies which don't need to be pushed to the server. - _.each(self.slices, function (slice) { - if (slice.nodeModulesPath) { - // Copy the package's npm dependencies into the bundle. - builder.copyDirectory({ - from: slice.nodeModulesPath, - to: path.join('/npm', slice.pkg.name, slice.sliceName), - depend: false - }); - } + _.each(self.nodeModulesDirectories, function (nmd) { + builder.copyDirectory({ + from: nmd.sourcePath, + to: nmd.preferredBundlePath, + depend: false + }); }); // Control file @@ -759,6 +790,54 @@ _.extend(ServerTarget.prototype, { }); +//////////////////// InProcessTarget //////////////////// + +var InProcessTarget = function (name, options) { + var self = this; + Target.apply(this, arguments); +}; + +inherits(InProcessTarget, Target); + +_.extend(InProcessTarget.prototype, { + // Load all of the JavaScript in this target into the current process + load: function (builder, nodeModulesMode) { + var self = this; + + // Eval each JavaScript file, providing a 'Npm' symbol in the same + // way that the server environment would + _.each(self.js, function (file) { + var Npm = { + require: function (name) { + if (! file.nodeModulesDirectory) { + // No Npm.depends associated with this package + return require(name); + } + + var nodeModuleDir = + path.join(file.nodeModulesDirectory.sourcePath, name); + + if (fs.existsSync(nodeModuleDir)) { + return require(nodeModuleDir); + } + try { + return require(name); + } catch (e) { + throw new Error("Can't load npm module '" + name + + "' while loading " + file.targetPath + + ". Check your Npm.depends().'"); + } + } + }; + // \n is necessary in case final line is a //-comment + var wrapped = "(function(Npm){" + + file.contents().toString('utf8') + "\n})"; + var func = require('vm').runInThisContext(wrapped, file.targetPath, true); + func(Npm); + }); + } +}); + /////////////////////////////////////////////////////////////////////////////// // writeSiteArchive /////////////////////////////////////////////////////////////////////////////// @@ -986,3 +1065,18 @@ exports.bundle = function (appDir, outputPath, options) { }; } }; + +// Public entry point is unipackage.load, but for now it shares a lot +// of innards with the bundler (this possibly will get factored out at +// some point) so the implementation is here +exports._load = function (library, packages) { + var target = new InProcessTarget("load", { + library: library, + arch: "server" + }); + + target.determineLoadOrder({ packages: packages }); + target.emitResources(); + // I love it when a plan comes together + target.load(); +}; \ No newline at end of file diff --git a/tools/linker.js b/tools/linker.js index 2820d87ded..4d1e28cf71 100644 --- a/tools/linker.js +++ b/tools/linker.js @@ -10,8 +10,11 @@ var packageDot = function (name) { }; var generateBoundary = function () { - // XXX we really want to call Random.id() here but we don't yet have - // infrastructure for including Meteor packages into the tools. + // In a perfect world we would call Packages.random.Random.id(). + // But we can't do that this is part of the code that is used to + // compile and load packages. So let it slide for now and provide a + // version based on (the completely non-cryptographic) Math.random, + // which is good enough for this particular application. var alphabet = "23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz"; var digits = []; for (var i = 0; i < 17; i++) { diff --git a/tools/meteor.js b/tools/meteor.js index a1c66ffe06..5c0dc3977c 100644 --- a/tools/meteor.js +++ b/tools/meteor.js @@ -1030,6 +1030,14 @@ Fiber(function () { if (!files.in_checkout() && !process.env.METEOR_TEST_NO_SPRINGBOARD) toolsSpringboard(); + // Load any needed unipackages + require('./unipackage.js').load({ + library: context.library, + packages: ['random'], // add any unipackages here + release: context.releaseVersion + , + }); + if (argv['get-ready']) { getReady(); return; diff --git a/tools/unipackage.js b/tools/unipackage.js new file mode 100644 index 0000000000..12870562ee --- /dev/null +++ b/tools/unipackage.js @@ -0,0 +1,56 @@ +var _ = require('underscore'); +var library = require('./library.js'); +var bundler = require('./bundler.js'); + +// Load unipackages into the currently running node.js process. Use +// this to use unipackages (such as the DDP client) from command-line +// tools (such as 'meteor'.) The package's exports will be available +// as usual in Package.packagename. They will not be copied into your +// scope. +// +// Currently this may only be called once. This is because in the +// future we want to support packages that have portions that are +// conditionally included (whether slices like 'ddp.server', or units +// like an individual function in DomUtils) and the only way to add +// symbols to package's namespace once it's been initially set up is +// to use eval. We're not quite ready to sign up for eval because we'd +// first want to see how much that usage of it frustrates the +// JIT. (It's also because we currently go through the motions of +// setting up a 'proper' server environment and running any startup +// hooks -- this may or may not be the right call.) +// +// Options: +// - library: The Library to use to retrieve packages and their +// dependencies. Required. +// - packages: The packages to load, as an array of strings. Each +// string may be either "packagename" or "packagename.slice". +// - release: Optional. Not used to load packages! The release name to +// pass into the app with __meteor_runtime_config__ (essentially +// this determines what Meteor.release will return within the loaded +// environment) + +var load = function (options) { + options = options || {}; + if (typeof __meteor_bootstrap__ !== "undefined") + throw new Error("unipackage.load may only be called once"); + if (! (options.library instanceof library.Library)) + throw new Error("unipackage.load requires a library"); + + // Set up a minimal server-like environment (omitting the parts that + // are specific to the HTTP server.) Kind of a hack. I suspect this + // will get refactored before too long. Note that + // __meteor_bootstrap__.require is no longer provided. + __meteor_bootstrap__ = { startup_hooks: [] }; + __meteor_runtime_config__ = { meteorRelease: options.release }; + + // Load the code + bundler._load(options.library, options.packages || []); + + // Run any user startup hooks. + _.each(__meteor_bootstrap__.startup_hooks, function (x) { x(); }); +}; + +var unipackage = exports; +_.extend(exports, { + load: load +}); \ No newline at end of file