github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Refactor to create a concept of a serializable Plugin that can be loaded into a host nodejs process, and can be used for both general-purpose unipackage loading (eg, for DDP) and for bundler plugins.

Browse Source
This commit is contained in:
Geoff Schmidt
2013-04-21 06:07:03 -07:00
committed by David Glasser
parent 193adbea59
commit c599601ccf
4 changed files with 411 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

454
tools/bundler.js
View File

@@ -19,6 +19,13 @@
// be the principled mechanism by which a server program could read
// a client program so it can server it)
//
// - plugins: array of plugins in the star, each an object:
// - name: short, unique name for plugin, for referring to it
// programmatically
// - arch: typically 'js' (for a portable plugin) or eg
// 'js.linux.x86_64' for one that include native node_modules
// - path: directory (relative to star.json) containing this plugin
//
// /README: human readable instructions
//
// /main.js: script that can be run in node.js to start the site
@@ -122,6 +129,26 @@
// /node_modules: node_modules needed for server.js. omitted if
// deploying (see .bundle_version.txt above), copied if bundling,
// symlinked if developing locally.
//
//
// == Format of a program that is to be used as a plugin ==
//
// /program.json:
// - load: array with each item describing a JS file to load, in load order:
// - path: path of file, relative to program.json
// - node_modules: if Npm.require is called from this file, this is
// the path (relative to program.json) of the directory that should
// be search for npm modules
//
// Note that while the spec for "native.*" is going to change to
// represent an arbitrary POSIX (or Windows) process rather than
// assuming a nodejs host, these plugins will always refer to
// JavaScript code (that potentially might be a plugin to be loaded
// into an existing JS VM). But this seems to be a concern that is
// somewhat orthogonal to arch (these plugins can still use packages
// of arch "native.*".) There is probably a missing abstraction here
// somewhere (decoupling target type from architecture) but it can
// wait until later.
var path = require('path');
var files = require(path.join(__dirname, 'files.js'));
@@ -161,8 +188,8 @@ var inherits = function (child, parent) {
var NodeModulesDirectory = function (options) {
var self = this;
// The absolute path (on local disk at build time) to a directory
// that contains the built node_modules to use.
// The absolute path (on local disk) to a directory that contains
// the built node_modules to use.
self.sourcePath = options.sourcePath;
// The path (relative to the bundle root) where we would preferably
@@ -282,21 +309,16 @@ _.extend(File.prototype, {
///////////////////////////////////////////////////////////////////////////////
// options:
// - library: package library to use for resolving package dependenices
// - arch: the architecture to build
//
// see subclasses for additional options
var Target = function (name, options) {
var Target = function (options) {
var self = this;
// Package library to use for resolving package dependenices.
self.library = options.library;
// A name for this target.
self.name = name;
// Path of this target in the bundle, relative to the root of the bundle.
self.pathInBundle = path.join('programs', self.name);
// Something like "browser.w3c" or "native" or "native.osx.x86_64"
self.arch = options.arch;
@@ -530,7 +552,7 @@ _.extend(Target.prototype, {
//////////////////// ClientTarget ////////////////////
var ClientTarget = function (name, options) {
var ClientTarget = function (options) {
var self = this;
Target.apply(this, arguments);
@@ -706,7 +728,7 @@ _.extend(ClientTarget.prototype, {
// options:
// - clientTarget: the ClientTarget to serve up over HTTP as our client
// - releaseStamp: the Meteor release name (for retrieval at runtime)
var ServerTarget = function (name, options) {
var ServerTarget = function (options) {
var self = this;
Target.apply(this, arguments);
@@ -719,42 +741,90 @@ var ServerTarget = function (name, options) {
inherits(ServerTarget, Target);
// Code factored out of ServerTarget.write and Plugin.write
// Options:
// - load: array of objects with keys targetPath, data, nodeModulesDirectory
// - nodeModulesDirectories: array of NodeModulesDirectory referenced
// - extraControlInfo: extra keys for program.json
var writeServerTargetOrPlugin = function (builder, options) {
// Finalize choice of paths for node_modules directories -- These
// paths are no longer just "preferred"; they are the final paths
// that we will use
var nodeModulesDirectories = [];
_.each(options.nodeModulesDirectories || [], function (nmd) {
nodeModulesDirectories.push(new NodeModulesDirectory({
sourcePath: nmd.sourcePath,
preferredBundlePath: builder.generateFilename(nmd.preferredBundlePath,
{ directory: true })
}));
});
// JavaScript sources
var load = [];
_.each(options.load || [], function (item) {
if (! item.targetPath)
throw new Error("No targetPath?");
builder.write(item.targetPath, { data: item.data });
load.push({
path: item.targetPath,
node_modules: item.nodeModulesDirectory ?
item.nodeModulesDirectory.preferredBundlePath : undefined
});
});
// node_modules resources from the packages. Due to appropriate
// builder configuration, 'meteor bundle' and 'meteor deploy' copy
// them, and 'meteor run' symlinks them. If these contain
// arch-specific code then the target will end up having an
// appropriately specific arch.
_.each(nodeModulesDirectories, function (nmd) {
builder.copyDirectory({
from: nmd.sourcePath,
to: nmd.preferredBundlePath,
depend: false
});
});
// Control file
var json = _.extend({
load: load
}, options.extraControlInfo || {});
builder.writeJson('program.json', json);
};
_.extend(ServerTarget.prototype, {
// Output the finished target to disk
write: function (builder, nodeModulesMode) {
// options:
// - omitDependencyKit: if true, don't copy node_modules from dev_bundle
// - getRelativeTargetPath: a function that takes {forTarget:
// Target, relativeTo: Target} and return the path of one target
// in the bundle relative to another. hack to get the path of the
// client target.. we'll find a better solution here eventually
write: function (builder, options) {
var self = this;
var json = {
load: [],
client: path.join(path.relative(self.pathInBundle,
self.clientTarget.pathInBundle),
'program.json'),
config: {
meteorRelease: self.releaseStamp && self.releaseStamp !== "none" ?
self.releaseStamp : undefined
if (! options.omitDependencyKit)
builder.reserve("node_modules", { directory: true });
writeServerTargetOrPlugin(builder, {
load: _.map(self.js, function (file) {
return {
targetPath: file.targetPath,
data: file.contents(),
nodeModulesDirectory: file.nodeModulesDirectory
};
}),
nodeModulesDirectories: self.nodeModulesDirectories,
extraControlInfo: {
client: path.join(options.getRelativeTargetPath({
forTarget: self.clientTarget, relativeTo: self}),
'program.json'),
config: {
meteorRelease: self.releaseStamp && self.releaseStamp !== "none" ?
self.releaseStamp : undefined
}
}
};
// Finalize choice of paths for node_modules directories -- These
// paths are no longer just "preferred"; they are the final paths
// that we will use
_.each(self.nodeModulesDirectories, function (nmd) {
nmd.preferredBundlePath =
builder.generateFilename(nmd.preferredBundlePath, { directory: true });
});
// JavaScript sources
_.each(self.js, function (file) {
if (! file.targetPath)
throw new Error("No targetPath?");
builder.write(file.targetPath, { data: file.contents() });
json.load.push({
path: file.targetPath,
node_modules: file.nodeModulesDirectory ?
file.nodeModulesDirectory.preferredBundlePath : undefined
});
});
// Server driver
@@ -769,7 +839,7 @@ _.extend(ServerTarget.prototype, {
// kit. This one is copied in 'meteor bundle', symlinked in
// 'meteor run', and omitted by 'meteor deploy' (Galaxy provides a
// version that's appropriate for the server architecture.)
if (nodeModulesMode !== "skip") {
if (! options.omitDependencyKit) {
builder.copyDirectory({
from: path.join(files.get_dev_bundle(), 'lib', 'node_modules'),
to: 'node_modules',
@@ -777,73 +847,162 @@ _.extend(ServerTarget.prototype, {
depend: false
});
}
// Extra node_modules resources from the packages. 'meteor bundle'
// and 'meteor deploy' copy them, and 'meteor run' symlinks
// them. If these contain arch-specific code then the target will
// end up having an appropriately specific arch.
_.each(self.nodeModulesDirectories, function (nmd) {
builder.copyDirectory({
from: nmd.sourcePath,
to: nmd.preferredBundlePath,
depend: false
});
});
// Control file
builder.writeJson('program.json', json);
}
});
//////////////////// InProcessTarget ////////////////////
//////////////////// PluginTarget and Plugin ////////////////////
var InProcessTarget = function (name, options) {
var Plugin = function () {
var self = this;
// Array of objects with keys:
// - targetPath: relative path to use if saved to disk (or for stack traces)
// - source: JS source code to load, as a string
// - nodeModulesDirectory: a NodeModulesDirectory indicating which
// directory should be searched by Npm.require()
// note: this can't be called `load` at it would shadow `load()`
self.jsToLoad = [];
// node_modules directories that we need to copy into the target (or
// otherwise make available at runtime.) A map from an absolute path
// on disk (NodeModulesDirectory.sourcePath) to a
// NodeModulesDirectory object that we have created to represent it.
self.nodeModulesDirectories = {};
};
_.extend(Plugin.prototype, {
// Load the plugin into the current process. It gets its own unique
// Package object containing its own private copy of every
// unipackage that it uses. This Package object is returned.
//
// If `bindings` is provided, it is a object containing a set of
// variables to set in the global environment of the executed
// code. The keys are the variable names and the values are their
// values. In addition to the contents of `bindings`, Package and
// Npm will be provided.
//
// XXX throw an error if the plugin includes any "app-style" code
// that is built to put symbols in the global namespace rather than
// in a compartment of Package
load: function (bindings) {
var self = this;
var ret = {};
// Eval each JavaScript file, providing a 'Npm' symbol in the same
// way that the server environment would, and a 'Package' symbol
// so the plugin has its own private universe of loaded packages
_.each(self.jsToLoad, function (item) {
var env = _.extend({
Package: ret,
Npm: {
require: function (name) {
if (! item.nodeModulesDirectory) {
// No Npm.depends associated with this package
return require(name);
}
var nodeModuleDir =
path.join(item.nodeModulesDirectory.sourcePath, name);
if (fs.existsSync(nodeModuleDir)) {
return require(nodeModuleDir);
}
try {
return require(name);
} catch (e) {
throw new Error("Can't load npm module '" + name +
"' while loading " + item.targetPath +
". Check your Npm.depends().'");
}
}
}
}, bindings || {});
// \n is necessary in case final line is a //-comment
var wrapped = "(function(" + _.keys(env).join(",") + "){" +
item.source + "\n})";
var func = require('vm').runInThisContext(wrapped, item.targetPath, true);
func.apply({}, _.values(env));
});
return ret;
},
// Write this plugin out to disk
write: function (builder) {
var self = this;
writeServerTargetOrPlugin(builder, {
load: _.map(self.jsToLoad, function (item) {
return {
targetPath: item.targetPath,
source: new Buffer(item.source, 'utf8'),
nodeModulesDirectory: file.nodeModulesDirectory
};
}),
nodeModulesDirectories: self.nodeModulesDirectories
});
},
// `dir` is a directory on disk that contains a program with arch
// matching js.* (eg, a plugin previously written out with write())
initFromDisk: function (dir) {
var self = this;
var json =
JSON.parse(fs.readFileSync(path.join(dir, 'program.json')));
_.each(json.load, function (item) {
if (item.path.match(/\.\./))
throw new Error("bad path in plugin bundle");
var nmd = undefined;
if (item.node_modules) {
var node_modules = path.join(dir, item.node_modules);
if (! node_modules in self.nodeModulesDirectories)
self.nodeModulesDirectories[node_modules] =
new NodeModulesDirectory({
sourcePath: node_modules,
preferredBundlePath: item.node_modules
});
nmd = self.nodeModulesDirectories[node_modules];
}
self.jsToLoad.push({
targetPath: item.path,
source: fs.readFileSync(path.join(dir, item.path)),
nodeModulesDirectory: nmd
});
});
}
});
var PluginTarget = function (options) {
var self = this;
Target.apply(this, arguments);
if (! archinfo.matches(self.arch, "native"))
throw new Error("InProcessTarget targeting something incompatible?");
throw new Error("PluginTarget targeting something incompatible?");
};
inherits(InProcessTarget, Target);
inherits(PluginTarget, Target);
_.extend(InProcessTarget.prototype, {
// Load all of the JavaScript in this target into the current process
load: function (builder, nodeModulesMode) {
_.extend(PluginTarget.prototype, {
toPlugin: function () {
var self = this;
var ret = new Plugin;
// Eval each JavaScript file, providing a 'Npm' symbol in the same
// way that the server environment would
_.each(self.js, function (file) {
var Npm = {
require: function (name) {
if (! file.nodeModulesDirectory) {
// No Npm.depends associated with this package
return require(name);
}
var nodeModuleDir =
path.join(file.nodeModulesDirectory.sourcePath, name);
if (fs.existsSync(nodeModuleDir)) {
return require(nodeModuleDir);
}
try {
return require(name);
} catch (e) {
throw new Error("Can't load npm module '" + name +
"' while loading " + file.targetPath +
". Check your Npm.depends().'");
}
}
};
// \n is necessary in case final line is a //-comment
var wrapped = "(function(Npm){" +
file.contents().toString('utf8') + "\n})";
var func = require('vm').runInThisContext(wrapped, file.targetPath, true);
func(Npm);
ret.jsToLoad.push({
targetPath: file.targetPath,
source: file.contents().toString('utf8'),
nodeModulesDirectory: file.nodeModulesDirectory
});
});
ret.nodeModulesDirectories = self.nodeModulesDirectories;
return ret;
}
});
@@ -851,9 +1010,10 @@ _.extend(InProcessTarget.prototype, {
// writeSiteArchive
///////////////////////////////////////////////////////////////////////////////
// targets is an array of Targets to include in the bundle. outputPath
// is the path of a directory that should be created to contain the
// generated site archive.
// targets is a set of Targets to include in the bundle, as a map from
// target name (to use in the bundle) to a Target. outputPath is the
// path of a directory that should be created to contain the generated
// site archive.
//
// Returns dependencyInfo (in the format expected by watch.Watcher)
// for all files and directories that ultimately went into the bundle.
@@ -874,14 +1034,47 @@ var writeSiteArchive = function (targets, outputPath, options) {
programs: []
};
// Pick a path in the bundle for each target
var paths = {};
_.each(targets, function (target, name) {
var p = path.join('programs', name);
builder.reserve(p, { directory: true });
paths[name] = p;
});
// Hack to let servers find relative paths to clients. Should find
// another solution eventually (probably some kind of mount
// directive that mounts the client bundle in the server at runtime)
var getRelativeTargetPath = function (options) {
var pathForTarget = function (target) {
var name;
_.each(targets, function (t, n) {
if (t === target)
name = n;
});
if (! name)
throw new Error("missing target?");
if (! (name in paths))
throw new Error("missing target path?");
return paths[name];
};
return path.relative(pathForTarget(options.relativeTo),
pathForTarget(options.forTarget));
};
// Write out each target
_.each(targets, function (target) {
target.pathInBundle = path.join('programs', target.name);
target.write(builder.enter(target.pathInBundle), options.nodeModulesMode);
_.each(targets, function (target, name) {
target.pathInBundle = path.join('programs', name);
target.write(builder.enter(paths[name]),
{ omitDependencyKit: options.nodeModulesMode === "skip",
getRelativeTargetPath: getRelativeTargetPath });
json.programs.push({
name: target.name,
name: name,
arch: target.mostCompatibleArch(),
path: target.pathInBundle
path: paths[name]
});
});
@@ -923,7 +1116,7 @@ var writeSiteArchive = function (targets, outputPath, options) {
// bundle. A naive merge like this doesn't work in general but
// should work in this case.
var fileDeps = {}, directoryDeps = {};
var dependencySources = targets.concat([builder]);
var dependencySources = [builder].concat(_.values(targets));
_.each(dependencySources, function (s) {
var info = s.getDependencyInfo();
_.extend(fileDeps, info.files);
@@ -1008,17 +1201,20 @@ exports.bundle = function (appDir, outputPath, options) {
try {
// Create targets
var client = new ClientTarget("client", {
var client = new ClientTarget({
library: library,
arch: "browser"
});
var server = new ServerTarget("server", {
var server = new ServerTarget({
library: library,
arch: archinfo.host(),
clientTarget: client,
releaseStamp: options.releaseStamp
});
var targets = [client, server];
var targets = {
client: client,
server: server
};
// Create a Package object that represents the app
var app = library.getForApp(appDir, ignoreFiles);
@@ -1075,17 +1271,43 @@ exports.bundle = function (appDir, outputPath, options) {
}
};
// Public entry point is unipackage.load, but for now it shares a lot
// of innards with the bundler (this possibly will get factored out at
// some point) so the implementation is here
exports._load = function (library, packages) {
var target = new InProcessTarget("load", {
library: library,
// Return a Plugin object. It can either be loaded into memory with
// load(), which returns the `Package` object inside the plugin's
// namespace, or saved to disk with write(builder).
//
// options:
// - library: required. the Library for resolving package dependencies
// - use: list of packages to use in the plugin, as strings (foo or foo.bar)
// - sources: source files to use (paths on local disk)
// - npmDependencies: map from npm package name to required version
// - npmDir: where to keep the npm cache and npm version shrinkwrap
// info. required if npmDependencies present.
exports.buildPlugin = function (options) {
if (options.npmDependencies && ! options.npmDir)
throw new Error("Must indicate .npm directory to use");
var pkg = new packages.Package(options.library);
// It would be nice to have a way to say "make this package
// anonymous" without also saying "make its namespace the same as
// the global namespace." Though it would be an easy refactor, we
// don't have that yet, so just make up a random name.
var pkgName = "plugin" + Math.floor(Math.random() * 100000);
pkg.initFromOptions(pkgName, {
sliceName: "plugin",
use: options.use || [],
sources: options.sources || [],
npmDependencies: options.npmDependencies,
npmDir: options.npmDir
});
var target = new PluginTarget({
library: options.library,
arch: archinfo.host()
});
target.determineLoadOrder({ packages: packages });
target.determineLoadOrder({ packages: [pkg] });
target.emitResources();
// I love it when a plan comes together
target.load();
};
return target.toPlugin();
};

7
tools/meteor.js
View File

@@ -1062,13 +1062,6 @@ Fiber(function () {
if (!files.in_checkout() && !process.env.METEOR_TEST_NO_SPRINGBOARD)
toolsSpringboard();
// Load any needed unipackages
require('./unipackage.js').load({
library: context.library,
packages: [], // add any unipackages here
release: context.releaseVersion
});
if (argv['get-ready']) {
getReady();
return;

50
tools/packages.js
View File

@@ -404,16 +404,17 @@ var nextPackageId = 1;
var Package = function (library) {
var self = this;
// Fields set by init_*:
// name: package name, or null for an app pseudo-package or collection
// sourceRoot: base directory for resolving source files, null for collection
// serveRoot: base directory for serving files, null for collection
// A unique ID (guaranteed to not be reused in this process -- if
// the package is reloaded, it will get a different id the second
// time)
self.id = nextPackageId++;
// The name of the package, or null for an app pseudo-package or
// collection. The package's exports will reside in Package.<name>.
// When it is null it is linked like an application instead of like
// a package.
self.name = null;
// The path from which this package was loaded. null if loaded from
// unipackage.
self.sourceRoot = null;
@@ -514,6 +515,45 @@ _.extend(Package.prototype, {
preheat: function () {
},
// Programmatically create a package from scratch. For now, cannot
// create browser packages.
//
// Options:
// - sliceName
// - use
// - sources
// - npmDependencies
// - npmDir
initFromOptions: function (name, options) {
var self = this;
self.name = name;
var isPortable = true;
var nodeModulesPath = null;
if (options.npmDependencies) {
meteorNpm.ensureOnlyExactVersions(option.npmDependencies);
meteorNpm.updateDependencies(name, options.npmDir,
options.npmDependencies);
if (! meteorNpm.dependenciesArePortable(options.npmDir))
isPortable = false;
nodeModulesPath = path.join(options.npmDir, 'node_modules');
}
var arch = isPortable ? "native" : archinfo.host();
var slice = new Slice(self, {
name: options.sliceName,
arch: arch,
uses: _.map(["meteor"].concat(options.use || []), function (spec) {
return { spec: spec }
}),
sources: options.sources || [],
nodeModulesPath: nodeModulesPath
});
self.slices.push(slice);
self.defaultSlices = {'native': [options.sliceName]};
},
// loads a package's package.js file into memory, using
// runInThisContext. Wraps the contents of package.js in a closure,
// supplying pseudo-globals 'Package' and 'Npm'.

49
tools/unipackage.js
View File

@@ -4,20 +4,12 @@ var bundler = require('./bundler.js');
// Load unipackages into the currently running node.js process. Use
// this to use unipackages (such as the DDP client) from command-line
// tools (such as 'meteor'.) The package's exports will be available
// as usual in Package.packagename. They will not be copied into your
// scope.
//
// Currently this may only be called once. This is because in the
// future we want to support packages that have portions that are
// conditionally included (whether slices like 'ddp.server', or units
// like an individual function in DomUtils) and the only way to add
// symbols to package's namespace once it's been initially set up is
// to use eval. We're not quite ready to sign up for eval because we'd
// first want to see how much that usage of it frustrates the
// JIT. (It's also because we currently go through the motions of
// setting up a 'proper' server environment and running any startup
// hooks -- this may or may not be the right call.)
// tools (such as 'meteor'.) The requested packages will be loaded
// together will all of their dependencies, and each time you call
// this function you load another, distinct copy of all of the
// packages. The return value is an object that maps package name to
// package exports (that is, it is the Package object from inside the
// sandbox created for the newly loaded packages.)
//
// Options:
// - library: The Library to use to retrieve packages and their
@@ -28,11 +20,18 @@ var bundler = require('./bundler.js');
// pass into the app with __meteor_runtime_config__ (essentially
// this determines what Meteor.release will return within the loaded
// environment)
//
// Example usage:
// var Meteor = require('./unipackage.js').load({
// library: context.library,
// packages: ['livedata'],
// release: context.releaseVersion
// }).meteor.Meteor;
// var reverse = Meteor.connect('reverse.meteor.com');
// console.log(reverse.call('reverse', 'hello world'));
var load = function (options) {
options = options || {};
if (typeof __meteor_bootstrap__ !== "undefined")
throw new Error("unipackage.load may only be called once");
if (! (options.library instanceof library.Library))
throw new Error("unipackage.load requires a library");
@@ -40,17 +39,25 @@ var load = function (options) {
// are specific to the HTTP server.) Kind of a hack. I suspect this
// will get refactored before too long. Note that
// __meteor_bootstrap__.require is no longer provided.
__meteor_bootstrap__ = { startup_hooks: [] };
__meteor_runtime_config__ = { meteorRelease: options.release };
var env = {
__meteor_bootstrap__: { startup_hooks: [] },
__meteor_runtime_config__: { meteorRelease: options.release }
};
// Load the code
bundler._load(options.library, options.packages || []);
var plugin = bundler.buildPlugin({
library: options.library,
use: options.packages || []
});
var ret = plugin.load(env);
// Run any user startup hooks.
_.each(__meteor_bootstrap__.startup_hooks, function (x) { x(); });
_.each(env.__meteor_bootstrap__.startup_hooks, function (x) { x(); });
return ret;
};
var unipackage = exports;
_.extend(exports, {
load: load
});
});