github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite bundler and redesign bundle format to support multiple targets. When doing Npm.require on the server, don't infer the node_modules path from the path to the file; read that information out of bundle metadata.

Browse Source
This commit is contained in:
Geoff Schmidt
2013-04-05 02:20:29 -07:00
committed by David Glasser
parent 9d8354b429
commit cb064eca31
7 changed files with 855 additions and 532 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1144
tools/bundler.js
View File

File diff suppressed because it is too large Load Diff

3
tools/library.js
View File

@@ -83,8 +83,7 @@ _.extend(Library.prototype, {
if (name in self.overrides)
packageDir = self.overrides[name];
// Try local directories ('packages' subdirectory in an app,
// PACKAGE_DIRS environment variable, git checkout.)
// Try localPackageDirs
if (! packageDir) {
for (var i = 0; i < self.localPackageDirs.length; ++i) {
var packageDir = path.join(self.localPackageDirs[i], name);

2
tools/meteor.js
View File

@@ -96,7 +96,7 @@ Fiber(function () {
// Let the user provide additional package directories to search
// in PACKAGE_DIRS (colon-separated.)
if (process.env.PACKAGE_DIRS)
localPackageDirs.push.apply(packageDirs,
localPackageDirs.push.apply(localPackageDirs,
process.env.PACKAGE_DIRS.split(':'));
// If we're running out of a git checkout of meteor, use the packages from

57
tools/packages.js
View File

@@ -47,10 +47,10 @@ var Package = function (library) {
// package name to true.
self.unordered = {};
// Files that we want to monitor for changes in development mode, such as
// source files and package.js. Maps relative paths to the SHA of file
// contents. Set only after ensureCompiled().
self.dependencyFileShas = {};
// Files and directories that we want to monitor for changes in
// development mode, such as source files and package.js, in the
// format accepted by watch.Watcher.
self.dependencyInfo = { files: {}, directories: {} };
// All symbols exported from the JavaScript code in this
// package. Map from role to where to array of string symbol (eg
@@ -104,6 +104,10 @@ var Package = function (library) {
// XXX NOTE: this is set by Library reaching into us
self.inWarehouse = false;
// True if we've run installNpmDependencies. (It's slow and there's
// no need to do it more than once.)
self.npmUpdated = false;
// functions that can be called when the package is scanned --
// visible as `Package` when package.js is executed
self.packageFacade = {
@@ -188,16 +192,20 @@ var Package = function (library) {
};
_.extend(Package.prototype, {
// Add a dependency (in the sense of dependencyFileShas) on a
// Add a dependency (in the sense of dependencyInfo) on a
// file. If hash is supplied it should be the sha1 of the file
// contents. If omitted it will be computed. relPath will be
// resolved to an absolute path (relative to self.sourceRoot.)
_addDependency: function (relPath, contents) {
_addDependency: function (relPath, contents, onlyIfExists) {
var self = this;
var absPath = path.resolve(self.sourceRoot, relPath);
if (onlyIfExists && ! fs.existsSync(absPath))
return;
if (! contents)
contents = fs.readFileSync(absPath);
self.dependencyFileShas[absPath] = bundler.sha1(contents);
self.dependencyInfo.files[absPath] = bundler.sha1(contents);
},
// loads a package's package.js file into memory, using
@@ -414,6 +422,7 @@ _.extend(Package.prototype, {
packages = _.union(packages, project.get_packages(appDir));
// XXX this read has a race with the actual read that is used
self._addDependency(path.join(appDir, '.meteor', 'packages'));
self._addDependency(path.join(appDir, '.meteor', 'release'), null, true);
_.each(["use", "test"], function (role) {
_.each(["client", "server"], function (where) {
@@ -439,6 +448,32 @@ _.extend(Package.prototype, {
};
self.forceExports = {use: {client: [], server: []},
test: {client: [], server: []}};
// Directories to monitor for new files
var allExts = [];
_.each(["use", "test"], function (role) {
_.each(["client", "server"], function (where) {
allExts = _.union(allExts, self.registeredExtensions(role, where));
});
});
self.dependencyInfo.directories[path.resolve(appDir)] = {
include: _.map(allExts, function (ext) {
return new RegExp('\\.' + ext.slice(1) + "$");
}),
exclude: ignoreFiles
};
// Inside the packages directory, only look for new packages
// (which we can detect by the appearance of a package.js file.)
// Other than that, packages explicitly call out the files they
// use.
self.dependencyInfo.directories[path.resolve(appDir, 'packages')] = {
include: [ /^package\.js$/ ],
exclude: ignoreFiles
};
// Exclude .meteor/local and everything under it.
self.dependencyInfo.directories[
path.resolve(appDir, '.meteor', 'local')] = { exclude: [/.?/] };
},
// Process all source files through the appropriate handlers and run
@@ -621,18 +656,26 @@ _.extend(Package.prototype, {
// @param npmDependencies {Object} eg {gcd: "0.0.0", tar: "0.1.14"}
installNpmDependencies: function(quiet) {
var self = this;
// Nothing to do if there's no Npm.depends().
if (!self.npmDependencies)
return;
// Warehouse packages come with their NPM dependencies and are read-only.
if (self.inWarehouse)
return;
// No need to do it more than once.
if (self.npmUpdated)
return;
// go through a specialized npm dependencies update process, ensuring we
// don't get new versions of any (sub)dependencies. this process also runs
// mostly safely multiple times in parallel (which could happen if you have
// two apps running locally using the same package)
meteorNpm.updateDependencies(
self.name, self.npmDir(), self.npmDependencies, quiet);
self.npmUpdated = true;
},
npmDir: function () {

126
tools/server/server.js
View File

@@ -24,18 +24,18 @@ if (require('semver').lt(process.version, MIN_NODE_VERSION)) {
// Keepalives so that when the outer server dies unceremoniously and
// doesn't kill us, we quit ourselves. A little gross, but better than
// pidfiles.
var init_keepalive = function () {
var keepalive_count = 0;
var initKeepalive = function () {
var keepaliveCount = 0;
process.stdin.on('data', function (data) {
keepalive_count = 0;
keepaliveCount = 0;
});
process.stdin.resume();
setInterval(function () {
keepalive_count ++;
if (keepalive_count >= 3) {
keepaliveCount ++;
if (keepaliveCount >= 3) {
console.log("Failed to receive keepalive! Exiting.");
process.exit(1);
}
@@ -105,31 +105,14 @@ var categorizeRequest = function (req) {
};
};
// add any runtime configuration options needed to app_html
var runtime_config = function (app_html) {
var insert = '';
if (typeof __meteor_runtime_config__ === 'undefined')
return app_html;
app_html = app_html.replace(
"// ##RUNTIME_CONFIG##",
"__meteor_runtime_config__ = " +
JSON.stringify(__meteor_runtime_config__) + ";");
return app_html;
};
var htmlAttributes = function (app_html, request) {
var htmlAttributes = function (template, request) {
var attributes = '';
_.each(__meteor_bootstrap__.htmlAttributeHooks || [], function (hook) {
var attribute = hook(request);
if (attribute !== null && attribute !== undefined && attribute !== '')
attributes += ' ' + attribute;
});
return app_html.replace('##HTML_ATTRIBUTES##', attributes);
return template.replace('##HTML_ATTRIBUTES##', attributes);
};
// Serve app HTML for this URL?
@@ -156,7 +139,16 @@ var appUrl = function (url) {
}
var run = function () {
var bundle_dir = path.join(__dirname, '..');
var serverDir = __dirname;
// read our control file
var serverJson =
JSON.parse(fs.readFileSync(path.join(serverDir, 'app.json'), 'utf8'));
// read the control for the client we'll be serving up
var clientJsonPath = path.join(serverDir, serverJson.client);
var clientDir = path.dirname(clientJsonPath);
var clientJson = JSON.parse(fs.readFileSync(clientJsonPath, 'utf8'));
// check environment
var port = process.env.PORT ? parseInt(process.env.PORT) : 80;
@@ -167,17 +159,19 @@ var run = function () {
// webserver
var app = connect.createServer();
var static_cacheable_path = path.join(bundle_dir, 'static_cacheable');
if (fs.existsSync(static_cacheable_path))
var staticCacheablePath = path.join(clientDir, clientJson.staticCacheable);
if (staticCacheablePath)
// cacheable files are files that should never change. Typically
// named by their hash (eg meteor bundled js and css files).
// cache them ~forever (1yr)
//
// 'root' option is to work around an issue in connect/gzippo.
// See https://github.com/meteor/meteor/pull/852
app.use(gzippo.staticGzip(static_cacheable_path,
app.use(gzippo.staticGzip(staticCacheablePath,
{clientMaxAge: 1000 * 60 * 60 * 24 * 365,
root: '/'}));
// cache non-cacheable file anyway. This isn't really correct, as
// users can change the files and changes won't propogate
// immediately. However, if we don't cache them, browsers will
@@ -186,22 +180,23 @@ var run = function () {
// bust caches. That way we can both get good caching behavior and
// allow users to change assets without delay.
// https://github.com/meteor/meteor/issues/773
app.use(gzippo.staticGzip(path.join(bundle_dir, 'static'),
{clientMaxAge: 1000 * 60 * 60 * 24,
root: '/'}));
// read bundle config file
var info_raw =
fs.readFileSync(path.join(bundle_dir, 'app.json'), 'utf8');
var info = JSON.parse(info_raw);
var bundle = {manifest: info.manifest, root: bundle_dir};
var staticPath = path.join(clientDir, clientJson.static);
if (staticPath)
app.use(gzippo.staticGzip(staticPath,
{clientMaxAge: 1000 * 60 * 60 * 24,
root: '/'}));
// start up app
__meteor_bootstrap__ = {
startup_hooks: [],
app: app,
// metadata about this bundle
bundle: bundle,
// XXX this could use some refactoring to better distinguish
// server and client
bundle: {
manifest: clientJson.manifest,
root: clientDir
},
// function that takes a connect `req` object and returns a summary
// object with information about the request. See
// #BrowserIdentifcation
@@ -216,38 +211,30 @@ var run = function () {
};
__meteor_runtime_config__ = {};
if (info.release) {
__meteor_runtime_config__.meteorRelease = info.release;
if (serverJson.config && serverJson.config.release) {
__meteor_runtime_config__.meteorRelease = serverJson.config.release;
}
Fiber(function () {
// (put in a fiber to let Meteor.db operations happen during loading)
// load app code
_.each(info.load, function (filename) {
var code = fs.readFileSync(path.join(bundle_dir, filename));
_.each(serverJson.load, function (fileInfo) {
var code = fs.readFileSync(path.join(serverDir, fileInfo.path));
// even though the npm packages are correctly placed in
// node_modules/ relative to the package source, we can't just
// use standard `require` because packages are loaded using
// runInThisContext. see #runInThisContext
var Npm = {
// require an npm module used by your package, or one from the
// dev bundle if you are in an app or your package isn't using
// said npm module
require: function(name) {
var filePathParts = filename.split(path.sep);
// XXX it's weird that we're dependent on the dir structure
if (!(filePathParts[0] === 'app'
&& (filePathParts[1] === 'packages'
|| filePathParts[1] === 'package-tests'))) {
return require(name); // current no support for npm outside packages. load from dev bundle only
require: function (name) {
if (! fileInfo.node_modules) {
// current no support for npm outside packages. load from
// dev bundle only
return require(name);
}
var packageName = filePathParts[2].replace(/\.js$/, '');
var nodeModuleDir = path.join(
__dirname,
'..', // get out of server
'npm', packageName, name);
var nodeModuleDir =
path.join(__dirname, fileInfo.node_modules, name);
if (fs.existsSync(nodeModuleDir)) {
return require(nodeModuleDir);
@@ -255,6 +242,11 @@ var run = function () {
try {
return require(name);
} catch (e) {
// Try to guess the package name so we can print a nice
// error message
var filePathParts = fileInfo.path.split(path.sep);
var packageName = filePathParts[2].replace(/\.js$/, '');
// XXX better message
throw new Error(
"Can't find npm module '" + name +
@@ -265,8 +257,7 @@ var run = function () {
};
// \n is necessary in case final line is a //-comment
var wrapped = "(function(Npm){" + code + "\n})";
// See #runInThisContext
//
// it's tempting to run the code in a new context so we can
// precisely control the enviroment the user code sees. but,
// this is harder than it looks. you get a situation where []
@@ -279,7 +270,7 @@ var run = function () {
// runIn[Foo]Context that causes it to print out a descriptive
// error message on parse error. it's what require() uses to
// generate its errors.
var func = require('vm').runInThisContext(wrapped, filename, true);
var func = require('vm').runInThisContext(wrapped, fileInfo.path, true);
// Setting `this` to `global` allows you to do a top-level
// "this.foo = " to define global variables when using "use strict"
// (http://es5.github.io/#x15.3.4.4); this is the only way to do
@@ -291,9 +282,12 @@ var run = function () {
// Actually serve HTML. This happens after user code, so that
// packages can insert connect middlewares and update
// __meteor_runtime_config__
var app_html = fs.readFileSync(path.join(bundle_dir, 'app.html'), 'utf8');
app_html = runtime_config(app_html);
var boilerplateHtmlPath = path.join(clientDir, clientJson.page);
var boilerplateHtml =
fs.readFileSync(boilerplateHtmlPath, 'utf8').replace(
"// ##RUNTIME_CONFIG##",
"__meteor_runtime_config__ = " +
JSON.stringify(__meteor_runtime_config__) + ";");
app.use(function (req, res, next) {
if (! appUrl(req.url))
@@ -303,7 +297,7 @@ var run = function () {
res.writeHead(200, {'Content-Type': 'text/html; charset=utf-8'});
var requestSpecificHtml = htmlAttributes(app_html, request);
var requestSpecificHtml = htmlAttributes(boilerplateHtml, request);
res.write(requestSpecificHtml);
res.end();
});
@@ -326,7 +320,7 @@ var run = function () {
}).run();
if (argv.keepalive)
init_keepalive();
initKeepalive();
};
run();

15
tools/tests/test_bundler_npm.js
View File

@@ -6,6 +6,13 @@ var meteorNpm = require(path.join(__dirname, '..', 'meteor_npm.js'));
var tmpPackageDirContainer = tmpDir();
var testPackageDir = path.join(tmpPackageDirContainer, 'test-package');
lib = new library.Library({
localPackageDirs: [
tmpPackageDirContainer,
path.join(files.getCurrentToolsDir(), 'packages')
]
});
var updateTestPackage = function(npmDependencies) {
if (!fs.existsSync(testPackageDir))
fs.mkdirSync(testPackageDir);
@@ -100,9 +107,9 @@ var _assertCorrectPackageNpmDir = function(deps) {
var _assertCorrectBundleNpmContents = function(bundleDir, deps) {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(bundleDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
var bundledPackageNodeModulesDir = path.join(bundleDir, 'npm', 'test-package');
var bundledPackageNodeModulesDir = path.join(bundleDir, 'programs', 'server', 'npm', 'test-package');
// bundle actually has the npm modules
_.each(deps, function(version, name) {
@@ -128,8 +135,6 @@ var looksInstalled = function (nodeModulesDir, name) {
/// TESTS
///
var lib = new library.Library();
console.log("app that uses gcd - clean run");
assert.doesNotThrow(function () {
updateTestPackage({gcd: '0.0.0'});
@@ -158,6 +163,7 @@ assert.doesNotThrow(function () {
assert(fs.existsSync(path.join(nodeModulesDir)));
files.rm_recursive(nodeModulesDir);
assert(!fs.existsSync(path.join(nodeModulesDir)));
lib.refresh();
// while bundling, verify that we don't call `npm install
// name@version unnecessarily` -- calling `npm install` is enough,
@@ -199,6 +205,7 @@ assert.doesNotThrow(function () {
files.rm_recursive(nodeModulesMimeDir);
assert(!fs.existsSync(path.join(nodeModulesMimeDir)));
lib.refresh();
var result = bundler.bundle(appWithPackageDir, tmpOutputDir, {nodeModulesMode: 'skip', releaseStamp: 'none', library: lib});
assert.strictEqual(result.errors, false, result.errors && result.errors[0]);
_assertCorrectPackageNpmDir({gcd: '0.0.0', mime: '1.2.7', semver: '1.1.0'});

40
tools/tests/test_bundler_options.js
View File

@@ -10,7 +10,9 @@ var emptyAppDir = path.join(__dirname, 'empty-app');
/// TESTS
///
var lib = new library.Library();
lib = new library.Library({
localPackageDirs: [ path.join(files.getCurrentToolsDir(), 'packages') ]
});
console.log("nodeModules: 'skip'");
assert.doesNotThrow(function () {
@@ -20,16 +22,18 @@ assert.doesNotThrow(function () {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
// no top level node_modules directory
assert(!fs.existsSync(path.join(tmpOutputDir, "server", "node_modules")));
assert(!fs.existsSync(path.join(tmpOutputDir,
"programs", "server", "node_modules")));
// yes package node_modules directory
assert(fs.lstatSync(path.join(
tmpOutputDir, "npm", "livedata"))
tmpOutputDir, "programs", "server", "npm", "livedata"))
.isDirectory());
// verify that contents are minified
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html"));
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs",
"client", "app.html"));
assert(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml));
assert(!(/src=\"\/packages/.test(appHtml)));
});
@@ -42,14 +46,15 @@ assert.doesNotThrow(function () {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
// verify that contents are not minified
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html"));
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs",
"client", "app.html"));
assert(!(/src=\"\/[0-9a-f]{40,40}.js\"/.test(appHtml)));
assert(/src=\"\/packages\/meteor/.test(appHtml));
assert(/src=\"\/packages\/deps/.test(appHtml));
// verify that tests aren't included
assert(!(/src=\"\/packages\/meteor\/url_tests.js/.test(appHtml)));
assert(!(/src=\"\/package-tests\/meteor/.test(appHtml)));
});
console.log("nodeModules: 'skip', no minify, testPackages: ['meteor']");
@@ -61,9 +66,10 @@ assert.doesNotThrow(function () {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
// verify that tests for the meteor package are included
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "app.html"));
var appHtml = fs.readFileSync(path.join(tmpOutputDir, "programs",
"client", "app.html"));
assert(/src=\"\/package-tests\/meteor.js/.test(appHtml));
});
@@ -75,11 +81,11 @@ assert.doesNotThrow(function () {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
// node_modules directory exists and is not a symlink
assert(!fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink());
assert(!fs.lstatSync(path.join(tmpOutputDir, "programs", "server", "node_modules")).isSymbolicLink());
// node_modules contains fibers
assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers")));
assert(fs.existsSync(path.join(tmpOutputDir, "programs", "server", "node_modules", "fibers")));
});
console.log("nodeModules: 'symlink'");
@@ -90,14 +96,14 @@ assert.doesNotThrow(function () {
// sanity check -- main.js has expected contents.
assert.strictEqual(fs.readFileSync(path.join(tmpOutputDir, "main.js"), "utf8").trim(),
"require('./server/server.js');");
"require('./programs/server/server.js');");
// node_modules directory exists and is a symlink
assert(fs.lstatSync(path.join(tmpOutputDir, "server", "node_modules")).isSymbolicLink());
assert(fs.lstatSync(path.join(tmpOutputDir, "programs", "server", "node_modules")).isSymbolicLink());
// node_modules contains fibers
assert(fs.existsSync(path.join(tmpOutputDir, "server", "node_modules", "fibers")));
assert(fs.existsSync(path.join(tmpOutputDir, "programs", "server", "node_modules", "fibers")));
// package node_modules directory also a symlink
assert(fs.lstatSync(path.join(
tmpOutputDir, "npm", "livedata"))
tmpOutputDir, "programs", "server", "npm", "livedata"))
.isSymbolicLink());
});