github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'sourcemaps' into linker

Browse Source
This commit is contained in:
David Glasser
2013-07-12 12:31:59 -07:00
parent 82552022d8 7b85bbf99a
commit 3e2eb429ed
24 changed files with 1174 additions and 688 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
meteor
View File

@@ -1,6 +1,6 @@
#!/bin/bash
BUNDLE_VERSION=0.3.10
BUNDLE_VERSION=0.3.11
# OS Check. Put here because here is where we download the precompiled
# bundles that are arch specific.

8
packages/coffeescript/.npm/plugin/compileCoffeescript/npm-shrinkwrap.json generated
View File

@@ -2,6 +2,14 @@
"dependencies": {
"coffee-script": {
"version": "1.6.3"
},
"source-map": {
"version": "0.1.24",
"dependencies": {
"amdefine": {
"version": "0.0.5"
}
}
}
}
}

2
packages/coffeescript/package.js
View File

@@ -8,7 +8,7 @@ Package._transitional_registerBuildPlugin({
sources: [
'plugin/compile-coffeescript.js'
],
npmDependencies: {"coffee-script": "1.6.3"}
npmDependencies: {"coffee-script": "1.6.3", "source-map": "0.1.24"}
});
Package.on_test(function (api) {

90
packages/coffeescript/plugin/compile-coffeescript.js
View File

@@ -2,6 +2,7 @@ var fs = Npm.require('fs');
var path = Npm.require('path');
var coffee = Npm.require('coffee-script');
var _ = Npm.require('underscore');
var sourcemap = Npm.require('source-map');
var stripExportedVars = function (source, exports) {
if (!exports || _.isEmpty(exports))
@@ -28,33 +29,50 @@ var stripExportedVars = function (source, exports) {
// XXX relax these assumptions by doing actual JS parsing (eg with jsparse).
// I'd do this now, but there's no easy way to "unparse" a jsparse AST.
var foundVarLine = false;
lines = _.map(lines, function (line) {
if (foundVarLine)
return line;
for (var i = 0; i < lines.length; i++) {
var line = lines[i];
var match = /^var (.+)([,;])$/.exec(line);
if (!match)
return line;
foundVarLine = true;
continue;
// If there's an assignment on this line, we assume that there are ONLY
// assignments and that the var we are looking for is not declared. (Part
// of our strong assumption about the layout of this code.)
if (match[1].indexOf('=') !== -1)
return line;
continue;
// We want to replace the line with something no shorter, so that all
// records in the source map continue to point at valid
// characters.
var replaceLine = function (x) {
if (x.length >= lines[i].length) {
lines[i] = x;
} else {
lines[i] = x + new Array(1 + (lines[i].length - x.length)).join(' ');
}
};
var vars = match[1].split(', ');
vars = _.difference(vars, exports);
if (!_.isEmpty(vars))
return "var " + vars.join(', ') + match[2];
// We got rid of all the vars on this line. Drop the whole line if this
// didn't continue to the next line, otherwise keep just the 'var '.
return match[2] === ';' ? '' : 'var';
});
if (!_.isEmpty(vars)) {
replaceLine("var " + vars.join(', ') + match[2]);
} else {
// We got rid of all the vars on this line. Drop the whole line if this
// didn't continue to the next line, otherwise keep just the 'var '.
if (match[2] === ';')
replaceLine('');
else
replaceLine('var');
}
break;
}
return lines.join('\n');
};
var addSharedHeader = function (source) {
var addSharedHeader = function (source, sourceMap) {
var sourceMapJSON = JSON.parse(sourceMap);
// We want the symbol "share" to be visible to all CoffeeScript files in the
// package (and shared between them), but not visible to JavaScript
// files. (That's because we don't want to introduce two competing ways to
@@ -73,17 +91,42 @@ var addSharedHeader = function (source) {
// If the file begins with "use strict", we need to keep that as the first
// statement.
return source.replace(/^(?:(['"])use strict\1;\n)?/, function (match) {
return match + header;
source = source.replace(/^(?:((['"])use strict\2;)\n)?/, function (match, useStrict) {
if (match) {
// There's a "use strict"; we keep this as the first statement and insert
// our header at the end of the line that it's on. This doesn't change
// line numbers or the part of the line that previous may have been
// annotated, so we don't need to update the source map.
return useStrict + " " + header;
} else {
// There's no use strict, so we can just add the header at the very
// beginning. This adds a line to the file, so we update the source map to
// add a single un-annotated line to the beginning.
sourceMapJSON.mappings = ";" + sourceMapJSON.mappings;
return header;
}
});
return {
source: source,
sourceMap: JSON.stringify(sourceMapJSON)
};
};
var handler = function (compileStep) {
var source = compileStep.read().toString('utf8');
var outputFile = compileStep.inputPath + ".js";
var options = {
bare: true,
filename: compileStep.inputPath,
literate: path.extname(compileStep.inputPath) === '.litcoffee'
literate: path.extname(compileStep.inputPath) === '.litcoffee',
// Return a source map.
sourceMap: true,
// Include the original source in the source map (sourcesContent field).
inline: true,
// This becomes the "file" field of the source map.
generatedFile: "/" + outputFile,
// This becomes the "sources" field of the source map.
sourceFiles: [compileStep.pathForSourceMap]
};
try {
@@ -98,13 +141,14 @@ var handler = function (compileStep) {
}
compileStep.addJavaScript({
path: compileStep.inputPath + ".js",
path: outputFile,
sourcePath: compileStep.inputPath,
data: output,
lineForLine: false,
linkerUnitTransform: function (source, exports) {
return addSharedHeader(stripExportedVars(source, exports));
}
data: output.js,
linkerFileTransform: function (sourceWithMap, exports) {
var stripped = stripExportedVars(sourceWithMap.source, exports);
return addSharedHeader(stripped, sourceWithMap.sourceMap);
},
sourceMap: output.v3SourceMap
});
};

40
packages/http/httpcall_tests.js
View File

@@ -441,33 +441,37 @@ if (Meteor.isServer) {
}));
};
// no such file
do_test("/nosuchfile", 200, /DOCTYPE/);
do_test("/../nosuchfile", 403);
do_test("/%2e%2e/nosuchfile", 403);
do_test("/%2E%2E/nosuchfile", 403);
do_test("/%2d%2d/nosuchfile", 200, /DOCTYPE/);
// existing static file
var succeeds = [
"/packages/http/test_static.serveme",
do_test("/packages/http/test_static.serveme", 200, /static file serving/);
// no such file, so return the default app HTML.
var getsAppHtml = [
// This file doesn't exist.
"/nosuchfile",
// Our static file serving doesn't process .. or its encoded version, so
// any of these return the app HTML.
"/../nosuchfile",
"/%2e%2e/nosuchfile",
"/%2E%2E/nosuchfile",
"/%2d%2d/nosuchfile",
"/packages/http/../http/test_static.serveme",
"/packages/http/%2e%2e/http/test_static.serveme",
"/packages/http/%2E%2E/http/test_static.serveme",
"/packages/http/../../packages/http/test_static.serveme",
"/packages/http/%2e%2e/%2e%2e/packages/http/test_static.serveme",
"/packages/http/%2E%2E/%2E%2E/packages/http/test_static.serveme",
// ... and they *definitely* shouldn't be able to escape the app bundle.
"/packages/http/../../../../../../packages/http/test_static.serveme",
"/../../../../../../../../../../../bin/ls",
"/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/bin/ls",
"/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/bin/ls"
];
_.each(succeeds, function (path) {
do_test(path, 200, /static file serving/);
_.each(getsAppHtml, function (x) {
do_test(x, 200, /<title>Tests<\/title/);
});
do_test("/packages/http/../../../../../../packages/http/test_static.serveme", 403);
// file outside of our app
do_test("/../../../../../../../../../../../bin/ls", 403);
do_test("/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/bin/ls", 403);
do_test("/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/bin/ls", 403);
}
]);
}

2
packages/js-analyze/js_analyze.js
View File

@@ -148,7 +148,7 @@ JSAnalyze.findAssignedGlobals = function (source) {
// causes escope to not bother to resolve references in the eval's scope.
// This is because an eval can pull references inward:
//
/ function outer() {
// function outer() {
// var i = 42;
// function inner() {
// eval('var i = 0');

50
packages/logging/logging.js
View File

@@ -75,11 +75,11 @@ var logInBrowser = function (obj) {
// @returns {Object: { line: Number, file: String }}
Log._getCallerDetails = function () {
var getStack = function () {
var orig = Error.prepareStackTrace;
Error.prepareStackTrace = function(_, stack){ return stack; };
// We do NOT use Error.prepareStackTrace here (a V8 extension that gets us a
// pre-parsed stack) since it's impossible to compose it with the use of
// Error.prepareStackTrace used on the server for source maps.
var err = new Error;
var stack = err.stack;
Error.prepareStackTrace = orig;
return stack;
};
@@ -87,33 +87,37 @@ Log._getCallerDetails = function () {
if (!stack) return {};
var isV8 = false;
var lines = stack;
// check for V8 specifics
if (_.isArray(stack))
isV8 = true;
else
lines = stack.split('\n');
var index = 1;
var line = lines[index];
var lines = stack.split('\n');
// looking for the first line outside the logging package
while ((isV8 ? line.getFileName() || '' : line)
.indexOf('/packages/logging.js') !== -1)
line = lines[++index];
// looking for the first line outside the logging package (or an
// eval if we find that first)
var line;
for (var i = 1; i < lines.length; ++i) {
line = lines[i];
if (line.match(/^\s*at eval \(eval/)) {
return {file: "eval"};
}
// XXX probably wants to be / or .js in case no source maps
if (!line.match(/packages\/logging(?:\/|(?:\.tests)?\.js)/))
break;
}
var details = {};
// The format for FF is functionName@filePath:lineNumber
// For V8 call built-in function
details.line = isV8 ? line.getLineNumber() : line.split(':').slice(-1)[0];
// The format for FF is 'functionName@filePath:lineNumber'
// The format for V8 is 'functionName (packages/logging/logging.js:81)' or
// 'packages/logging/logging.js:81'
var match = /(?:[@(]| at )([^(]+?):([0-9:]+)(?:\)|$)/.exec(line);
if (!match)
return details;
// in case the matched block here is line:column
details.line = match[2].split(':')[0];
// Possible format: https://foo.bar.com/scripts/file.js?random=foobar
// For FF we parse the line, for V8 we call built-in function
// XXX: if you can write the following in better way, please do it
details.file = isV8 ? line.getFileName() || (line.isEval() ? 'eval' : '')
: line.split('@')[1].split(':').slice(0, -1).join(':');
details.file = details.file.split('/').slice(-1)[0].split('?')[0];
// XXX: what about evals?
details.file = match[1].split('/').slice(-1)[0].split('?')[0];
return details;
};

7
packages/logging/logging_test.js
View File

@@ -4,14 +4,11 @@ Tinytest.add("logging - _getCallerDetails", function (test) {
// in Chrome and Firefox, other browsers don't give us an ability to get
// stacktrace.
if ((new Error).stack) {
//test.equal(details, { file: 'logging_test.js', line: 2 });
// XXX: When we have source maps, we should uncomment the test above and
// remove this one
test.isTrue(details.file === 'logging.tests.js');
test.equal(details.file, 'tinytest.js');
// evaled statements shouldn't crash
var code = "Log._getCallerDetails().file";
test.matches(eval(code), /^eval|logging.tests.js$/);
test.matches(eval(code), /^eval|tinytest.js$/);
}
});

1
packages/meteor/package.js
View File

@@ -24,7 +24,6 @@ Package.on_use(function (api, where) {
// dynamic variables, bindEnvironment
// XXX move into a separate package?
api.use('underscore', ['client', 'server']);
api.add_files('dynamics_browser.js', 'client');
api.add_files('dynamics_nodejs.js', 'server');

2
packages/mongo-livedata/.npm/package/npm-shrinkwrap.json generated
View File

@@ -7,7 +7,7 @@
"version": "0.1.9"
},
"kerberos": {
"version": "0.0.2"
"version": "0.0.3"
}
}
}

7
packages/templating/plugin/compile-templates.js
View File

@@ -42,11 +42,12 @@ Plugin.registerSourceHandler("html", function (compileStep) {
var ext = path.extname(compileStep.inputPath);
var basename = path.basename(compileStep.inputPath, ext);
// XXX generate a source map
compileStep.addJavaScript({
path: path.join(path_part, "template." + basename + ".js"),
sourcePath: compileStep.inputPath,
data: results.js,
lineForLine: false
data: results.js
});
}
});
});

17
packages/webapp/.npm/package/npm-shrinkwrap.json generated
View File

@@ -43,6 +43,23 @@
}
}
},
"send": {
"version": "0.1.0",
"dependencies": {
"debug": {
"version": "0.7.2"
},
"mime": {
"version": "1.2.6"
},
"fresh": {
"version": "0.1.0"
},
"range-parser": {
"version": "0.0.4"
}
}
},
"useragent": {
"version": "2.0.1"
}

1
packages/webapp/package.js
View File

@@ -4,6 +4,7 @@ Package.describe({
});
Npm.depends({connect: "2.7.10",
send: "0.1.0",
useragent: "2.0.1"});
Package.on_use(function (api) {

139
packages/webapp/webapp_server.js
View File

@@ -5,10 +5,12 @@ var http = Npm.require("http");
var os = Npm.require("os");
var path = Npm.require("path");
var url = Npm.require("url");
var crypto = Npm.require("crypto");
var connect = Npm.require('connect');
var optimist = Npm.require('optimist');
var useragent = Npm.require('useragent');
var send = Npm.require('send');
// @export WebApp
WebApp = {};
@@ -49,6 +51,12 @@ var initKeepalive = function () {
};
var sha1 = function (contents) {
var hash = crypto.createHash('sha1');
hash.update(contents);
return hash.digest('hex');
};
// #BrowserIdentification
//
// We have multiple places that want to identify the browser: the
@@ -207,34 +215,121 @@ var runWebAppServer = function () {
next();
}
});
// Parse the query string into res.query. Only oauth_server cares about this,
// but it's overkill to have that package depend on its own copy of connect
// just for this simple processing.
// Parse the query string into res.query. Used by oauth_server, but it's
// generally pretty handy..
app.use(connect.query());
// Auto-compress any json, javascript, or text.
app.use(connect.compress());
if (clientJson.staticCacheable) {
// cacheable files are files that should never change. Typically
// named by their hash (eg meteor bundled js and css files).
// cache them ~forever (1yr)
app.use(connect.static(path.join(clientDir, clientJson.staticCacheable),
{maxAge: 1000 * 60 * 60 * 24 * 365}));
}
var staticFiles = {};
_.each(clientJson.manifest, function (item) {
if (item.url && item.where === "client") {
staticFiles[url.parse(item.url).pathname] = {
path: item.path,
cacheable: item.cacheable,
// Link from source to its map
sourceMapUrl: item.sourceMapUrl
};
// cache non-cacheable file anyway. This isn't really correct, as
// users can change the files and changes won't propogate
// immediately. However, if we don't cache them, browsers will
// 'flicker' when rerendering images. Eventually we will probably want
// to rewrite URLs of static assets to include a query parameter to
// bust caches. That way we can both get good caching behavior and
// allow users to change assets without delay.
// https://github.com/meteor/meteor/issues/773
if (clientJson.static) {
app.use(connect.static(path.join(clientDir, clientJson.static),
{maxAge: 1000 * 60 * 60 * 24}));
}
if (item.sourceMap) {
// Serve the source map too, under the specified URL. We assume all
// source maps are cacheable.
staticFiles[url.parse(item.sourceMapUrl).pathname] = {
path: item.sourceMap,
cacheable: true
};
}
}
});
// Serve static files from the manifest.
// This is inspired by the 'static' middleware.
app.use(function (req, res, next) {
if ('GET' != req.method && 'HEAD' != req.method) {
next();
return;
}
var pathname = connect.utils.parseUrl(req).pathname;
try {
pathname = decodeURIComponent(pathname);
} catch (e) {
next();
return;
}
if (!_.has(staticFiles, pathname)) {
next();
return;
}
// We don't need to call pause because, unlike 'static', once we call into
// 'send' and yield to the event loop, we never call another handler with
// 'next'.
var info = staticFiles[pathname];
// Cacheable files are files that should never change. Typically
// named by their hash (eg meteor bundled js and css files).
// We cache them ~forever (1yr).
//
// We cache non-cacheable files anyway. This isn't really correct, as users
// can change the files and changes won't propagate immediately. However, if
// we don't cache them, browsers will 'flicker' when rerendering
// images. Eventually we will probably want to rewrite URLs of static assets
// to include a query parameter to bust caches. That way we can both get
// good caching behavior and allow users to change assets without delay.
// https://github.com/meteor/meteor/issues/773
var maxAge = info.cacheable
? 1000 * 60 * 60 * 24 * 365
: 1000 * 60 * 60 * 24;
// Set the X-SourceMap header, which current Chrome understands.
// (The files also contain '//#' comments which FF 24 understands and
// Chrome doesn't understand yet.)
//
// Eventually we should set the SourceMap header but the current version of
// Chrome and no version of FF supports it.
//
// To figure out if your version of Chrome should support the SourceMap
// header,
// - go to chrome://version. Let's say the Chrome version is
// 28.0.1500.71 and the Blink version is 537.36 (@153022)
// - go to http://src.chromium.org/viewvc/blink/branches/chromium/1500/Source/core/inspector/InspectorPageAgent.cpp?view=log
// where the "1500" is the third part of your Chrome version
// - find the first revision that is no greater than the "153022"
// number. That's probably the first one and it probably has
// a message of the form "Branch 1500 - blink@r149738"
// - If *that* revision number (149738) is at least 151755,
// then Chrome should support SourceMap (not just X-SourceMap)
// (The change is https://codereview.chromium.org/15832007)
//
// You also need to enable source maps in Chrome: open dev tools, click
// the gear in the bottom right corner, and select "enable source maps".
//
// Firefox 23+ supports source maps but doesn't support either header yet,
// so we include the '//#' comment for it:
// https://bugzilla.mozilla.org/show_bug.cgi?id=765993
// In FF 23 you need to turn on `devtools.debugger.source-maps-enabled`
// in `about:config` (it is on by default in FF 24).
if (info.sourceMapUrl)
res.setHeader('X-SourceMap', info.sourceMapUrl);
send(req, path.join(clientDir, info.path))
.maxage(maxAge)
.hidden(true) // if we specified a dotfile in the manifest, serve it
.on('error', function (err) {
Log.error("Error serving static file " + err);
res.writeHead(500);
res.end();
})
.on('directory', function () {
Log.error("Unexpected directory " + info.path);
res.writeHead(500);
res.end();
})
.pipe(res);
});
// Packages and apps can add handlers to this via WebApp.connectHandlers.
// They are inserted before our default handler.

14
scripts/generate-dev-bundle.sh
View File

@@ -110,6 +110,20 @@ npm install kexec@0.1.1
npm install shell-quote@0.0.1
npm install byline@2.0.3
# Using the unreleased 1.1 branch. We can probably switch to a built NPM version
# when it gets released.
npm install https://github.com/ariya/esprima/tarball/5044b87f94fb802d9609f1426c838874ec2007b3
# Fork of source-map which fixes one function with empty maps.
# https://github.com/mozilla/source-map/pull/70
# See also below, where we get it into source-map-support.
npm install https://github.com/meteor/source-map/tarball/4a52398901fdb4b55b06ef4dd8b69f8256072b09
# Fork of node-source-map-support which allows us to specify our own
# retrieveSourceMap function, and uses the above version of source-map.
# XXX send a pull request
npm install https://github.com/meteor/node-source-map-support/tarball/d048eaa765bf743ddaad64716647f8760e2b8507
# uglify-js has a bug which drops 'undefined' in arrays:
# https://github.com/mishoo/UglifyJS2/pull/97
npm install https://github.com/meteor/UglifyJS2/tarball/aa5abd14d3

31
tools/builder.js
View File

@@ -272,6 +272,19 @@ _.extend(Builder.prototype, {
return relPath;
},
// Convenience wrapper around generateFilename and write.
//
// (Note that in the object returned by builder.enter, this method
// is patched through directly rather than rewriting its inputs and
// outputs. This is only valid because it does nothing with its inputs
// and outputs other than send pass them to other methods.)
writeToGeneratedFilename: function (relPath, writeOptions) {
var self = this;
var generated = self.generateFilename(relPath);
self.write(generated, writeOptions);
return generated;
},
// Recursively copy a directory and all of its contents into the
// bundle. But if the symlink option was passed to the Builder
// constructor, then make a symlink instead, if possible.
@@ -377,10 +390,11 @@ _.extend(Builder.prototype, {
var self = this;
var methods = ["write", "writeJson", "reserve", "generateFilename",
"copyDirectory", "enter"];
var ret = {};
var subBuilder = {};
var relPathWithSep = relPath + path.sep;
_.each(methods, function (method) {
ret[method] = function (/* arguments */) {
subBuilder[method] = function (/* arguments */) {
var args = _.toArray(arguments);
if (method !== "copyDirectory") {
@@ -401,17 +415,24 @@ _.extend(Builder.prototype, {
// sub-bundle, not the parent bundle
if (ret.substr(0, 1) === '/')
ret = ret.substr(1);
if (ret.substr(0, relPath.length) !== relPath)
if (ret.substr(0, relPathWithSep.length) !== relPathWithSep)
throw new Error("generateFilename returned path outside of " +
"sub-bundle?");
ret = ret.substr(relPath.length);
ret = ret.substr(relPathWithSep.length);
}
return ret;
};
});
return ret;
// Methods that don't have to fix up arguments or return values, because
// they are implemented purely in terms of other methods which do.
var passThroughMethods = ["writeToGeneratedFilename"];
_.each(passThroughMethods, function (method) {
subBuilder[method] = self[method];
});
return subBuilder;
},
// Move the completed bundle into its final location (outputPath)

24
tools/buildmessage.js
View File

@@ -349,32 +349,40 @@ var error = function (message, options) {
// Record an exception. The message as well as any file and line
// information be read directly out of the exception. If not in a job,
// throws the exception instead. The first character of the error's
// message is downcased. Also capture the user portion of the stack.
// throws the exception instead. Also capture the user portion of the stack.
//
// There is special handling for files.FancySyntaxError exceptions. We
// will grab the file and location information where the syntax error
// actually occurred, rather than the place where the exception was
// thrown.
var exception = function (error) {
if (! currentJob)
throw new Error("Error: " + error.message);
if (! currentJob) {
// XXX this may be the wrong place to do this, but it makes syntax errors in
// files loaded via unipackage.load have context.
if (error instanceof files.FancySyntaxError) {
error.message = "Syntax error: " + error.message + " at " +
error.file + ":" + error.line + ":" + error.column;
}
throw error;
}
var message = error.message.slice(0,1).toLowerCase() + error.message.slice(1);
var message = error.message;
if (error instanceof files.FancySyntaxError) {
// No stack, because FancySyntaxError isn't a real Error and has no stack
// property!
currentJob.addMessage({
message: message,
stack: parseStack(error),
file: error.file,
line: error.line,
column: error.column
});
} else {
var locus = parseStack(error)[0];
var stack = parseStack(error);
var locus = stack[0];
currentJob.addMessage({
message: message,
stack: parseStack(error),
stack: stack,
func: locus.func,
file: locus.file,
line: locus.line,

299
tools/bundler.js
View File

@@ -70,26 +70,16 @@
// parameter when used
// - size: size of file in bytes
// - hash: sha1 hash of the file contents
// - sourceMap: optional path to source map file (relative to program.json)
// Additionally there will be an entry with where equal to
// "internal", path equal to page (above), and hash equal to the
// sha1 of page (before replacements.) Currently this is used to
// trigger HTML5 appcache reloads at the right time (if the
// 'appcache' package is being used.)
//
// - static: a path, relative to program.json, to a directory. If the
// server is too dumb to read 'manifest', it can just serve all of
// the files in this directory (with a relatively short cache
// expiry time.)
// XXX do not use this. It will go away soon.
//
// - static_cacheable: just like 'static' but resources that can be
// cached aggressively (cacheable: true in the manifest)
// XXX do not use this. It will go away soon.
//
// Convention:
//
// page is 'app.html', static is 'static', and staticCacheable is
// 'static_cacheable'.
// page is 'app.html'.
//
//
// == Format of a program when arch is "native.*" ==
@@ -115,6 +105,8 @@
// be search for npm modules
// - staticDirectory: directory to search for static assets when
// Assets.getText and Assets.getBinary are called from this file.
// - sourceMap: if present, path of a file that contains a source
// map for this file, relative to program.json
//
// /config.json:
//
@@ -177,6 +169,7 @@ var builder = require(path.join(__dirname, 'builder.js'));
var unipackage = require(path.join(__dirname, 'unipackage.js'));
var Fiber = require('fibers');
var Future = require(path.join('fibers', 'future'));
var sourcemap = require('source-map');
// files to ignore when bundling. node has no globs, so use regexps
var ignoreFiles = [
@@ -194,6 +187,18 @@ var inherits = function (child, parent) {
child.prototype.constructor = child;
};
var rejectBadPath = function (p) {
if (p.match(/\.\./))
throw new Error("bad path: " + p);
};
var stripLeadingSlash = function (p) {
if (p.charAt(0) !== '/')
throw new Error("bad path: " + p);
return p.slice(1);
};
///////////////////////////////////////////////////////////////////////////////
// NodeModulesDirectory
///////////////////////////////////////////////////////////////////////////////
@@ -231,19 +236,27 @@ var StaticDirectory = function (options) {
// Allowed options:
// - sourcePath: path to file on disk that will provide our contents
// - data: contents of the file as a Buffer
// - sourceMap: if 'data' is given, can be given instead of sourcePath. a string
// - cacheable
var File = function (options) {
var self = this;
if (options.data && ! (options.data instanceof Buffer)) {
if (options.data && ! (options.data instanceof Buffer))
throw new Error('File contents must be provided as a Buffer');
}
if (! options.sourcePath && ! options.data)
throw new Error("Must provide either sourcePath or data");
// The absolute path in the filesystem from which we loaded (or will
// load) this file (null if the file does not correspond to one on
// disk.)
self.sourcePath = options.sourcePath;
// If this file was generated, a sourceMap (as a string) with debugging
// information, as well as the "root" that paths in it should be resolved
// against. Set with setSourceMap.
self.sourceMap = null;
self.sourceMapRoot = null;
// Where this file is intended to reside within the target's
// filesystem.
self.targetPath = null;
@@ -251,7 +264,7 @@ var File = function (options) {
// The URL at which this file is intended to be served, relative to
// the base URL at which the target is being served (ignored if this
// file is not intended to be served over HTTP.)
self.url = null
self.url = null;
// Is this file guaranteed to never change, so that we can let it be
// cached forever? Only makes sense of self.url is set.
@@ -282,8 +295,9 @@ _.extend(File.prototype, {
contents: function (encoding) {
var self = this;
if (! self._contents) {
if (! self.sourcePath)
if (! self.sourcePath) {
throw new Error("Have neither contents nor sourcePath for file");
}
else
self._contents = fs.readFileSync(self.sourcePath);
}
@@ -291,6 +305,15 @@ _.extend(File.prototype, {
return encoding ? self._contents.toString(encoding) : self._contents;
},
setContents: function (b) {
var self = this;
if (!(b instanceof Buffer))
throw new Error("Must set contents to a Buffer");
self._contents = b;
// Un-cache hash.
self._hash = null;
},
size: function () {
var self = this;
return self.contents().length;
@@ -304,6 +327,7 @@ _.extend(File.prototype, {
var self = this;
self.url = "/" + self.hash() + suffix;
self.cacheable = true;
self.targetPath = self.hash() + suffix;
},
// Append "?<hash>" to the URL and mark the file as cacheable.
@@ -336,10 +360,10 @@ _.extend(File.prototype, {
setTargetPathFromRelPath: function (relPath) {
var self = this;
// XXX hack
if (relPath.match(/^\/packages\//) || relPath.match(/^\/static\//))
if (relPath.match(/^packages\//) || relPath.match(/^static\//))
self.targetPath = relPath;
else
self.targetPath = path.join('/app', relPath);
self.targetPath = path.join('app', relPath);
},
setStaticDirectory: function (relPath, staticSourceDirectory) {
@@ -348,18 +372,30 @@ _.extend(File.prototype, {
// static/packages specific to this package. Application assets (e.g. those
// inside private/) go in static/app/.
// XXX same hack as above
// XXX XXX is this all still true?
// XXX rename static -> assets on server
var bundlePath;
if (relPath.match(/^\/packages\//)) {
if (relPath.match(/^packages\//)) {
var dir = path.dirname(relPath);
var base = path.basename(relPath, ".js");
bundlePath = path.join('/static', dir, base);
bundlePath = path.join('static', dir, base);
} else {
bundlePath = path.join('/static', 'app');
bundlePath = path.join('static', 'app');
}
self.staticDirectory = new StaticDirectory({
sourcePath: staticSourceDirectory,
bundlePath: bundlePath
});
},
// Set a source map for this File. sourceMap is given as a string.
setSourceMap: function (sourceMap, root) {
var self = this;
if (typeof sourceMap !== "string")
throw new Error("sourceMap must be given as a string");
self.sourceMap = sourceMap;
self.sourceMapRoot = root;
}
});
@@ -431,8 +467,7 @@ _.extend(Target.prototype, {
test: options.test || []
});
// Link JavaScript, put resources in load order, and copy them to
// the bundle
// Link JavaScript and set up self.js, etc.
self._emitResources();
// Minify, if requested
@@ -442,7 +477,7 @@ _.extend(Target.prototype, {
self.minifyCss();
}
// Process asset directories (eg, /public)
// Process asset directories (eg, '/public')
// XXX this should probably be part of the appDir reader
_.each(options.assetDirs || [], function (ad) {
self.addAssetDir(ad);
@@ -454,11 +489,6 @@ _.extend(Target.prototype, {
self._addCacheBusters("js");
self._addCacheBusters("css");
}
// XXX extra thing we have to do on the client. could this move
// into ClientTarget.write()?
if (self.assignTargetPaths)
self.assignTargetPaths();
},
// Determine the packages to load, create Slices for
@@ -574,9 +604,8 @@ _.extend(Target.prototype, {
}
},
// Sort the slices in dependency order, then, slice by slice, write
// their resources into the bundle (which includes running the
// JavaScript linker.)
// Process all of the sorted slices (which includes running the JavaScript
// linker).
_emitResources: function () {
var self = this;
@@ -588,7 +617,7 @@ _.extend(Target.prototype, {
var isApp = ! slice.pkg.name;
// Emit the resources
_.each(slice.getResources(self.arch), function (resource) {
_.each(slice.getResources(self.arch), function (resource) {
if (_.contains(["js", "css", "static"], resource.type)) {
if (resource.type === "css" && ! isBrowser)
// XXX might be nice to throw an error here, but then we'd
@@ -604,15 +633,17 @@ _.extend(Target.prototype, {
cacheable: false
});
var relPath;
if (resource.type === "static" && isNative)
relPath = path.join("static", resource.servePath);
else {
relPath = stripLeadingSlash(resource.servePath);
}
f.setTargetPathFromRelPath(relPath);
if (isBrowser) {
f.setUrlFromRelPath(resource.servePath);
} else if (isNative) {
var relPath;
if (resource.type === "static")
relPath = path.join(path.sep, "static", resource.servePath);
else
relPath = resource.servePath;
f.setTargetPathFromRelPath(relPath);
if (resource.type === "js")
f.setStaticDirectory(relPath, resource.staticDirectory);
}
@@ -634,6 +665,10 @@ _.extend(Target.prototype, {
f.nodeModulesDirectory = nmd;
}
if (resource.type === "js" && resource.sourceMap) {
f.setSourceMap(resource.sourceMap, path.dirname(relPath));
}
self[resource.type].push(f);
return;
}
@@ -743,8 +778,13 @@ _.extend(Target.prototype, {
var f = new File({ sourcePath: absPath });
if (setUrl)
f.setUrlFromRelPath(assetPath);
// XXX why is this separate from _emitResources ?
// XXX fix up server static resources
var relPath = assetDir.useSubDirectory
? path.join('static', 'app', assetPath)
: assetPath;
if (setTargetPath)
f.setTargetPathFromRelPath(path.join('/static', 'app', assetPath));
f.setTargetPathFromRelPath(relPath);
self.dependencyInfo.files[absPath] = f.hash();
self.static.push(f);
});
@@ -791,22 +831,6 @@ _.extend(ClientTarget.prototype, {
self.css[0].setUrlToHash(".css");
},
assignTargetPaths: function () {
var self = this;
_.each(["js", "css", "static"], function (type) {
_.each(self[type], function (file) {
if (! file.targetPath) {
if (! file.url)
throw new Error("Client file with no URL?");
var parts = file.url.replace(/\?.*$/, '').split('/').slice(1);
parts.unshift(file.cacheable ? "static_cacheable" : "static");
file.targetPath = path.sep + path.join.apply(path, parts);
}
});
});
},
generateHtmlBoilerplate: function () {
var self = this;
@@ -829,26 +853,74 @@ _.extend(ClientTarget.prototype, {
// the target
write: function (builder) {
var self = this;
var manifest = [];
builder.reserve("program.json");
builder.reserve("app.html");
// Resources served via HTTP
_.each(["js", "css", "static"], function (type) {
_.each(self[type], function (file) {
writeFile(file, builder);
manifest.push({
path: file.targetPath,
where: "client",
type: type,
cacheable: file.cacheable,
url: file.url,
size: file.size(),
hash: file.hash()
// Helper to iterate over all resources that we serve over HTTP.
var eachResource = function (f) {
_.each(["js", "css", "static"], function (type) {
_.each(self[type], function (file) {
f(file, type);
});
});
};
// Reserve all file names from the manifest, so that interleaved
// generateFilename calls don't overlap with them.
eachResource(function (file, type) {
builder.reserve(file.targetPath);
});
// Build up a manifest of all resources served via HTTP.
var manifest = [];
eachResource(function (file, type) {
var fileContents = file.contents();
var manifestItem = {
path: file.targetPath,
where: "client",
type: type,
cacheable: file.cacheable,
url: file.url
};
if (file.sourceMap) {
// Add anti-XSSI header to this file which will be served over
// HTTP. Note that the Mozilla and WebKit implementations differ as to
// what they strip: Mozilla looks for the four punctuation characters
// but doesn't care about the newline; WebKit only looks for the first
// three characters (not the single quote) and then strips everything up
// to a newline.
// https://groups.google.com/forum/#!topic/mozilla.dev.js-sourcemap/3QBr4FBng5g
var mapData = new Buffer(")]}'\n" + file.sourceMap, 'utf8');
manifestItem.sourceMap = builder.writeToGeneratedFilename(
file.targetPath + '.map', {data: mapData});
// Use a SHA to make this cacheable.
var sourceMapBaseName = file.hash() + ".map";
// XXX When we can, drop all of this and just use the SourceMap
// header. FF doesn't support that yet, though:
// https://bugzilla.mozilla.org/show_bug.cgi?id=765993
// Note: if we use the older '//@' comment, FF 24 will print a lot
// of warnings to the console. So we use the newer '//#' comment...
// which Chrome (28) doesn't support. So we also set X-SourceMap
// in webapp_server.
file.setContents(Buffer.concat([
file.contents(),
new Buffer("\n//# sourceMappingURL=" + sourceMapBaseName + "\n")
]));
manifestItem.sourceMapUrl = require('url').resolve(
file.url, sourceMapBaseName);
}
// Set this now, in case we mutated the file's contents.
manifestItem.size = file.size();
manifestItem.hash = file.hash();
writeFile(file, builder);
manifest.push(manifestItem);
});
// HTML boilerplate (the HTML served to make the client load the
@@ -864,17 +936,8 @@ _.extend(ClientTarget.prototype, {
// Control file
builder.writeJson('program.json', {
format: "browser-program-pre1",
manifest: manifest,
page: 'app.html',
// XXX the following are for use by 'legacy' (read: current)
// server.js implementations which aren't smart enough to read
// the manifest and instead want all of the resources in a
// directory together so they can just point gzippo at it. we
// should remove this and make the server work from the
// manifest.
static: 'static',
staticCacheable: 'static_cacheable'
manifest: manifest
});
return "program.json";
}
@@ -899,6 +962,7 @@ var JsImage = function () {
// - source: JS source code to load, as a string
// - nodeModulesDirectory: a NodeModulesDirectory indicating which
// directory should be searched by Npm.require()
// - sourceMap: if set, source map for this code, as a string
// note: this can't be called `load` at it would shadow `load()`
self.jsToLoad = [];
@@ -1002,11 +1066,15 @@ _.extend(JsImage.prototype, {
}, bindings || {});
try {
// XXX Get the actual source file path -- item.targetPath is
// not actually correct (it's the path in the bundle rather
// than in the source tree.) Moreover, we need to do source
// mapping.
files.runJavaScript(item.source, item.targetPath, env);
// XXX XXX Get the actual source file path -- item.targetPath
// is not actually correct (it's the path in the bundle rather
// than in the source tree.)
files.runJavaScript(item.source.toString('utf8'), {
filename: item.targetPath,
symbols: env,
sourceMap: item.sourceMap,
sourceMapRoot: item.sourceMapRoot
});
} catch (e) {
buildmessage.exception(e);
// Recover by skipping the rest of the load
@@ -1025,7 +1093,7 @@ _.extend(JsImage.prototype, {
write: function (builder) {
var self = this;
builder.reserve("program.js");
builder.reserve("program.json");
// Finalize choice of paths for node_modules directories -- These
// paths are no longer just "preferred"; they are the final paths
@@ -1045,14 +1113,28 @@ _.extend(JsImage.prototype, {
if (! item.targetPath)
throw new Error("No targetPath?");
builder.write(item.targetPath, { data: new Buffer(item.source, 'utf8') });
load.push({
path: item.targetPath,
var loadPath = builder.writeToGeneratedFilename(
item.targetPath,
{ data: new Buffer(item.source, 'utf8') });
var loadItem = {
path: loadPath,
node_modules: item.nodeModulesDirectory ?
item.nodeModulesDirectory.preferredBundlePath : undefined,
staticDirectory: item.staticDirectory ?
item.staticDirectory.bundlePath : undefined
});
};
if (item.sourceMap) {
// Write the source map.
// XXX this code is very similar to saveAsUnipackage.
loadItem.sourceMap = builder.writeToGeneratedFilename(
item.targetPath + '.map',
{ data: new Buffer(item.sourceMap, 'utf8') }
);
loadItem.sourceMapRoot = item.sourceMapRoot;
}
load.push(loadItem);
});
// node_modules resources from the packages. Due to appropriate
@@ -1070,9 +1152,9 @@ _.extend(JsImage.prototype, {
// Control file
builder.writeJson('program.json', {
load: load,
format: "javascript-image-pre1",
arch: self.arch
arch: self.arch,
load: load
});
return "program.json";
}
@@ -1093,11 +1175,11 @@ JsImage.readFromDisk = function (controlFilePath) {
ret.arch = json.arch;
_.each(json.load, function (item) {
if (item.path.match(/\.\./))
throw new Error("bad path in plugin bundle");
rejectBadPath(item.path);
var nmd = undefined;
if (item.node_modules) {
rejectBadPath(item.node_modules);
var node_modules = path.join(dir, item.node_modules);
if (! (node_modules in ret.nodeModulesDirectories)) {
ret.nodeModulesDirectories[node_modules] =
@@ -1109,14 +1191,22 @@ JsImage.readFromDisk = function (controlFilePath) {
nmd = ret.nodeModulesDirectories[node_modules];
}
ret.jsToLoad.push({
var loadItem = {
targetPath: item.path,
source: fs.readFileSync(path.join(dir, item.path)),
nodeModulesDirectory: nmd,
staticDirectory: new StaticDirectory({
sourcePath: item.staticDirectory
})
});
};
if (item.sourceMap) {
// XXX this is the same code as initFromUnipackage
rejectBadPath(item.sourceMap);
loadItem.sourceMap = fs.readFileSync(
path.join(dir, item.sourceMap), 'utf8');
loadItem.sourceMapRoot = item.sourceMapRoot;
}
ret.jsToLoad.push(loadItem);
});
return ret;
@@ -1144,7 +1234,9 @@ _.extend(JsImageTarget.prototype, {
targetPath: file.targetPath,
source: file.contents().toString('utf8'),
nodeModulesDirectory: file.nodeModulesDirectory,
staticDirectory: file.staticDirectory
staticDirectory: file.staticDirectory,
sourceMap: file.sourceMap,
sourceMapRoot: file.sourceMapRoot
});
});
@@ -1380,7 +1472,7 @@ var writeSiteArchive = function (targets, outputPath, options) {
// Affordances for standalone use
if (targets.server) {
// add program.json as the first argument after "node main.js" to the boot script.
var stub = new Buffer("process.argv.splice(2, 0, 'program.json');\nrequire('./programs/server/boot.js');\n", 'utf8');
var stub = new Buffer("process.argv.splice(2, 0, 'program.json');\nprocess.chdir(require('path').join(__dirname, 'programs', 'server'));\nrequire('./programs/server/boot.js');\n", 'utf8');
builder.write('main.js', { data: stub });
builder.write('README', { data: new Buffer(
@@ -1526,9 +1618,8 @@ exports.bundle = function (appDir, outputPath, options) {
assetDirs = assetDirs || [];
var clientAssetDirs = getValidAssetDirs(assetDirs, {
exclude: ignoreFiles,
setUrl: true
// No need to set targetPath when the asset dir is added;
// the target path will be set later in assignTargetPaths.
setUrl: true,
setTargetPath: true
});
client.make({
@@ -1560,9 +1651,11 @@ exports.bundle = function (appDir, outputPath, options) {
assetDirs = assetDirs || [];
var serverAssetDirs = getValidAssetDirs(assetDirs, {
exclude: ignoreFiles,
setTargetPath: true
// We need to set the target path when the asset dir is added,
// because the target path comes from the asset's path.
setTargetPath: true,
// XXX this is a hack, re-assess how the subdirs are named
useSubDirectory: true
});
var targetOptions = {
library: library,

226
tools/files.js
View File

@@ -10,10 +10,31 @@ var os = require('os');
var util = require('util');
var _ = require('underscore');
var Future = require('fibers/future');
var sourcemap = require('source-map');
var sourcemap_support = require('source-map-support');
var cleanup = require('./cleanup.js');
var buildmessage = require('./buildmessage.js');
var parsedSourceMaps = {};
var nextStackFilenameCounter = 1;
var retrieveSourceMap = function (pathForSourceMap) {
if (_.has(parsedSourceMaps, pathForSourceMap))
return {map: parsedSourceMaps[pathForSourceMap]};
return null;
};
sourcemap_support.install({
// Use the source maps specified to runJavaScript instead of parsing source
// code for them.
retrieveSourceMap: retrieveSourceMap,
// For now, don't fix the source line in uncaught exceptions, because we
// haven't fixed handleUncaughtExceptions in source-map-support to properly
// locate the source files.
handleUncaughtExceptions: false
});
var files = exports;
_.extend(exports, {
// A sort comparator to order files into load order.
@@ -595,36 +616,76 @@ _.extend(exports, {
return future.wait();
},
// Return the result of evaluating `code` using
// `runInThisContext`. `code` will be wrapped in a closure. You can
// pass additional values to bind in the closure in `env`, the keys
// being the symbols to bind and the values being their
// values. `filename` is the filename to use in exceptions that come
// from inside this code.
// Return the result of evaluating `code` using `runInThisContext`. `code`
// will be wrapped in a closure. You can pass additional values to bind in the
// closure in `options.symbols`, the keys being the symbols to bind and the
// values being their values. `options.filename` is the filename to use in
// exceptions that come from inside this code. `options.sourceMap` is an
// optional source map that represents the file.
//
// The really special thing about this function is that if a parse
// error occurs, we will raise an exception of type
// files.FancySyntaxError, from which you may read 'message', 'file',
// 'line', 'column', and 'columnEnd' attributes ... v8 is
// normally reluctant to reveal this information but will write it
// to stderr if you pass it an undocumented flag. Unforunately
// though node doesn't have dup2 so we can't intercept the write. So
// instead -- only if the parse does in fact fail, to determine the
// error we start a subprocess, redirect its stderr, grab the output
// and parse it.
runJavaScript: function (code, filename, env) {
// The really special thing about this function is that if a parse error
// occurs, we will raise an exception of type files.FancySyntaxError, from
// which you may read 'message', 'file', 'line', and 'column' attributes
// ... v8 is normally reluctant to reveal this information but will write it
// to stderr if you pass it an undocumented flag. Unforunately though node
// doesn't have dup2 so we can't intercept the write. So instead we use a
// completely different parser with a better error handling API. Ah well.
runJavaScript: function (code, options) {
if (typeof code !== 'string')
throw new Error("code must be a string");
options = options || {};
var filename = options.filename || "<anonymous>";
var keys = [], values = [];
// don't assume that _.keys and _.values are guaranteed to
// enumerate in the same order
for (var k in env) {
keys.push(k);
values.push(env[k]);
_.each(options.symbols, function (value, name) {
keys.push(name);
values.push(value);
});
var stackFilename = filename;
if (options.sourceMap) {
// We want to generate an arbitrary filename that we use to associate the
// file with its source map.
stackFilename = "<runJavaScript-" + nextStackFilenameCounter++ + ">";
}
var chunks = [];
var header = "(function(" + keys.join(',') + "){";
chunks.push(header);
if (options.sourceMap) {
var consumer = new sourcemap.SourceMapConsumer(options.sourceMap);
chunks.push(sourcemap.SourceNode.fromStringWithSourceMap(
code, consumer));
} else {
chunks.push(code);
}
// \n is necessary in case final line is a //-comment
var footer = "\n})";
var wrapped = header + code + footer;
chunks.push("\n})");
var wrapped;
var parsedSourceMap = null;
if (options.sourceMap) {
var node = new sourcemap.SourceNode(null, null, null, chunks);
var results = node.toStringWithSourceMap({
file: stackFilename
});
wrapped = results.code;
parsedSourceMap = results.map.toJSON();
if (options.sourceMapRoot) {
// Add the specified root to any root that may be in the file.
parsedSourceMap.sourceRoot = path.join(
options.sourceMapRoot, parsedSourceMap.sourceRoot || '');
}
// source-map-support doesn't ever look at the sourcesContent field, so
// there's no point in keeping it in memory.
delete parsedSourceMap.sourcesContent;
parsedSourceMaps[stackFilename] = parsedSourceMap;
} else {
wrapped = chunks.join('');
};
try {
// See #runInThisContext
@@ -636,84 +697,75 @@ _.extend(exports, {
//
// Pass 'true' as third argument if we want the parse error on
// stderr (which we don't.)
var func = require('vm').runInThisContext(wrapped, filename);
} catch (e) {
// Got, presumably, a parse error. OK, we're going to start
// another copy of node and feed it the offending code on
// stdin. It should give us the error on stderr.
var script = require('vm').createScript(wrapped, stackFilename);
} catch (nodeParseError) {
if (!(nodeParseError instanceof SyntaxError))
throw nodeParseError;
// Got a parse error. Unfortunately, we can't actually get the location of
// the parse error from the SyntaxError; Node has some hacky support for
// displaying it over stderr if you pass an undocumented third argument to
// stackFilename, but that's not what we want. See
// https://github.com/joyent/node/issues/3452
// for more information. One thing to try (and in fact, what an early
// version of this function did) is to actually fork a new node
// to run the code and parse its output. We instead run an entirely
// different JS parser, from the esprima project, but which at least
// has a nice API for reporting errors.
var esprima = require('esprima');
try {
esprima.parse(wrapped);
} catch (esprimaParseError) {
// Is this actually an Esprima syntax error?
if (!('index' in esprimaParseError &&
'lineNumber' in esprimaParseError &&
'column' in esprimaParseError &&
'description' in esprimaParseError)) {
throw esprimaParseError;
}
var err = new files.FancySyntaxError;
var Future = require('fibers/future');
var future = new Future;
err.message = esprimaParseError.description;
var child_process = require("child_process");
var proc = child_process.execFile(
process.argv[0], [], {
stdio: ['pipe']
}, function (error, stdout, stderr) {
if (! error || error.code === 0)
future.return(null); // huh? didn't fail?
else
future.return(stderr);
});
proc.stdin.write(wrapped);
proc.stdin.end();
var stderr = future.wait();
if (parsedSourceMap) {
// XXX this duplicates code in computeGlobalReferences
var consumer2 = new sourcemap.SourceMapConsumer(parsedSourceMap);
var original = consumer2.originalPositionFor({
line: esprimaParseError.lineNumber,
column: esprimaParseError.column - 1
});
if (original.source) {
err.file = original.source;
err.line = original.line;
err.column = original.column + 1;
throw err;
}
}
if (stderr === null)
throw new Error("subprocess parsed bad code successfully?");
/* stderr will look something like this (note leading blank line:)
"
[stdin]:1
couaoeua aaaaaa nsolexloeuaoeuao
^^^^^^
SyntaxError: Unexpected identifier
at Object.<anonymous> ([stdin]-wrapper:6:22)
at Module._compile (module.js:449:26)
at evalScript (node.js:282:25)
at Socket.<anonymous> (node.js:152:11)
at Socket.EventEmitter.emit (events.js:93:17)
at Pipe.onread (net.js:418:51)"
*/
var err = new files.FancySyntaxError;
err.file = filename;
var lines = stderr.split('\n');
// line number
var m = lines[1].match(/:(\d+)\s*$/);
if (! m)
throw new Error("can't parse line number from '" + lines[1] + "'");
err.line = +m[1];
// column range
m = lines[3].match(/^(\s*)(\^+)\s*$/)
if (! m)
throw new Error("can't parse column indicator from '" + lines[3] + "'");
err.column = m[1].length + 1;
err.columnEnd = err.column + m[2].length - 1;
// message
m = lines[4].match(/^SyntaxError:\s*(.*)$/);
if (! m)
throw new Error("can't parse error message from '" + lines[4] + "'");
err.message = m[1];
// adjust errors on line 1 to account for our header
if (err.line === 1) {
err.column -= header.length;
err.columnEnd -= header.length;
err.file = filename; // *not* stackFilename
err.line = esprimaParseError.lineNumber;
err.column = esprimaParseError.column;
// adjust errors on line 1 to account for our header
if (err.line === 1) {
err.column -= header.length;
}
throw err;
}
throw err;
// What? Node thought that this was a parse error and esprima didn't? Eh,
// just throw Node's error and don't care too much about the line numbers
// being right.
throw nodeParseError;
}
var func = script.runInThisContext();
return (buildmessage.markBoundary(func)).apply(null, values);
},
// - message: an error message from the parser
// - file: filename
// - line: 1-based
// - column: 1-based
// - columnEnd: 1-based
FancySyntaxError: function () {},
OfflineError: function (error) {

4
tools/library.js
View File

@@ -7,7 +7,7 @@ var bundler = require('./bundler.js');
var buildmessage = require('./buildmessage.js');
var fs = require('fs');
// Under the hood, packages in the library (/package/foo), and user
// Under the hood, packages in the library (/packages/foo), and user
// applications, are both Packages -- they are just represented
// differently on disk.
@@ -374,4 +374,4 @@ _.extend(exports, {
return out;
}
});
});

700
tools/linker.js
View File

@@ -1,5 +1,6 @@
var fs = require('fs');
var _ = require('underscore');
var sourcemap = require('source-map');
var buildmessage = require('./buildmessage');
var packageDot = function (name) {
@@ -9,21 +10,6 @@ var packageDot = function (name) {
return "Package['" + name + "']";
};
var generateBoundary = function () {
// In a perfect world we would call Packages.random.Random.id().
// But we can't do that this is part of the code that is used to
// compile and load packages. So let it slide for now and provide a
// version based on (the completely non-cryptographic) Math.random,
// which is good enough for this particular application.
var alphabet = "23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz";
var digits = [];
for (var i = 0; i < 17; i++) {
var index = Math.floor(Math.random() * alphabet.length);
digits[i] = alphabet.substr(index, 1);
}
return "__imports_" + digits.join("") + "__";
};
///////////////////////////////////////////////////////////////////////////////
// Module
///////////////////////////////////////////////////////////////////////////////
@@ -40,9 +26,6 @@ var Module = function (options) {
// files in the module. array of File
self.files = [];
// boundary to use to mark where import should go in final phase
self.boundary = generateBoundary();
// options
self.forceExport = options.forceExport || [];
self.useGlobalNamespace = options.useGlobalNamespace;
@@ -77,6 +60,13 @@ _.extend(Module.prototype, {
return _.max(maxInFile);
},
runLinkerFileTransforms: function (exports) {
var self = this;
_.each(self.files, function (f) {
f.runLinkerFileTransform(exports);
});
},
// Figure out which vars need to be specifically put in the module
// scope.
//
@@ -87,15 +77,15 @@ _.extend(Module.prototype, {
// and see what your globals are. Probably this means we need to
// move the emission of the Package-scope Variables section (but not
// the actual static analysis) to the final phase.
computeModuleScopedVars: function () {
computeModuleScopeVars: function (exports) {
var self = this;
if (!self.jsAnalyze) {
// We don't have access to static analysis, probably because we *are* the
// js-analyze package. Let's do a stupid heuristic: any exports that have
// no dots are module scoped vars. (This works for
// no dots are module scope vars. (This works for
// js-analyze.JSAnalyze...)
return _.filter(self.getExports(), function (e) {
return _.filter(exports, function (e) {
return e.indexOf('.') === -1;
});
}
@@ -107,68 +97,68 @@ _.extend(Module.prototype, {
});
globalReferences = _.uniq(globalReferences);
return globalReferences;
return _.isEmpty(globalReferences) ? undefined : globalReferences;
},
// Output is a list of objects with keys 'source' and 'servePath'.
getLinkedFiles: function () {
// Output is a list of objects with keys 'source', 'servePath', 'sourceMap',
// 'sourcePath'
getPrelinkedFiles: function (moduleExports) {
var self = this;
if (! self.files.length && ! self.useGlobalNamespace)
if (! self.files.length)
return [];
var moduleExports = self.getExports();
// If we don't want to create a separate scope for this module,
// then our job is much simpler. And we can get away with
// preserving the line numbers.
if (self.useGlobalNamespace) {
var ret = [{
source: self.boundary,
servePath: self.importStubServePath
}];
return _.map(self.files, function (file) {
var node = file.getPrelinkedOutput({ preserveLineNumbers: true,
exports: moduleExports });
var results = node.toStringWithSourceMap({
file: file.servePath
}); // results has 'code' and 'map' attributes
var sourceMap = results.map.toJSON();
// No use generating empty source maps.
if (_.isEmpty(sourceMap.sources))
sourceMap = null;
else
sourceMap = JSON.stringify(sourceMap);
return ret.concat(_.map(self.files, function (file) {
return {
source: file.getLinkedOutput({ preserveLineNumbers: true,
exports: moduleExports }),
servePath: file.servePath
source: results.code,
servePath: file.servePath,
sourceMap: sourceMap
};
}));
});
}
// Otherwise..
// Find the maximum line length. The extra three are for the
// comments that will be emitted when we skip a unit.
var sourceWidth = _.max([68, self.maxLineLength(120 - 2)]) + 3;
// Figure out which variables are module scope
var moduleScopedVars = self.computeModuleScopedVars();
// Find the maximum line length.
var sourceWidth = _.max([68, self.maxLineLength(120 - 2)]);
// Prologue
var combined = "(function () {\n\n";
combined += self.boundary;
if (moduleScopedVars.length) {
combined += "/* Package-scope variables */\n";
combined += "var " + moduleScopedVars.join(', ') + ";\n\n";
}
var chunks = [];
// Emit each file
_.each(self.files, function (file) {
combined += file.getLinkedOutput({ sourceWidth: sourceWidth,
exports: moduleExports });
combined += "\n";
if (!_.isEmpty(chunks))
chunks.push("\n\n\n\n\n\n");
chunks.push(file.getPrelinkedOutput({ sourceWidth: sourceWidth,
exports: moduleExports }));
});
// Epilogue
combined += self.getExportCode();
combined += "\n})();";
var node = new sourcemap.SourceNode(null, null, null, chunks);
var results = node.toStringWithSourceMap({
file: self.combinedServePath
}); // results has 'code' and 'map' attributes
return [{
source: combined,
servePath: self.combinedServePath
source: results.code,
servePath: self.combinedServePath,
sourceMap: results.map.toString()
}];
},
@@ -181,51 +171,11 @@ _.extend(Module.prototype, {
var exports = {};
_.each(self.files, function (file) {
_.each(file.units, function (unit) {
_.extend(exports, unit.exports);
});
_.extend(exports, file.exports);
});
return _.union(_.keys(exports), self.forceExport);
},
// Return code that saves our exports to Package.packagename.foo.bar
getExportCode: function () {
var self = this;
if (! self.name)
return "";
// If we're a no-exports module, then we have no export code (not even
// creating Package.foo).
if (self.noExports)
return "";
if (self.useGlobalNamespace)
// Haven't thought about this case. When would this happen?
throw new Error("Not implemented: exports from global namespace");
var buf = "/* Exports */\n";
buf += "if (typeof Package === 'undefined') Package = {};\n";
buf += packageDot(self.name) + " = ";
var exports = self.getExports();
// Even if there are no exports, we need to define Package.foo, because the
// existence of Package.foo is how another package (eg, one that weakly
// depends on foo) can tell if foo is loaded.
if (exports.length === 0)
return buf + "{};\n";
// Given exports like Foo, Bar.Baz, Bar.Quux.A, and Bar.Quux.B,
// construct an expression like
// {Foo: Foo, Bar: {Baz: Bar.Baz, Quux: {A: Bar.Quux.A, B: Bar.Quux.B}}}
var scratch = {};
_.each(self.getExports(), function (symbol) {
scratch[symbol] = symbol;
});
var exportTree = buildSymbolTree(scratch);
buf += writeSymbolTree(exportTree, 0);
buf += ";\n";
return buf;
}
});
// Given 'symbolMap' like {Foo: 's1', 'Bar.Baz': 's2', 'Bar.Quux.A': 's3', 'Bar.Quux.B': 's4'}
@@ -285,44 +235,103 @@ var File = function (inputFile, module) {
// the path where this file would prefer to be served if possible
self.servePath = inputFile.servePath;
// the path to use for error message
// The relative path of this input file in its source tree (eg,
// package or app.) Used for source maps, error messages..
self.sourcePath = inputFile.sourcePath;
// should line and column be included in errors?
self.includePositionInErrors = inputFile.includePositionInErrors;
// The individual @units in the file. Array of Unit. Concatenating
// the source of each unit, in order, will give self.source.
self.units = [];
// A function which transforms the source code once all exports are
// known. (eg, for CoffeeScript.)
self.linkerUnitTransform =
inputFile.linkerUnitTransform || function (source, exports) {
return source;
self.linkerFileTransform =
inputFile.linkerFileTransform || function (sourceWithMap, exports) {
return sourceWithMap;
};
// If true, don't wrap this individual file in a closure.
self.bare = !!inputFile.bare;
// A source map (generated by something like CoffeeScript) for the input file.
self.sourceMap = inputFile.sourceMap;
// The Module containing this file.
self.module = module;
self._unitize();
// symbols mentioned in @export, @require, @provide, or @weak
// directives. each is a map from the symbol (given as a string) to
// true. (only @export is actually implemented)
self.exports = {};
self.requires = {};
self.provides = {};
self.weaks = {};
self._scanForComments();
};
_.extend(File.prototype, {
// Return the union of the global references in all of the units in
// this file that we are actually planning to use. Array of string.
// Return the globals in this file as an array of symbol names. For
// example: if the code references 'Foo.bar.baz' and 'Quux', and
// neither are declared in a scope enclosing the point where they're
// referenced, then globalReferences would include ["Foo", "Quux"].
computeGlobalReferences: function () {
var self = this;
var globalReferences = [];
_.each(self.units, function (unit) {
if (unit.include)
globalReferences = globalReferences.concat(unit.computeGlobalReferences());
});
return globalReferences;
var jsAnalyze = self.module.jsAnalyze;
// If we don't have a JSAnalyze object, we probably are the js-analyze
// package itself. Assume we have no global references. At the module level,
// we'll assume that exports are global references.
if (!jsAnalyze)
return [];
try {
return _.keys(jsAnalyze.findAssignedGlobals(self.source));
} catch (e) {
if (!e.$ParseError)
throw e;
var errorOptions = {
file: self.sourcePath,
line: e.lineNumber,
column: e.column
};
if (self.sourceMap) {
var parsed = new sourcemap.SourceMapConsumer(self.sourceMap);
var original = parsed.originalPositionFor(
{line: e.lineNumber, column: e.column - 1});
if (original.source) {
errorOptions.file = original.source;
errorOptions.line = original.line;
errorOptions.column = original.column + 1;
}
}
buildmessage.error(e.description, errorOptions);
// Recover by pretending that this file is empty (which
// includes replacing its source code with '' in the output)
self.source = "";
self.sourceMap = null;
return [];
}
},
// Relative path to use in source maps to indicate this file. No
// leading slash.
_pathForSourceMap: function () {
var self = this;
if (self.module.name)
return self.module.name + "/" + self.sourcePath;
else
return require('path').basename(self.sourcePath);
},
runLinkerFileTransform: function (exports) {
var self = this;
var sourceAndMap = self.linkerFileTransform(
{source: self.source, sourceMap: self.sourceMap},
exports);
self.source = sourceAndMap.source;
self.sourceMap = sourceAndMap.sourceMap;
},
// Options:
@@ -331,73 +340,121 @@ _.extend(File.prototype, {
// sourceWidth is ignored.
// - sourceWidth: width in columns to use for the source code
// - exports: the module's exports
getLinkedOutput: function (options) {
//
// Returns a SourceNode.
getPrelinkedOutput: function (options) {
var self = this;
// XXX XXX if a unit is not going to be used, prepend each line with '//'
// The newline after the source closes a '//' comment.
if (options.preserveLineNumbers) {
// Ugly version
var body = self.linkerUnitTransform(self.source, options.exports);
if (body.length && body[body.length - 1] !== '\n')
body += '\n';
return self.bare ? body : ("(function(){" + body + "})();\n");
var mapNode;
if (self.sourceMap) {
mapNode = sourcemap.SourceNode.fromStringWithSourceMap(
self.source, new sourcemap.SourceMapConsumer(self.sourceMap));
} else {
// This is an app file that was always JS. The output file here is going
// to be the same name as the input file (because _pathForSourceMap in
// apps is the basename of the source file), and having a JS file
// pointing to a source map pointing to a JS file of the same name will
// (a) be confusing (b) be unnecessary since we aren't renumbering
// anything and (c) confuse at least Chrome.
mapNode = self.source;
}
return new sourcemap.SourceNode(null, null, null, [
self.bare ? "" : "(function(){",
mapNode,
(self.source.length && self.source[self.source.length - 1] !== '\n'
? "\n" : ""),
self.bare ? "" : "\n})();\n"
]);
}
// Pretty version
var buf = "";
var chunks = [];
// Prologue
if (!self.bare)
buf += "(function () {\n\n";
if (! self.bare)
chunks.push("(function () {\n\n");
// Banner
var bannerLines = [self.servePath.slice(1)];
if (self.bare) {
bannerLines.push(
"This file is in bare mode and is not in its own closure.");
}
var width = options.sourceWidth || 70;
var bannerWidth = width + 3;
var divider = new Array(bannerWidth + 1).join('/') + "\n";
var spacer = "// " + new Array(bannerWidth - 6 + 1).join(' ') + " //\n";
var padding = new Array(bannerWidth + 1).join(' ');
var padding = bannerPadding(bannerWidth);
chunks.push(banner(bannerLines, bannerWidth));
var blankLine = new Array(width + 1).join(' ') + " //\n";
buf += divider + spacer;
buf += "// " + (self.servePath.slice(1) + padding).slice(0, bannerWidth - 6) +
" //\n";
if (self.bare) {
var bareText = "This file is in bare mode and is not in its own closure.";
buf += "// " + (bareText + padding).slice(0, bannerWidth - 6) + " //\n";
}
buf += spacer + divider + blankLine;
chunks.push(blankLine);
// Code, with line numbers
// You might prefer your line numbers at the beginning of the
// line, with /* .. */. Well, that requires parsing the source for
// comments, because you have to do something different if you're
// already inside a comment.
var num = 1;
_.each(self.units, function (unit) {
var unitSource = self.linkerUnitTransform(unit.source, options.exports);
var lines = unitSource.split('\n');
var numberifyLines = function (f) {
var num = 1;
var lines = self.source.split('\n');
_.each(lines, function (line) {
if (! unit.include)
line = "// " + line;
if (line.length > width)
buf += line + "\n";
else
buf += (line + padding).slice(0, width) + " // " + num + "\n";
var suffix = "\n";
if (line.length <= width) {
suffix = padding.slice(line.length, width) + " // " + num + "\n";
}
f(line, suffix, num);
num++;
});
});
};
var lines = self.source.split('\n');
if (self.sourceMap) {
var buf = "";
numberifyLines(function (line, suffix) {
buf += line;
buf += suffix;
});
// The existing source map is valid because all we're doing is adding
// things to the end of lines, which doesn't affect the source map. (If
// we wanted to be picky, we could add some explicitly non-mapped regions
// to the source map to cover the suffixes, which would make this
// equivalent to the "no source map coming in" case, but this doesn't seem
// that important.)
chunks.push(sourcemap.SourceNode.fromStringWithSourceMap(
self.source,
new sourcemap.SourceMapConsumer(self.sourceMap)));
} else {
// There are probably ways to make a more compact source map. For example,
// the only change we make is to append a comment, so we can probably emit
// one mapping for the whole file. For the moment, we'll do it by the book
// just to see how it goes.
numberifyLines(function (line, suffix, num) {
chunks.push(new sourcemap.SourceNode(num, 0, self._pathForSourceMap(),
line));
chunks.push(suffix);
});
}
// Footer
buf += divider;
if (! self.bare)
chunks.push(dividerLine(bannerWidth) + "\n}).call(this);\n");
// Epilogue
if (!self.bare)
buf += "\n}).call(this);\n";
buf += "\n\n\n\n\n";
var node = new sourcemap.SourceNode(null, null, null, chunks);
return buf;
// If we're working directly from the original source here (and not from the
// output of a transformation that had a source map), include the original
// source in the source map. (If we are working on generated code, the
// source map we received should have already contained the original
// source.)
if (!self.sourceMap)
node.setSourceContent(self._pathForSourceMap(), self.source);
return node;
},
// If "line" contains nothing but a comment (of either syntax), return the
@@ -418,159 +475,73 @@ _.extend(File.prototype, {
return null;
},
// Split file and populate self.units
// XXX it is an error to declare a @unit not at toplevel (eg, inside a
// function or object..) We don't detect this but we might have to to
// give an acceptable user experience..
_unitize: function () {
// Scan for @export, etc.
_scanForComments: function () {
var self = this;
var lines = self.source.split("\n");
var buf = "";
var unit = new Unit(
null, true, self, self.includePositionInErrors ? 0 : null);
self.units.push(unit);
var lineCount = 0;
_.each(lines, function (line) {
var commentBody = self._getSingleLineCommentBody(line);
if (!commentBody)
return;
if (commentBody) {
// XXX overly permissive. should detect errors
var match = /^@unit(?:\s+(\S+))?$/.exec(commentBody);
if (match) {
unit.source = buf;
buf = line;
unit = new Unit(match[1] || null, false, self,
self.includePositionInErrors ? lineCount : null);
self.units.push(unit);
lineCount++;
return;
}
// XXX overly permissive. should detect errors
var match = /^@(export|require|provide|weak)(\s+.*)$/.exec(commentBody);
if (match) {
var what = match[1];
var symbols = _.map(match[2].split(/,/), function (s) {
return s.trim();
});
// XXX overly permissive. should detect errors
match = /^@(export|require|provide|weak)(\s+.*)$/.exec(commentBody);
if (match) {
var what = match[1];
var symbols = _.map(match[2].split(/,/), function (s) {
return s.trim();
var badSymbols = _.reject(symbols, function (s) {
// XXX should be unicode-friendlier
return s.match(/^([_$a-zA-Z][_$a-zA-Z0-9]*)(\.[_$a-zA-Z][_$a-zA-Z0-9]*)*$/);
});
if (!_.isEmpty(badSymbols)) {
buildmessage.error("bad symbols for @" + what + ": " +
JSON.stringify(badSymbols),
{ file: self.sourcePath });
// recover by ignoring
} else if (self.module.noExports && what === "export") {
buildmessage.error("@export not allowed in this slice",
{ file: self.sourcePath });
// recover by ignoring
} else {
_.each(symbols, function (s) {
if (s.length)
self[what + "s"][s] = true;
});
var badSymbols = _.reject(symbols, function (s) {
// XXX should be unicode-friendlier
return s.match(/^([_$a-zA-Z][_$a-zA-Z0-9]*)(\.[_$a-zA-Z][_$a-zA-Z0-9]*)*$/);
});
if (!_.isEmpty(badSymbols)) {
buildmessage.error("bad symbols for @" + what + ": " +
JSON.stringify(badSymbols),
{ file: self.sourcePath });
// recover by ignoring
} else if (self.module.noExports && what === "export") {
buildmessage.error("@export not allowed in this slice",
{ file: self.sourcePath });
// recover by ignoring
} else {
_.each(symbols, function (s) {
if (s.length)
unit[what + "s"][s] = true;
});
}
/* fall through */
}
}
if (lineCount !== 0)
buf += "\n";
lineCount++;
buf += line;
});
unit.source = buf;
}
});
///////////////////////////////////////////////////////////////////////////////
// Unit
///////////////////////////////////////////////////////////////////////////////
// Given a list of lines (not newline-terminated), returns a string placing them
// in a pretty banner of width bannerWidth. All lines must have length at most
// (bannerWidth - 6); if bannerWidth is not provided, the smallest width that
// fits is used.
var banner = function (lines, bannerWidth) {
if (!bannerWidth)
bannerWidth = 6 + _.max(lines, function (x) { return x.length; }).length;
var Unit = function (name, mandatory, file, lineOffset) {
var self = this;
var divider = dividerLine(bannerWidth);
var spacer = "// " + new Array(bannerWidth - 6 + 1).join(' ') + " //\n";
var padding = bannerPadding(bannerWidth);
// name of the unit, or null if none provided
self.name = name;
// source code for this unit (a string)
self.source = null;
// true if this unit is to always be included
self.mandatory = !! mandatory;
// true if we should include this unit in the linked output
self.include = self.mandatory;
// The File containing the unit.
self.file = file;
// offset of 'self.source' in the original input file, in whole
// lines (partial lines are not supported.) Used to generate correct
// line number information in error messages. Set to null to omit
// line/column information (you'll need to do this, for, eg,
// coffeescript output, given that we don't have sourcemaps here
// yet.)
self.lineOffset = lineOffset;
// symbols mentioned in @export, @require, @provide, or @weak
// directives. each is a map from the symbol (given as a string) to
// true.
self.exports = {};
self.requires = {};
self.provides = {};
self.weaks = {};
var buf = divider + spacer;
_.each(lines, function (line) {
buf += "// " + (line + padding).slice(0, bannerWidth - 6) + " //\n";
});
buf += spacer + divider;
return buf;
};
var dividerLine = function (bannerWidth) {
return new Array(bannerWidth + 1).join('/') + "\n";
};
var bannerPadding = function (bannerWidth) {
return new Array(bannerWidth + 1).join(' ');
};
_.extend(Unit.prototype, {
// Return the globals in unit file as an array of symbol names. For
// example: if the code references 'Foo.bar.baz' and 'Quux', and
// neither are declared in a scope enclosing the point where they're
// referenced, then globalReferences would include ["Foo", "Quux"].
//
// XXX Doing this at the unit level means that we need to also look
// for var declarations in various units, and use them to create
// a graph of unit dependencies such that in:
// // @unit X
// var A;
// // @unit Y
// A = 5;
// including Y requires including X. Since we don't do that, @unit
// is currently broken. It's also unused and undocumented :)
computeGlobalReferences: function () {
var self = this;
var jsAnalyze = self.file.module.jsAnalyze;
// If we don't have a JSAnalyze object, we probably are the js-analyze
// package itself. Assume we have no global references. At the module level,
// we'll assume that exports are global references.
if (!jsAnalyze)
return [];
try {
return _.keys(jsAnalyze.findAssignedGlobals(self.source));
} catch (e) {
if (!e.$ParseError)
throw e;
buildmessage.error(e.description, {
file: self.file.sourcePath,
line: self.lineOffset === null ? null : e.lineNumber + self.lineOffset,
column: self.lineOffset === null ? null : e.column,
downcase: true
});
// Recover by pretending that this unit is empty (which
// includes replacing its source code with '' in the output)
self.source = "";
return [];
}
}
});
///////////////////////////////////////////////////////////////////////////////
// Top-level entry point
@@ -596,12 +567,10 @@ _.extend(Unit.prototype, {
// bundle, not in error messages, but it's still nice to make it
// look good)
// - sourcePath: path to use in error messages
// - includePositionInErrors: true to include line and column
// information in errors. set to false if, eg, this is the output
// of coffeescript. (XXX replace with real sourcemaps)
// - linkerUnitTransform: if given, this function will be called
// when the module is being linked with the source of the unit
// and an array of the exports of the module; the unit's source will
// - sourceMap: an optional source map (as string) for the input file
// - linkerFileTransform: if given, this function will be called
// when the module is being linked with the source of the file
// and an array of the exports of the module; the file's source will
// be replaced by what the function returns.
//
// forceExport: an array of symbols (as dotted strings) to force the
@@ -628,8 +597,9 @@ _.extend(Unit.prototype, {
//
// Output is an object with keys:
// - files: is an array of output files in the same format as inputFiles
// - EXCEPT THAT, for now, sourcePath is omitted and is replaced with
// sourceMap (a string) (XXX)
// - exports: the exports, as a list of string ('Foo', 'Thing.Stuff', etc)
// - boundary: an opaque value that must be passed along with 'files' to link()
var prelink = function (options) {
if (options.noExports && options.forceExport &&
! _.isEmpty(options.forceExport)) {
@@ -650,16 +620,39 @@ var prelink = function (options) {
module.addFile(inputFile);
});
var files = module.getLinkedFiles();
// 1) Figure out what this entire module exports.
// 2) Run the linkerFileTransforms, which depend on the exports. (This is, eg,
// CoffeeScript arranging to not close over the exports.)
// 3) Do static analysis to compute module-scoped variables; this has to be
// done based on the *output* of the transforms. Error recovery from the
// static analysis mutates the sources, so this has to be done before
// concatenation.
// 4) Finally, concatenate.
var exports = module.getExports();
module.runLinkerFileTransforms(exports);
var packageScopeVariables = module.computeModuleScopeVars(exports);
var files = module.getPrelinkedFiles(exports);
return {
files: files,
exports: exports,
boundary: module.boundary
packageScopeVariables: packageScopeVariables
};
};
var SOURCE_MAP_INSTRUCTIONS_COMMENT = banner([
"This is a generated file. You can view the original",
"source in your browser if your browser supports source maps.",
"",
"If you are using Chrome, open the Developer Tools and click the gear",
"icon in its lower right corner. In the General Settings panel, turn",
"on 'Enable source maps'.",
"",
"If you are using Firefox 23, go to `about:config` and set the",
"`devtools.debugger.source-maps-enabled` preference to true.",
"(The preference should be on by default in Firefox 24; versions",
"older than 23 do not support source maps.)"
]);
// Finish the linking.
//
@@ -673,35 +666,71 @@ var prelink = function (options) {
//
// prelinkFiles: the 'files' output from prelink()
//
// boundary: the 'boundary' output from prelink()
//
// Output is an array of final output files in the same format as the
// 'inputFiles' argument to prelink().
var link = function (options) {
var importCode = options.useGlobalNamespace ?
getImportCode(options.imports, "/* Imports for global scope */\n\n", true) :
getImportCode(options.imports, "/* Imports */\n");
if (options.useGlobalNamespace) {
var ret = [];
if (!_.isEmpty(options.imports)) {
ret.push({
source: getImportCode(options.imports,
"/* Imports for global scope */\n\n", true),
servePath: options.importStubServePath
});
}
return ret.concat(options.prelinkFiles);
}
var header = getHeader({
imports: options.imports,
packageScopeVariables: options.packageScopeVariables
});
var footer = getFooter({
exports: options.exports,
name: options.name
});
var ret = [];
_.each(options.prelinkFiles, function (file) {
var source = file.source;
var parts = source.split(options.boundary);
if (parts.length > 2)
throw new Error("Boundary appears more than once?");
if (parts.length === 2) {
source = parts[0] + importCode + parts[1];
if (source.length === 0)
return; // empty global-imports file -- elide
if (file.sourceMap) {
var chunks = [header];
if (options.includeSourceMapInstructions)
chunks.push("\n" + SOURCE_MAP_INSTRUCTIONS_COMMENT + "\n\n");
chunks.push(sourcemap.SourceNode.fromStringWithSourceMap(
file.source, new sourcemap.SourceMapConsumer(file.sourceMap)));
chunks.push(footer);
var node = new sourcemap.SourceNode(null, null, null, chunks);
var results = node.toStringWithSourceMap({
file: file.servePath
});
ret.push({
source: results.code,
servePath: file.servePath,
sourceMap: results.map.toString()
});
} else {
ret.push({
source: header + file.source + footer,
servePath: file.servePath
});
}
ret.push({
source: source,
servePath: file.servePath
});
});
return ret;
};
var getHeader = function (options) {
var chunks = [];
chunks.push("(function () {\n\n" );
chunks.push(getImportCode(options.imports, "/* Imports */\n"));
if (options.packageScopeVariables
&& !_.isEmpty(options.packageScopeVariables)) {
chunks.push("/* Package-scope variables */\n");
chunks.push("var " + options.packageScopeVariables.join(', ') + ";\n\n");
}
return chunks.join('');
};
var getImportCode = function (imports, header, omitvar) {
var self = this;
@@ -712,10 +741,10 @@ var getImportCode = function (imports, header, omitvar) {
_.each(imports, function (name, symbol) {
scratch[symbol] = packageDot(name) + "." + symbol;
});
var imports = buildSymbolTree(scratch);
var tree = buildSymbolTree(scratch);
var buf = header;
_.each(imports, function (node, key) {
_.each(tree, function (node, key) {
buf += (omitvar ? "" : "var " ) +
key + " = " + writeSymbolTree(node) + ";\n";
});
@@ -725,6 +754,37 @@ var getImportCode = function (imports, header, omitvar) {
return buf;
};
var getFooter = function (options) {
var chunks = [];
if (options.name && options.exports && !_.isEmpty(options.exports)) {
chunks.push("/* Exports */\n");
chunks.push("if (typeof Package === 'undefined') Package = {};\n");
chunks.push(packageDot(options.name), " = ");
// Even if there are no exports, we need to define Package.foo, because the
// existence of Package.foo is how another package (eg, one that weakly
// depends on foo) can tell if foo is loaded.
if (_.isEmpty(options.exports)) {
chunks.push("{};\n");
} else {
// Given exports like Foo, Bar.Baz, Bar.Quux.A, and Bar.Quux.B,
// construct an expression like
// {Foo: Foo, Bar: {Baz: Bar.Baz, Quux: {A: Bar.Quux.A, B: Bar.Quux.B}}}
var scratch = {};
_.each(options.exports, function (symbol) {
scratch[symbol] = symbol;
});
var exportTree = buildSymbolTree(scratch);
chunks.push(writeSymbolTree(exportTree, 0));
chunks.push(";\n");
}
}
chunks.push("\n})();\n");
return chunks.join('');
};
var linker = module.exports = {
prelink: prelink,
link: link

139
tools/packages.js
View File

@@ -12,6 +12,7 @@ var archinfo = require(path.join(__dirname, 'archinfo.js'));
var linker = require(path.join(__dirname, 'linker.js'));
var unipackage = require('./unipackage.js');
var fs = require('fs');
var sourcemap = require('source-map');
// Find all files under `rootPath` that have an extension in
// `extensions` (an array of extensions without leading dot), and
@@ -61,6 +62,12 @@ var scanForSources = function (rootPath, extensions, ignoreFiles) {
});
};
var rejectBadPath = function (p) {
if (p.match(/\.\./))
throw new Error("bad path: " + p);
};
///////////////////////////////////////////////////////////////////////////////
// Slice
///////////////////////////////////////////////////////////////////////////////
@@ -147,14 +154,15 @@ var Slice = function (pkg, options) {
// Are we allowed to have exports? (eg, test slices don't export.)
self.noExports = !!options.noExports;
// Prelink output. 'boundary' is a magic cookie used for inserting
// imports. 'prelinkFiles' is the partially linked JavaScript code
// (an array of objects with keys 'source' and 'servePath', both
// strings -- see prelink() in linker.js) Both of these are inputs
// into the final link phase, which inserts the final JavaScript
// resources into 'resources'. Set only when isBuilt is true.
self.boundary = null;
// Prelink output. 'prelinkFiles' is the partially linked JavaScript code (an
// array of objects with keys 'source' and 'servePath', both strings -- see
// prelink() in linker.js) 'packageScopeVariables' are are variables that are
// syntactically globals in our input files and which we capture with a
// package-scope closure. Both of these are inputs into the final link phase,
// which inserts the final JavaScript resources into 'resources'. Set only
// when isBuilt is true.
self.prelinkFiles = null;
self.packageScopeVariables = null;
// All of the data provided for eventual inclusion in the bundle,
// other than JavaScript that still needs to be fed through the
@@ -173,6 +181,8 @@ var Slice = function (pkg, options) {
// honored for "static", ignored for "head" and "body", sometimes
// honored for CSS but ignored if we are concatenating.
//
// sourceMap: Allowed only for "js". If present, a string.
//
// Set only when isBuilt is true.
self.resources = null;
@@ -191,7 +201,7 @@ _.extend(Slice.prototype, {
// through the appropriate handlers and run the prelink phase on any
// resulting JavaScript. Also add all provided source files to the
// package dependencies. Sets fields such as dependencies, exports,
// boundary, prelinkFiles, and resources.
// prelinkFiles, packageScopeVariables, and resources.
build: function () {
var self = this;
var isApp = ! self.pkg.name;
@@ -258,6 +268,8 @@ _.extend(Slice.prototype, {
// can ensure that the version of the file that you use is
// exactly the one that is recorded in the dependency
// information.
// - pathForSourceMap: If this file is to be included in a source map,
// this is the name you should use for it in the map.
// - rootOutputPath: on browser targets, for resources such as
// stylesheet and static assets, this is the root URL that
// will get prepended to the paths you pick for your output
@@ -282,7 +294,7 @@ _.extend(Slice.prototype, {
// effect, such as minification.)
// - addJavaScript({ path: "my/program.js", data: "my code",
// sourcePath: "src/my/program.js",
// lineForLine: true, bare: true})
// bare: true })
// Add JavaScript code, which will be namespaced into this
// package's environment (eg, it will see only the exports of
// this package's imports), and which will be subject to
@@ -291,11 +303,7 @@ _.extend(Slice.prototype, {
// that will be used in any error messages generated (eg,
// "foo.js:4:1: syntax error"). It must be present and should
// be relative to the project root. Typically 'inputPath' will
// do handsomely. Set the misleadingly named lineForLine
// option to true if line X, column Y in the input corresponds
// to line X, column Y in the output. This will enable line
// and column reporting in error messages. (XXX replace this
// with source maps) "bare" means to not wrap the file in
// do handsomely. "bare" means to not wrap the file in
// a closure, so that its vars are shared with other files
// in the module.
// - addAsset({ path: "my/image.png", data: Buffer })
@@ -308,11 +316,11 @@ _.extend(Slice.prototype, {
// Assets.getText or Assets.getBinary.
// - error({ message: "There's a problem in your source file",
// sourcePath: "src/my/program.ext", line: 12,
// column: 20, columnEnd: 25, func: "doStuff" })
// column: 20, func: "doStuff" })
// Flag an error -- at a particular location in a source
// file, if you like (you can even indicate a function name
// to show in the error, like in stack traces.) sourcePath,
// line, column, columnEnd, and func are all optional.
// line, column, and func are all optional.
//
// XXX for now, these handlers must only generate portable code
// (code that isn't dependent on the arch, other than 'browser'
@@ -350,6 +358,14 @@ _.extend(Slice.prototype, {
inputSize: contents.length,
inputPath: relPath,
_fullInputPath: absPath, // avoid, see above..
// XXX duplicates _pathForSourceMap() in linker
pathForSourceMap: (
self.pkg.name
? self.pkg.name + "/" + relPath
: path.basename(relPath)),
// null if this is an app. intended to be used for the sources
// dictionary for source maps.
packageName: self.pkg.name,
rootOutputPath: self.pkg.serveRoot,
arch: self.arch,
fileOptions: fileOptions,
@@ -396,9 +412,9 @@ _.extend(Slice.prototype, {
source: options.data,
sourcePath: options.sourcePath,
servePath: path.join(self.pkg.serveRoot, options.path),
includePositionInErrors: options.lineForLine,
linkerUnitTransform: options.linkerUnitTransform,
bare: !!options.bare
linkerFileTransform: options.linkerFileTransform,
bare: !!options.bare,
sourceMap: options.sourceMap
});
},
addAsset: function (options) {
@@ -450,8 +466,6 @@ _.extend(Slice.prototype, {
combinedServePath: isApp ? null :
"/packages/" + self.pkg.name +
(self.sliceName === "main" ? "" : ("." + self.sliceName)) + ".js",
// XXX report an error if there is a package called global-imports
importStubServePath: '/packages/global-imports.js',
name: self.pkg.name || null,
forceExport: self.forceExport,
noExports: self.noExports,
@@ -475,8 +489,8 @@ _.extend(Slice.prototype, {
});
self.prelinkFiles = results.files;
self.boundary = results.boundary;
self.exports = results.exports;
self.packageScopeVariables = results.packageScopeVariables;
self.resources = resources;
self.isBuilt = true;
},
@@ -530,17 +544,23 @@ _.extend(Slice.prototype, {
var files = linker.link({
imports: imports,
useGlobalNamespace: isApp,
// XXX report an error if there is a package called global-imports
importStubServePath: isApp && '/packages/global-imports.js',
prelinkFiles: self.prelinkFiles,
boundary: self.boundary
exports: self.exports,
packageScopeVariables: self.packageScopeVariables,
includeSourceMapInstructions: archinfo.matches(self.arch, "browser"),
name: self.pkg.name || null
});
// Add each output as a resource
var jsResources = _.map(files, function (file) {
return {
type: "js",
data: new Buffer(file.source, 'utf8'),
data: new Buffer(file.source, 'utf8'), // XXX encoding
servePath: file.servePath,
staticDirectory: self.staticDirectory
staticDirectory: self.staticDirectory,
sourceMap: file.sourceMap
};
});
@@ -590,7 +610,6 @@ _.extend(Slice.prototype, {
data: compileStep.read().toString('utf8'),
path: compileStep.inputPath,
sourcePath: compileStep.inputPath,
lineForLine: true,
// XXX eventually get rid of backward-compatibility "raw" name
bare: compileStep.fileOptions.bare || compileStep.fileOptions.raw
});
@@ -1298,8 +1317,10 @@ _.extend(Package.prototype, {
};
try {
files.runJavaScript(code.toString('utf8'), 'package.js',
{ Package: Package, Npm: Npm });
files.runJavaScript(code.toString('utf8'), {
filename: 'package.js',
symbols: { Package: Package, Npm: Npm }
});
} catch (e) {
buildmessage.exception(e);
@@ -1793,8 +1814,8 @@ _.extend(Package.prototype, {
self.testSlices = mainJson.testSlices;
_.each(mainJson.plugins, function (pluginMeta) {
if (pluginMeta.path.match(/\.\./))
throw new Error("bad path in unipackage");
rejectBadPath(pluginMeta.path);
var plugin = bundler.readJsImage(path.join(dir, pluginMeta.path));
if (! archinfo.matches(archinfo.host(), plugin.arch)) {
@@ -1820,8 +1841,7 @@ _.extend(Package.prototype, {
_.each(mainJson.slices, function (sliceMeta) {
// aggressively sanitize path (don't let it escape to parent
// directory)
if (sliceMeta.path.match(/\.\./))
throw new Error("bad path in unipackage");
rejectBadPath(sliceMeta.path);
var sliceJson = JSON.parse(
fs.readFileSync(path.join(dir, sliceMeta.path)));
var sliceBasePath = path.dirname(path.join(dir, sliceMeta.path));
@@ -1832,8 +1852,7 @@ _.extend(Package.prototype, {
var nodeModulesPath = null;
if (sliceJson.node_modules) {
if (sliceJson.node_modules.match(/\.\./))
throw new Error("bad node_modules path in unipackage");
rejectBadPath(sliceJson.node_modules);
nodeModulesPath = path.join(sliceBasePath, sliceJson.node_modules);
}
@@ -1858,13 +1877,12 @@ _.extend(Package.prototype, {
slice.isBuilt = true;
slice.exports = sliceJson.exports || [];
slice.boundary = sliceJson.boundary;
slice.packageScopeVariables = sliceJson.packageScopeVariables || [];
slice.prelinkFiles = [];
slice.resources = [];
_.each(sliceJson.resources, function (resource) {
if (resource.file.match(/\.\./))
throw new Error("bad resource file path in unipackage");
rejectBadPath(resource.file);
var fd = fs.openSync(path.join(sliceBasePath, resource.file), "r");
try {
@@ -1878,10 +1896,16 @@ _.extend(Package.prototype, {
throw new Error("couldn't read entire resource");
if (resource.type === "prelink") {
slice.prelinkFiles.push({
var prelinkFile = {
source: data.toString('utf8'),
servePath: resource.servePath
});
};
if (resource.sourceMap) {
rejectBadPath(resource.sourceMap);
prelinkFile.sourceMap = fs.readFileSync(
path.join(sliceBasePath, resource.sourceMap), 'utf8');
}
slice.prelinkFiles.push(prelinkFile);
} else if (_.contains(["head", "body", "css", "js", "static"],
resource.type)) {
slice.resources.push({
@@ -1934,7 +1958,7 @@ _.extend(Package.prototype, {
var buildInfoJson = {
dependencies: { files: {}, directories: {} },
source: options.buildOfPath || undefined,
source: options.buildOfPath || undefined
};
builder.reserve("unipackage.json");
@@ -1977,6 +2001,7 @@ _.extend(Package.prototype, {
var sliceJson = {
format: "unipackage-slice-pre1",
exports: slice.exports,
packageScopeVariables: slice.packageScopeVariables,
uses: _.map(slice.uses, function (u) {
var specParts = u.spec.split('.');
if (specParts.length > 2)
@@ -1990,7 +2015,6 @@ _.extend(Package.prototype, {
}),
node_modules: slice.nodeModulesPath ? 'npm/node_modules' : undefined,
resources: [],
boundary: slice.boundary,
staticDirectory: path.join(sliceDir, self.serveRoot)
};
@@ -2028,13 +2052,11 @@ _.extend(Package.prototype, {
if (_.contains(["head", "body"], resource.type))
return; // already did this one
var resourcePath = builder.generateFilename(
path.join(sliceDir, resource.servePath));
builder.write(resourcePath, { data: resource.data });
sliceJson.resources.push({
type: resource.type,
file: resourcePath,
file: builder.writeToGeneratedFilename(
path.join(sliceDir, resource.servePath),
{ data: resource.data }),
length: resource.data.length,
offset: 0,
servePath: resource.servePath || undefined
@@ -2043,21 +2065,26 @@ _.extend(Package.prototype, {
// Output prelink resources
_.each(slice.prelinkFiles, function (file) {
var resourcePath = builder.generateFilename(
path.join(sliceDir, file.servePath));
var data = new Buffer(file.source, 'utf8');
builder.write(resourcePath, {
data: data
});
sliceJson.resources.push({
var resource = {
type: 'prelink',
file: resourcePath,
file: builder.writeToGeneratedFilename(
path.join(sliceDir, file.servePath),
{ data: data }),
length: data.length,
offset: 0,
servePath: file.servePath || undefined
});
};
if (file.sourceMap) {
// Write the source map.
resource.sourceMap = builder.writeToGeneratedFilename(
path.join(sliceDir, file.servePath + '.map'),
{ data: new Buffer(file.sourceMap, 'utf8') }
);
}
sliceJson.resources.push(resource);
});
// If slice has included node_modules, copy them in

55
tools/server/boot.js
View File

@@ -3,6 +3,7 @@ var fs = require("fs");
var path = require("path");
var Future = require(path.join("fibers", "future"));
var _ = require('underscore');
var sourcemap_support = require('source-map-support');
// This code is duplicated in tools/server/server.js.
var MIN_NODE_VERSION = 'v0.8.24';
@@ -13,15 +14,16 @@ if (require('semver').lt(process.version, MIN_NODE_VERSION)) {
}
// read our control files
var serverJson =
JSON.parse(fs.readFileSync(path.join(__dirname, process.argv[2]), 'utf8'));
var serverJsonPath = path.resolve(process.argv[2]);
var serverDir = path.dirname(serverJsonPath);
var serverJson = JSON.parse(fs.readFileSync(serverJsonPath, 'utf8'));
var configJson =
JSON.parse(fs.readFileSync(path.join(__dirname, 'config.json'), 'utf8'));
JSON.parse(fs.readFileSync(path.resolve(serverDir, 'config.json'), 'utf8'));
// Set up environment
__meteor_bootstrap__ = {
startup_hooks: [],
serverDir: __dirname,
serverDir: serverDir,
configJson: configJson };
__meteor_runtime_config__ = { meteorRelease: configJson.release };
@@ -34,10 +36,49 @@ __meteor_runtime_config__ = { meteorRelease: configJson.release };
if (!process.env.NODE_ENV)
process.env.NODE_ENV = 'production';
// Map from load path to its source map.
var parsedSourceMaps = {};
// Read all the source maps into memory once.
_.each(serverJson.load, function (fileInfo) {
if (fileInfo.sourceMap) {
var rawSourceMap = fs.readFileSync(
path.resolve(serverDir, fileInfo.sourceMap), 'utf8');
// Parse the source map only once, not each time it's needed. Also remove
// the anti-XSSI header if it's there.
var parsedSourceMap = JSON.parse(rawSourceMap.replace(/^\)\]\}'/, ''));
// source-map-support doesn't ever look at the sourcesContent field, so
// there's no point in keeping it in memory.
delete parsedSourceMap.sourcesContent;
var url;
if (fileInfo.sourceMapRoot) {
// Add the specified root to any root that may be in the file.
parsedSourceMap.sourceRoot = path.join(
fileInfo.sourceMapRoot, parsedSourceMap.sourceRoot || '');
}
parsedSourceMaps[fileInfo.path] = parsedSourceMap;
}
});
var retrieveSourceMap = function (pathForSourceMap) {
if (_.has(parsedSourceMaps, pathForSourceMap))
return { map: parsedSourceMaps[pathForSourceMap] };
return null;
};
sourcemap_support.install({
// Use the source maps specified in program.json instead of parsing source
// code for them.
retrieveSourceMap: retrieveSourceMap,
// For now, don't fix the source line in uncaught exceptions, because we
// haven't fixed handleUncaughtExceptions in source-map-support to properly
// locate the source files.
handleUncaughtExceptions: false
});
Fiber(function () {
_.each(serverJson.load, function (fileInfo) {
var code = fs.readFileSync(path.join(__dirname, fileInfo.path));
var code = fs.readFileSync(path.resolve(serverDir, fileInfo.path));
var Npm = {
require: function (name) {
@@ -46,7 +87,7 @@ Fiber(function () {
}
var nodeModuleDir =
path.join(__dirname, fileInfo.node_modules, name);
path.resolve(serverDir, fileInfo.node_modules, name);
if (fs.existsSync(nodeModuleDir)) {
return require(nodeModuleDir);
@@ -67,7 +108,7 @@ Fiber(function () {
}
}
};
var staticDirectory = path.join(__dirname, fileInfo.staticDirectory);
var staticDirectory = path.resolve(serverDir, fileInfo.staticDirectory);
var getAsset = function (assetPath, encoding, callback) {
var fut;
if (! callback) {

2
tools/tests/test_bundler.js
View File

@@ -12,7 +12,7 @@
/*global*/ Fiber = require('fibers');
/*global*/ Future = require('fibers/future');
/*global*/ mainJSContents = "process.argv.splice(2, 0, 'program.json');\nrequire('./programs/server/boot.js');\n";
/*global*/ mainJSContents = "process.argv.splice(2, 0, 'program.json');\nprocess.chdir(require('path').join(__dirname, 'programs', 'server'));\nrequire('./programs/server/boot.js');\n";
var tmpBaseDir = files.mkdtemp('test_bundler');
var tmpCounter = 1;