diff --git a/meteor b/meteor index be28497200..58613553c4 100755 --- a/meteor +++ b/meteor @@ -1,6 +1,6 @@ #!/bin/bash -BUNDLE_VERSION=0.3.10 +BUNDLE_VERSION=0.3.11 # OS Check. Put here because here is where we download the precompiled # bundles that are arch specific. diff --git a/packages/coffeescript/.npm/plugin/compileCoffeescript/npm-shrinkwrap.json b/packages/coffeescript/.npm/plugin/compileCoffeescript/npm-shrinkwrap.json index ae2363c418..5df65e6c16 100644 --- a/packages/coffeescript/.npm/plugin/compileCoffeescript/npm-shrinkwrap.json +++ b/packages/coffeescript/.npm/plugin/compileCoffeescript/npm-shrinkwrap.json @@ -2,6 +2,14 @@ "dependencies": { "coffee-script": { "version": "1.6.3" + }, + "source-map": { + "version": "0.1.24", + "dependencies": { + "amdefine": { + "version": "0.0.5" + } + } } } } diff --git a/packages/coffeescript/package.js b/packages/coffeescript/package.js index f7db750b0c..2a13f962fb 100644 --- a/packages/coffeescript/package.js +++ b/packages/coffeescript/package.js @@ -8,7 +8,7 @@ Package._transitional_registerBuildPlugin({ sources: [ 'plugin/compile-coffeescript.js' ], - npmDependencies: {"coffee-script": "1.6.3"} + npmDependencies: {"coffee-script": "1.6.3", "source-map": "0.1.24"} }); Package.on_test(function (api) { diff --git a/packages/coffeescript/plugin/compile-coffeescript.js b/packages/coffeescript/plugin/compile-coffeescript.js index 58fc659788..988eb53188 100644 --- a/packages/coffeescript/plugin/compile-coffeescript.js +++ b/packages/coffeescript/plugin/compile-coffeescript.js @@ -2,6 +2,7 @@ var fs = Npm.require('fs'); var path = Npm.require('path'); var coffee = Npm.require('coffee-script'); var _ = Npm.require('underscore'); +var sourcemap = Npm.require('source-map'); var stripExportedVars = function (source, exports) { if (!exports || _.isEmpty(exports)) @@ -28,33 +29,50 @@ var stripExportedVars = function (source, exports) { // XXX relax these assumptions by doing actual JS parsing (eg with jsparse). // I'd do this now, but there's no easy way to "unparse" a jsparse AST. - var foundVarLine = false; - lines = _.map(lines, function (line) { - if (foundVarLine) - return line; + for (var i = 0; i < lines.length; i++) { + var line = lines[i]; var match = /^var (.+)([,;])$/.exec(line); if (!match) - return line; - foundVarLine = true; + continue; // If there's an assignment on this line, we assume that there are ONLY // assignments and that the var we are looking for is not declared. (Part // of our strong assumption about the layout of this code.) if (match[1].indexOf('=') !== -1) - return line; + continue; + + // We want to replace the line with something no shorter, so that all + // records in the source map continue to point at valid + // characters. + var replaceLine = function (x) { + if (x.length >= lines[i].length) { + lines[i] = x; + } else { + lines[i] = x + new Array(1 + (lines[i].length - x.length)).join(' '); + } + }; var vars = match[1].split(', '); vars = _.difference(vars, exports); - if (!_.isEmpty(vars)) - return "var " + vars.join(', ') + match[2]; - // We got rid of all the vars on this line. Drop the whole line if this - // didn't continue to the next line, otherwise keep just the 'var '. - return match[2] === ';' ? '' : 'var'; - }); + if (!_.isEmpty(vars)) { + replaceLine("var " + vars.join(', ') + match[2]); + } else { + // We got rid of all the vars on this line. Drop the whole line if this + // didn't continue to the next line, otherwise keep just the 'var '. + if (match[2] === ';') + replaceLine(''); + else + replaceLine('var'); + } + break; + } + return lines.join('\n'); }; -var addSharedHeader = function (source) { +var addSharedHeader = function (source, sourceMap) { + var sourceMapJSON = JSON.parse(sourceMap); + // We want the symbol "share" to be visible to all CoffeeScript files in the // package (and shared between them), but not visible to JavaScript // files. (That's because we don't want to introduce two competing ways to @@ -73,17 +91,42 @@ var addSharedHeader = function (source) { // If the file begins with "use strict", we need to keep that as the first // statement. - return source.replace(/^(?:(['"])use strict\1;\n)?/, function (match) { - return match + header; + source = source.replace(/^(?:((['"])use strict\2;)\n)?/, function (match, useStrict) { + if (match) { + // There's a "use strict"; we keep this as the first statement and insert + // our header at the end of the line that it's on. This doesn't change + // line numbers or the part of the line that previous may have been + // annotated, so we don't need to update the source map. + return useStrict + " " + header; + } else { + // There's no use strict, so we can just add the header at the very + // beginning. This adds a line to the file, so we update the source map to + // add a single un-annotated line to the beginning. + sourceMapJSON.mappings = ";" + sourceMapJSON.mappings; + return header; + } }); + return { + source: source, + sourceMap: JSON.stringify(sourceMapJSON) + }; }; var handler = function (compileStep) { var source = compileStep.read().toString('utf8'); + var outputFile = compileStep.inputPath + ".js"; var options = { bare: true, filename: compileStep.inputPath, - literate: path.extname(compileStep.inputPath) === '.litcoffee' + literate: path.extname(compileStep.inputPath) === '.litcoffee', + // Return a source map. + sourceMap: true, + // Include the original source in the source map (sourcesContent field). + inline: true, + // This becomes the "file" field of the source map. + generatedFile: "/" + outputFile, + // This becomes the "sources" field of the source map. + sourceFiles: [compileStep.pathForSourceMap] }; try { @@ -98,13 +141,14 @@ var handler = function (compileStep) { } compileStep.addJavaScript({ - path: compileStep.inputPath + ".js", + path: outputFile, sourcePath: compileStep.inputPath, - data: output, - lineForLine: false, - linkerUnitTransform: function (source, exports) { - return addSharedHeader(stripExportedVars(source, exports)); - } + data: output.js, + linkerFileTransform: function (sourceWithMap, exports) { + var stripped = stripExportedVars(sourceWithMap.source, exports); + return addSharedHeader(stripped, sourceWithMap.sourceMap); + }, + sourceMap: output.v3SourceMap }); }; diff --git a/packages/http/httpcall_tests.js b/packages/http/httpcall_tests.js index 115806ab05..fddf48b032 100644 --- a/packages/http/httpcall_tests.js +++ b/packages/http/httpcall_tests.js @@ -441,33 +441,37 @@ if (Meteor.isServer) { })); }; - // no such file - do_test("/nosuchfile", 200, /DOCTYPE/); - do_test("/../nosuchfile", 403); - do_test("/%2e%2e/nosuchfile", 403); - do_test("/%2E%2E/nosuchfile", 403); - do_test("/%2d%2d/nosuchfile", 200, /DOCTYPE/); - // existing static file - var succeeds = [ - "/packages/http/test_static.serveme", + do_test("/packages/http/test_static.serveme", 200, /static file serving/); + + // no such file, so return the default app HTML. + var getsAppHtml = [ + // This file doesn't exist. + "/nosuchfile", + + // Our static file serving doesn't process .. or its encoded version, so + // any of these return the app HTML. + "/../nosuchfile", + "/%2e%2e/nosuchfile", + "/%2E%2E/nosuchfile", + "/%2d%2d/nosuchfile", "/packages/http/../http/test_static.serveme", "/packages/http/%2e%2e/http/test_static.serveme", "/packages/http/%2E%2E/http/test_static.serveme", "/packages/http/../../packages/http/test_static.serveme", "/packages/http/%2e%2e/%2e%2e/packages/http/test_static.serveme", "/packages/http/%2E%2E/%2E%2E/packages/http/test_static.serveme", + + // ... and they *definitely* shouldn't be able to escape the app bundle. + "/packages/http/../../../../../../packages/http/test_static.serveme", + "/../../../../../../../../../../../bin/ls", + "/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/bin/ls", + "/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/bin/ls" ]; - _.each(succeeds, function (path) { - do_test(path, 200, /static file serving/); + + _.each(getsAppHtml, function (x) { + do_test(x, 200, /Tests<\/title/); }); - do_test("/packages/http/../../../../../../packages/http/test_static.serveme", 403); - - // file outside of our app - do_test("/../../../../../../../../../../../bin/ls", 403); - do_test("/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/%2e%2e/bin/ls", 403); - do_test("/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/%2E%2E/bin/ls", 403); - } ]); } diff --git a/packages/js-analyze/js_analyze.js b/packages/js-analyze/js_analyze.js index c1d7ba6a02..3a142fbfb5 100644 --- a/packages/js-analyze/js_analyze.js +++ b/packages/js-analyze/js_analyze.js @@ -148,7 +148,7 @@ JSAnalyze.findAssignedGlobals = function (source) { // causes escope to not bother to resolve references in the eval's scope. // This is because an eval can pull references inward: // - / function outer() { + // function outer() { // var i = 42; // function inner() { // eval('var i = 0'); diff --git a/packages/logging/logging.js b/packages/logging/logging.js index 227e99c782..db33a5ca07 100644 --- a/packages/logging/logging.js +++ b/packages/logging/logging.js @@ -75,11 +75,11 @@ var logInBrowser = function (obj) { // @returns {Object: { line: Number, file: String }} Log._getCallerDetails = function () { var getStack = function () { - var orig = Error.prepareStackTrace; - Error.prepareStackTrace = function(_, stack){ return stack; }; + // We do NOT use Error.prepareStackTrace here (a V8 extension that gets us a + // pre-parsed stack) since it's impossible to compose it with the use of + // Error.prepareStackTrace used on the server for source maps. var err = new Error; var stack = err.stack; - Error.prepareStackTrace = orig; return stack; }; @@ -87,33 +87,37 @@ Log._getCallerDetails = function () { if (!stack) return {}; - var isV8 = false; - var lines = stack; - // check for V8 specifics - if (_.isArray(stack)) - isV8 = true; - else - lines = stack.split('\n'); - var index = 1; - var line = lines[index]; + var lines = stack.split('\n'); - // looking for the first line outside the logging package - while ((isV8 ? line.getFileName() || '' : line) - .indexOf('/packages/logging.js') !== -1) - line = lines[++index]; + // looking for the first line outside the logging package (or an + // eval if we find that first) + var line; + for (var i = 1; i < lines.length; ++i) { + line = lines[i]; + if (line.match(/^\s*at eval \(eval/)) { + return {file: "eval"}; + } + + // XXX probably wants to be / or .js in case no source maps + if (!line.match(/packages\/logging(?:\/|(?:\.tests)?\.js)/)) + break; + } var details = {}; - // The format for FF is functionName@filePath:lineNumber - // For V8 call built-in function - details.line = isV8 ? line.getLineNumber() : line.split(':').slice(-1)[0]; + // The format for FF is 'functionName@filePath:lineNumber' + // The format for V8 is 'functionName (packages/logging/logging.js:81)' or + // 'packages/logging/logging.js:81' + var match = /(?:[@(]| at )([^(]+?):([0-9:]+)(?:\)|$)/.exec(line); + if (!match) + return details; + // in case the matched block here is line:column + details.line = match[2].split(':')[0]; // Possible format: https://foo.bar.com/scripts/file.js?random=foobar - // For FF we parse the line, for V8 we call built-in function // XXX: if you can write the following in better way, please do it - details.file = isV8 ? line.getFileName() || (line.isEval() ? 'eval' : '') - : line.split('@')[1].split(':').slice(0, -1).join(':'); - details.file = details.file.split('/').slice(-1)[0].split('?')[0]; + // XXX: what about evals? + details.file = match[1].split('/').slice(-1)[0].split('?')[0]; return details; }; diff --git a/packages/logging/logging_test.js b/packages/logging/logging_test.js index 16cc748f7e..4305abf7e0 100644 --- a/packages/logging/logging_test.js +++ b/packages/logging/logging_test.js @@ -4,14 +4,11 @@ Tinytest.add("logging - _getCallerDetails", function (test) { // in Chrome and Firefox, other browsers don't give us an ability to get // stacktrace. if ((new Error).stack) { - //test.equal(details, { file: 'logging_test.js', line: 2 }); - // XXX: When we have source maps, we should uncomment the test above and - // remove this one - test.isTrue(details.file === 'logging.tests.js'); + test.equal(details.file, 'tinytest.js'); // evaled statements shouldn't crash var code = "Log._getCallerDetails().file"; - test.matches(eval(code), /^eval|logging.tests.js$/); + test.matches(eval(code), /^eval|tinytest.js$/); } }); diff --git a/packages/meteor/package.js b/packages/meteor/package.js index 1d7cfb9622..95b38dfe9f 100644 --- a/packages/meteor/package.js +++ b/packages/meteor/package.js @@ -24,7 +24,6 @@ Package.on_use(function (api, where) { // dynamic variables, bindEnvironment // XXX move into a separate package? - api.use('underscore', ['client', 'server']); api.add_files('dynamics_browser.js', 'client'); api.add_files('dynamics_nodejs.js', 'server'); diff --git a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json index 03ac6db249..414c6ec42f 100644 --- a/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json +++ b/packages/mongo-livedata/.npm/package/npm-shrinkwrap.json @@ -7,7 +7,7 @@ "version": "0.1.9" }, "kerberos": { - "version": "0.0.2" + "version": "0.0.3" } } } diff --git a/packages/templating/plugin/compile-templates.js b/packages/templating/plugin/compile-templates.js index 1effe1a718..7f22bf6b83 100644 --- a/packages/templating/plugin/compile-templates.js +++ b/packages/templating/plugin/compile-templates.js @@ -42,11 +42,12 @@ Plugin.registerSourceHandler("html", function (compileStep) { var ext = path.extname(compileStep.inputPath); var basename = path.basename(compileStep.inputPath, ext); + // XXX generate a source map + compileStep.addJavaScript({ path: path.join(path_part, "template." + basename + ".js"), sourcePath: compileStep.inputPath, - data: results.js, - lineForLine: false + data: results.js }); } -}); \ No newline at end of file +}); diff --git a/packages/webapp/.npm/package/npm-shrinkwrap.json b/packages/webapp/.npm/package/npm-shrinkwrap.json index 8b3977779b..887dcb5e04 100644 --- a/packages/webapp/.npm/package/npm-shrinkwrap.json +++ b/packages/webapp/.npm/package/npm-shrinkwrap.json @@ -43,6 +43,23 @@ } } }, + "send": { + "version": "0.1.0", + "dependencies": { + "debug": { + "version": "0.7.2" + }, + "mime": { + "version": "1.2.6" + }, + "fresh": { + "version": "0.1.0" + }, + "range-parser": { + "version": "0.0.4" + } + } + }, "useragent": { "version": "2.0.1" } diff --git a/packages/webapp/package.js b/packages/webapp/package.js index e22134dfc5..351762533a 100644 --- a/packages/webapp/package.js +++ b/packages/webapp/package.js @@ -4,6 +4,7 @@ Package.describe({ }); Npm.depends({connect: "2.7.10", + send: "0.1.0", useragent: "2.0.1"}); Package.on_use(function (api) { diff --git a/packages/webapp/webapp_server.js b/packages/webapp/webapp_server.js index 0b49d60e60..3490d7eb09 100644 --- a/packages/webapp/webapp_server.js +++ b/packages/webapp/webapp_server.js @@ -5,10 +5,12 @@ var http = Npm.require("http"); var os = Npm.require("os"); var path = Npm.require("path"); var url = Npm.require("url"); +var crypto = Npm.require("crypto"); var connect = Npm.require('connect'); var optimist = Npm.require('optimist'); var useragent = Npm.require('useragent'); +var send = Npm.require('send'); // @export WebApp WebApp = {}; @@ -49,6 +51,12 @@ var initKeepalive = function () { }; +var sha1 = function (contents) { + var hash = crypto.createHash('sha1'); + hash.update(contents); + return hash.digest('hex'); +}; + // #BrowserIdentification // // We have multiple places that want to identify the browser: the @@ -207,34 +215,121 @@ var runWebAppServer = function () { next(); } }); - // Parse the query string into res.query. Only oauth_server cares about this, - // but it's overkill to have that package depend on its own copy of connect - // just for this simple processing. + // Parse the query string into res.query. Used by oauth_server, but it's + // generally pretty handy.. app.use(connect.query()); // Auto-compress any json, javascript, or text. app.use(connect.compress()); - if (clientJson.staticCacheable) { - // cacheable files are files that should never change. Typically - // named by their hash (eg meteor bundled js and css files). - // cache them ~forever (1yr) - app.use(connect.static(path.join(clientDir, clientJson.staticCacheable), - {maxAge: 1000 * 60 * 60 * 24 * 365})); - } + var staticFiles = {}; + _.each(clientJson.manifest, function (item) { + if (item.url && item.where === "client") { + staticFiles[url.parse(item.url).pathname] = { + path: item.path, + cacheable: item.cacheable, + // Link from source to its map + sourceMapUrl: item.sourceMapUrl + }; - // cache non-cacheable file anyway. This isn't really correct, as - // users can change the files and changes won't propogate - // immediately. However, if we don't cache them, browsers will - // 'flicker' when rerendering images. Eventually we will probably want - // to rewrite URLs of static assets to include a query parameter to - // bust caches. That way we can both get good caching behavior and - // allow users to change assets without delay. - // https://github.com/meteor/meteor/issues/773 - if (clientJson.static) { - app.use(connect.static(path.join(clientDir, clientJson.static), - {maxAge: 1000 * 60 * 60 * 24})); - } + if (item.sourceMap) { + // Serve the source map too, under the specified URL. We assume all + // source maps are cacheable. + staticFiles[url.parse(item.sourceMapUrl).pathname] = { + path: item.sourceMap, + cacheable: true + }; + } + } + }); + + // Serve static files from the manifest. + // This is inspired by the 'static' middleware. + app.use(function (req, res, next) { + if ('GET' != req.method && 'HEAD' != req.method) { + next(); + return; + } + var pathname = connect.utils.parseUrl(req).pathname; + + try { + pathname = decodeURIComponent(pathname); + } catch (e) { + next(); + return; + } + if (!_.has(staticFiles, pathname)) { + next(); + return; + } + + // We don't need to call pause because, unlike 'static', once we call into + // 'send' and yield to the event loop, we never call another handler with + // 'next'. + + var info = staticFiles[pathname]; + + // Cacheable files are files that should never change. Typically + // named by their hash (eg meteor bundled js and css files). + // We cache them ~forever (1yr). + // + // We cache non-cacheable files anyway. This isn't really correct, as users + // can change the files and changes won't propagate immediately. However, if + // we don't cache them, browsers will 'flicker' when rerendering + // images. Eventually we will probably want to rewrite URLs of static assets + // to include a query parameter to bust caches. That way we can both get + // good caching behavior and allow users to change assets without delay. + // https://github.com/meteor/meteor/issues/773 + var maxAge = info.cacheable + ? 1000 * 60 * 60 * 24 * 365 + : 1000 * 60 * 60 * 24; + + // Set the X-SourceMap header, which current Chrome understands. + // (The files also contain '//#' comments which FF 24 understands and + // Chrome doesn't understand yet.) + // + // Eventually we should set the SourceMap header but the current version of + // Chrome and no version of FF supports it. + // + // To figure out if your version of Chrome should support the SourceMap + // header, + // - go to chrome://version. Let's say the Chrome version is + // 28.0.1500.71 and the Blink version is 537.36 (@153022) + // - go to http://src.chromium.org/viewvc/blink/branches/chromium/1500/Source/core/inspector/InspectorPageAgent.cpp?view=log + // where the "1500" is the third part of your Chrome version + // - find the first revision that is no greater than the "153022" + // number. That's probably the first one and it probably has + // a message of the form "Branch 1500 - blink@r149738" + // - If *that* revision number (149738) is at least 151755, + // then Chrome should support SourceMap (not just X-SourceMap) + // (The change is https://codereview.chromium.org/15832007) + // + // You also need to enable source maps in Chrome: open dev tools, click + // the gear in the bottom right corner, and select "enable source maps". + // + // Firefox 23+ supports source maps but doesn't support either header yet, + // so we include the '//#' comment for it: + // https://bugzilla.mozilla.org/show_bug.cgi?id=765993 + // In FF 23 you need to turn on `devtools.debugger.source-maps-enabled` + // in `about:config` (it is on by default in FF 24). + if (info.sourceMapUrl) + res.setHeader('X-SourceMap', info.sourceMapUrl); + + send(req, path.join(clientDir, info.path)) + .maxage(maxAge) + .hidden(true) // if we specified a dotfile in the manifest, serve it + .on('error', function (err) { + Log.error("Error serving static file " + err); + res.writeHead(500); + res.end(); + }) + .on('directory', function () { + Log.error("Unexpected directory " + info.path); + res.writeHead(500); + res.end(); + }) + .pipe(res); + }); // Packages and apps can add handlers to this via WebApp.connectHandlers. // They are inserted before our default handler. diff --git a/scripts/generate-dev-bundle.sh b/scripts/generate-dev-bundle.sh index ab37cea852..c97eb65a37 100755 --- a/scripts/generate-dev-bundle.sh +++ b/scripts/generate-dev-bundle.sh @@ -110,6 +110,20 @@ npm install kexec@0.1.1 npm install shell-quote@0.0.1 npm install byline@2.0.3 +# Using the unreleased 1.1 branch. We can probably switch to a built NPM version +# when it gets released. +npm install https://github.com/ariya/esprima/tarball/5044b87f94fb802d9609f1426c838874ec2007b3 + +# Fork of source-map which fixes one function with empty maps. +# https://github.com/mozilla/source-map/pull/70 +# See also below, where we get it into source-map-support. +npm install https://github.com/meteor/source-map/tarball/4a52398901fdb4b55b06ef4dd8b69f8256072b09 + +# Fork of node-source-map-support which allows us to specify our own +# retrieveSourceMap function, and uses the above version of source-map. +# XXX send a pull request +npm install https://github.com/meteor/node-source-map-support/tarball/d048eaa765bf743ddaad64716647f8760e2b8507 + # uglify-js has a bug which drops 'undefined' in arrays: # https://github.com/mishoo/UglifyJS2/pull/97 npm install https://github.com/meteor/UglifyJS2/tarball/aa5abd14d3 diff --git a/tools/builder.js b/tools/builder.js index ba7a00b7d1..d8febca434 100644 --- a/tools/builder.js +++ b/tools/builder.js @@ -272,6 +272,19 @@ _.extend(Builder.prototype, { return relPath; }, + // Convenience wrapper around generateFilename and write. + // + // (Note that in the object returned by builder.enter, this method + // is patched through directly rather than rewriting its inputs and + // outputs. This is only valid because it does nothing with its inputs + // and outputs other than send pass them to other methods.) + writeToGeneratedFilename: function (relPath, writeOptions) { + var self = this; + var generated = self.generateFilename(relPath); + self.write(generated, writeOptions); + return generated; + }, + // Recursively copy a directory and all of its contents into the // bundle. But if the symlink option was passed to the Builder // constructor, then make a symlink instead, if possible. @@ -377,10 +390,11 @@ _.extend(Builder.prototype, { var self = this; var methods = ["write", "writeJson", "reserve", "generateFilename", "copyDirectory", "enter"]; - var ret = {}; + var subBuilder = {}; + var relPathWithSep = relPath + path.sep; _.each(methods, function (method) { - ret[method] = function (/* arguments */) { + subBuilder[method] = function (/* arguments */) { var args = _.toArray(arguments); if (method !== "copyDirectory") { @@ -401,17 +415,24 @@ _.extend(Builder.prototype, { // sub-bundle, not the parent bundle if (ret.substr(0, 1) === '/') ret = ret.substr(1); - if (ret.substr(0, relPath.length) !== relPath) + if (ret.substr(0, relPathWithSep.length) !== relPathWithSep) throw new Error("generateFilename returned path outside of " + "sub-bundle?"); - ret = ret.substr(relPath.length); + ret = ret.substr(relPathWithSep.length); } return ret; }; }); - return ret; + // Methods that don't have to fix up arguments or return values, because + // they are implemented purely in terms of other methods which do. + var passThroughMethods = ["writeToGeneratedFilename"]; + _.each(passThroughMethods, function (method) { + subBuilder[method] = self[method]; + }); + + return subBuilder; }, // Move the completed bundle into its final location (outputPath) diff --git a/tools/buildmessage.js b/tools/buildmessage.js index b8860e6928..db2e8b84ac 100644 --- a/tools/buildmessage.js +++ b/tools/buildmessage.js @@ -349,32 +349,40 @@ var error = function (message, options) { // Record an exception. The message as well as any file and line // information be read directly out of the exception. If not in a job, -// throws the exception instead. The first character of the error's -// message is downcased. Also capture the user portion of the stack. +// throws the exception instead. Also capture the user portion of the stack. // // There is special handling for files.FancySyntaxError exceptions. We // will grab the file and location information where the syntax error // actually occurred, rather than the place where the exception was // thrown. var exception = function (error) { - if (! currentJob) - throw new Error("Error: " + error.message); + if (! currentJob) { + // XXX this may be the wrong place to do this, but it makes syntax errors in + // files loaded via unipackage.load have context. + if (error instanceof files.FancySyntaxError) { + error.message = "Syntax error: " + error.message + " at " + + error.file + ":" + error.line + ":" + error.column; + } + throw error; + } - var message = error.message.slice(0,1).toLowerCase() + error.message.slice(1); + var message = error.message; if (error instanceof files.FancySyntaxError) { + // No stack, because FancySyntaxError isn't a real Error and has no stack + // property! currentJob.addMessage({ message: message, - stack: parseStack(error), file: error.file, line: error.line, column: error.column }); } else { - var locus = parseStack(error)[0]; + var stack = parseStack(error); + var locus = stack[0]; currentJob.addMessage({ message: message, - stack: parseStack(error), + stack: stack, func: locus.func, file: locus.file, line: locus.line, diff --git a/tools/bundler.js b/tools/bundler.js index b2a5deff97..80237fc8b4 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -70,26 +70,16 @@ // parameter when used // - size: size of file in bytes // - hash: sha1 hash of the file contents +// - sourceMap: optional path to source map file (relative to program.json) // Additionally there will be an entry with where equal to // "internal", path equal to page (above), and hash equal to the // sha1 of page (before replacements.) Currently this is used to // trigger HTML5 appcache reloads at the right time (if the // 'appcache' package is being used.) // -// - static: a path, relative to program.json, to a directory. If the -// server is too dumb to read 'manifest', it can just serve all of -// the files in this directory (with a relatively short cache -// expiry time.) -// XXX do not use this. It will go away soon. -// -// - static_cacheable: just like 'static' but resources that can be -// cached aggressively (cacheable: true in the manifest) -// XXX do not use this. It will go away soon. -// // Convention: // -// page is 'app.html', static is 'static', and staticCacheable is -// 'static_cacheable'. +// page is 'app.html'. // // // == Format of a program when arch is "native.*" == @@ -115,6 +105,8 @@ // be search for npm modules // - staticDirectory: directory to search for static assets when // Assets.getText and Assets.getBinary are called from this file. +// - sourceMap: if present, path of a file that contains a source +// map for this file, relative to program.json // // /config.json: // @@ -177,6 +169,7 @@ var builder = require(path.join(__dirname, 'builder.js')); var unipackage = require(path.join(__dirname, 'unipackage.js')); var Fiber = require('fibers'); var Future = require(path.join('fibers', 'future')); +var sourcemap = require('source-map'); // files to ignore when bundling. node has no globs, so use regexps var ignoreFiles = [ @@ -194,6 +187,18 @@ var inherits = function (child, parent) { child.prototype.constructor = child; }; +var rejectBadPath = function (p) { + if (p.match(/\.\./)) + throw new Error("bad path: " + p); +}; + +var stripLeadingSlash = function (p) { + if (p.charAt(0) !== '/') + throw new Error("bad path: " + p); + return p.slice(1); +}; + + /////////////////////////////////////////////////////////////////////////////// // NodeModulesDirectory /////////////////////////////////////////////////////////////////////////////// @@ -231,19 +236,27 @@ var StaticDirectory = function (options) { // Allowed options: // - sourcePath: path to file on disk that will provide our contents // - data: contents of the file as a Buffer +// - sourceMap: if 'data' is given, can be given instead of sourcePath. a string // - cacheable var File = function (options) { var self = this; - if (options.data && ! (options.data instanceof Buffer)) { + if (options.data && ! (options.data instanceof Buffer)) throw new Error('File contents must be provided as a Buffer'); - } + if (! options.sourcePath && ! options.data) + throw new Error("Must provide either sourcePath or data"); // The absolute path in the filesystem from which we loaded (or will // load) this file (null if the file does not correspond to one on // disk.) self.sourcePath = options.sourcePath; + // If this file was generated, a sourceMap (as a string) with debugging + // information, as well as the "root" that paths in it should be resolved + // against. Set with setSourceMap. + self.sourceMap = null; + self.sourceMapRoot = null; + // Where this file is intended to reside within the target's // filesystem. self.targetPath = null; @@ -251,7 +264,7 @@ var File = function (options) { // The URL at which this file is intended to be served, relative to // the base URL at which the target is being served (ignored if this // file is not intended to be served over HTTP.) - self.url = null + self.url = null; // Is this file guaranteed to never change, so that we can let it be // cached forever? Only makes sense of self.url is set. @@ -282,8 +295,9 @@ _.extend(File.prototype, { contents: function (encoding) { var self = this; if (! self._contents) { - if (! self.sourcePath) + if (! self.sourcePath) { throw new Error("Have neither contents nor sourcePath for file"); + } else self._contents = fs.readFileSync(self.sourcePath); } @@ -291,6 +305,15 @@ _.extend(File.prototype, { return encoding ? self._contents.toString(encoding) : self._contents; }, + setContents: function (b) { + var self = this; + if (!(b instanceof Buffer)) + throw new Error("Must set contents to a Buffer"); + self._contents = b; + // Un-cache hash. + self._hash = null; + }, + size: function () { var self = this; return self.contents().length; @@ -304,6 +327,7 @@ _.extend(File.prototype, { var self = this; self.url = "/" + self.hash() + suffix; self.cacheable = true; + self.targetPath = self.hash() + suffix; }, // Append "?<hash>" to the URL and mark the file as cacheable. @@ -336,10 +360,10 @@ _.extend(File.prototype, { setTargetPathFromRelPath: function (relPath) { var self = this; // XXX hack - if (relPath.match(/^\/packages\//) || relPath.match(/^\/static\//)) + if (relPath.match(/^packages\//) || relPath.match(/^static\//)) self.targetPath = relPath; else - self.targetPath = path.join('/app', relPath); + self.targetPath = path.join('app', relPath); }, setStaticDirectory: function (relPath, staticSourceDirectory) { @@ -348,18 +372,30 @@ _.extend(File.prototype, { // static/packages specific to this package. Application assets (e.g. those // inside private/) go in static/app/. // XXX same hack as above + // XXX XXX is this all still true? + // XXX rename static -> assets on server var bundlePath; - if (relPath.match(/^\/packages\//)) { + if (relPath.match(/^packages\//)) { var dir = path.dirname(relPath); var base = path.basename(relPath, ".js"); - bundlePath = path.join('/static', dir, base); + bundlePath = path.join('static', dir, base); } else { - bundlePath = path.join('/static', 'app'); + bundlePath = path.join('static', 'app'); } self.staticDirectory = new StaticDirectory({ sourcePath: staticSourceDirectory, bundlePath: bundlePath }); + }, + + // Set a source map for this File. sourceMap is given as a string. + setSourceMap: function (sourceMap, root) { + var self = this; + + if (typeof sourceMap !== "string") + throw new Error("sourceMap must be given as a string"); + self.sourceMap = sourceMap; + self.sourceMapRoot = root; } }); @@ -431,8 +467,7 @@ _.extend(Target.prototype, { test: options.test || [] }); - // Link JavaScript, put resources in load order, and copy them to - // the bundle + // Link JavaScript and set up self.js, etc. self._emitResources(); // Minify, if requested @@ -442,7 +477,7 @@ _.extend(Target.prototype, { self.minifyCss(); } - // Process asset directories (eg, /public) + // Process asset directories (eg, '/public') // XXX this should probably be part of the appDir reader _.each(options.assetDirs || [], function (ad) { self.addAssetDir(ad); @@ -454,11 +489,6 @@ _.extend(Target.prototype, { self._addCacheBusters("js"); self._addCacheBusters("css"); } - - // XXX extra thing we have to do on the client. could this move - // into ClientTarget.write()? - if (self.assignTargetPaths) - self.assignTargetPaths(); }, // Determine the packages to load, create Slices for @@ -574,9 +604,8 @@ _.extend(Target.prototype, { } }, - // Sort the slices in dependency order, then, slice by slice, write - // their resources into the bundle (which includes running the - // JavaScript linker.) + // Process all of the sorted slices (which includes running the JavaScript + // linker). _emitResources: function () { var self = this; @@ -588,7 +617,7 @@ _.extend(Target.prototype, { var isApp = ! slice.pkg.name; // Emit the resources - _.each(slice.getResources(self.arch), function (resource) { + _.each(slice.getResources(self.arch), function (resource) { if (_.contains(["js", "css", "static"], resource.type)) { if (resource.type === "css" && ! isBrowser) // XXX might be nice to throw an error here, but then we'd @@ -604,15 +633,17 @@ _.extend(Target.prototype, { cacheable: false }); + var relPath; + if (resource.type === "static" && isNative) + relPath = path.join("static", resource.servePath); + else { + relPath = stripLeadingSlash(resource.servePath); + } + f.setTargetPathFromRelPath(relPath); + if (isBrowser) { f.setUrlFromRelPath(resource.servePath); } else if (isNative) { - var relPath; - if (resource.type === "static") - relPath = path.join(path.sep, "static", resource.servePath); - else - relPath = resource.servePath; - f.setTargetPathFromRelPath(relPath); if (resource.type === "js") f.setStaticDirectory(relPath, resource.staticDirectory); } @@ -634,6 +665,10 @@ _.extend(Target.prototype, { f.nodeModulesDirectory = nmd; } + if (resource.type === "js" && resource.sourceMap) { + f.setSourceMap(resource.sourceMap, path.dirname(relPath)); + } + self[resource.type].push(f); return; } @@ -743,8 +778,13 @@ _.extend(Target.prototype, { var f = new File({ sourcePath: absPath }); if (setUrl) f.setUrlFromRelPath(assetPath); + // XXX why is this separate from _emitResources ? + // XXX fix up server static resources + var relPath = assetDir.useSubDirectory + ? path.join('static', 'app', assetPath) + : assetPath; if (setTargetPath) - f.setTargetPathFromRelPath(path.join('/static', 'app', assetPath)); + f.setTargetPathFromRelPath(relPath); self.dependencyInfo.files[absPath] = f.hash(); self.static.push(f); }); @@ -791,22 +831,6 @@ _.extend(ClientTarget.prototype, { self.css[0].setUrlToHash(".css"); }, - assignTargetPaths: function () { - var self = this; - _.each(["js", "css", "static"], function (type) { - _.each(self[type], function (file) { - if (! file.targetPath) { - if (! file.url) - throw new Error("Client file with no URL?"); - - var parts = file.url.replace(/\?.*$/, '').split('/').slice(1); - parts.unshift(file.cacheable ? "static_cacheable" : "static"); - file.targetPath = path.sep + path.join.apply(path, parts); - } - }); - }); - }, - generateHtmlBoilerplate: function () { var self = this; @@ -829,26 +853,74 @@ _.extend(ClientTarget.prototype, { // the target write: function (builder) { var self = this; - var manifest = []; builder.reserve("program.json"); + builder.reserve("app.html"); - // Resources served via HTTP - _.each(["js", "css", "static"], function (type) { - _.each(self[type], function (file) { - - writeFile(file, builder); - - manifest.push({ - path: file.targetPath, - where: "client", - type: type, - cacheable: file.cacheable, - url: file.url, - size: file.size(), - hash: file.hash() + // Helper to iterate over all resources that we serve over HTTP. + var eachResource = function (f) { + _.each(["js", "css", "static"], function (type) { + _.each(self[type], function (file) { + f(file, type); }); }); + }; + + // Reserve all file names from the manifest, so that interleaved + // generateFilename calls don't overlap with them. + eachResource(function (file, type) { + builder.reserve(file.targetPath); + }); + + // Build up a manifest of all resources served via HTTP. + var manifest = []; + eachResource(function (file, type) { + var fileContents = file.contents(); + + var manifestItem = { + path: file.targetPath, + where: "client", + type: type, + cacheable: file.cacheable, + url: file.url + }; + + if (file.sourceMap) { + // Add anti-XSSI header to this file which will be served over + // HTTP. Note that the Mozilla and WebKit implementations differ as to + // what they strip: Mozilla looks for the four punctuation characters + // but doesn't care about the newline; WebKit only looks for the first + // three characters (not the single quote) and then strips everything up + // to a newline. + // https://groups.google.com/forum/#!topic/mozilla.dev.js-sourcemap/3QBr4FBng5g + var mapData = new Buffer(")]}'\n" + file.sourceMap, 'utf8'); + manifestItem.sourceMap = builder.writeToGeneratedFilename( + file.targetPath + '.map', {data: mapData}); + + // Use a SHA to make this cacheable. + var sourceMapBaseName = file.hash() + ".map"; + // XXX When we can, drop all of this and just use the SourceMap + // header. FF doesn't support that yet, though: + // https://bugzilla.mozilla.org/show_bug.cgi?id=765993 + // Note: if we use the older '//@' comment, FF 24 will print a lot + // of warnings to the console. So we use the newer '//#' comment... + // which Chrome (28) doesn't support. So we also set X-SourceMap + // in webapp_server. + file.setContents(Buffer.concat([ + file.contents(), + new Buffer("\n//# sourceMappingURL=" + sourceMapBaseName + "\n") + ])); + manifestItem.sourceMapUrl = require('url').resolve( + file.url, sourceMapBaseName); + } + + // Set this now, in case we mutated the file's contents. + manifestItem.size = file.size(); + manifestItem.hash = file.hash(); + + writeFile(file, builder); + + manifest.push(manifestItem); }); // HTML boilerplate (the HTML served to make the client load the @@ -864,17 +936,8 @@ _.extend(ClientTarget.prototype, { // Control file builder.writeJson('program.json', { format: "browser-program-pre1", - manifest: manifest, page: 'app.html', - - // XXX the following are for use by 'legacy' (read: current) - // server.js implementations which aren't smart enough to read - // the manifest and instead want all of the resources in a - // directory together so they can just point gzippo at it. we - // should remove this and make the server work from the - // manifest. - static: 'static', - staticCacheable: 'static_cacheable' + manifest: manifest }); return "program.json"; } @@ -899,6 +962,7 @@ var JsImage = function () { // - source: JS source code to load, as a string // - nodeModulesDirectory: a NodeModulesDirectory indicating which // directory should be searched by Npm.require() + // - sourceMap: if set, source map for this code, as a string // note: this can't be called `load` at it would shadow `load()` self.jsToLoad = []; @@ -1002,11 +1066,15 @@ _.extend(JsImage.prototype, { }, bindings || {}); try { - // XXX Get the actual source file path -- item.targetPath is - // not actually correct (it's the path in the bundle rather - // than in the source tree.) Moreover, we need to do source - // mapping. - files.runJavaScript(item.source, item.targetPath, env); + // XXX XXX Get the actual source file path -- item.targetPath + // is not actually correct (it's the path in the bundle rather + // than in the source tree.) + files.runJavaScript(item.source.toString('utf8'), { + filename: item.targetPath, + symbols: env, + sourceMap: item.sourceMap, + sourceMapRoot: item.sourceMapRoot + }); } catch (e) { buildmessage.exception(e); // Recover by skipping the rest of the load @@ -1025,7 +1093,7 @@ _.extend(JsImage.prototype, { write: function (builder) { var self = this; - builder.reserve("program.js"); + builder.reserve("program.json"); // Finalize choice of paths for node_modules directories -- These // paths are no longer just "preferred"; they are the final paths @@ -1045,14 +1113,28 @@ _.extend(JsImage.prototype, { if (! item.targetPath) throw new Error("No targetPath?"); - builder.write(item.targetPath, { data: new Buffer(item.source, 'utf8') }); - load.push({ - path: item.targetPath, + var loadPath = builder.writeToGeneratedFilename( + item.targetPath, + { data: new Buffer(item.source, 'utf8') }); + var loadItem = { + path: loadPath, node_modules: item.nodeModulesDirectory ? item.nodeModulesDirectory.preferredBundlePath : undefined, staticDirectory: item.staticDirectory ? item.staticDirectory.bundlePath : undefined - }); + }; + + if (item.sourceMap) { + // Write the source map. + // XXX this code is very similar to saveAsUnipackage. + loadItem.sourceMap = builder.writeToGeneratedFilename( + item.targetPath + '.map', + { data: new Buffer(item.sourceMap, 'utf8') } + ); + loadItem.sourceMapRoot = item.sourceMapRoot; + } + + load.push(loadItem); }); // node_modules resources from the packages. Due to appropriate @@ -1070,9 +1152,9 @@ _.extend(JsImage.prototype, { // Control file builder.writeJson('program.json', { - load: load, format: "javascript-image-pre1", - arch: self.arch + arch: self.arch, + load: load }); return "program.json"; } @@ -1093,11 +1175,11 @@ JsImage.readFromDisk = function (controlFilePath) { ret.arch = json.arch; _.each(json.load, function (item) { - if (item.path.match(/\.\./)) - throw new Error("bad path in plugin bundle"); + rejectBadPath(item.path); var nmd = undefined; if (item.node_modules) { + rejectBadPath(item.node_modules); var node_modules = path.join(dir, item.node_modules); if (! (node_modules in ret.nodeModulesDirectories)) { ret.nodeModulesDirectories[node_modules] = @@ -1109,14 +1191,22 @@ JsImage.readFromDisk = function (controlFilePath) { nmd = ret.nodeModulesDirectories[node_modules]; } - ret.jsToLoad.push({ + var loadItem = { targetPath: item.path, source: fs.readFileSync(path.join(dir, item.path)), nodeModulesDirectory: nmd, staticDirectory: new StaticDirectory({ sourcePath: item.staticDirectory }) - }); + }; + if (item.sourceMap) { + // XXX this is the same code as initFromUnipackage + rejectBadPath(item.sourceMap); + loadItem.sourceMap = fs.readFileSync( + path.join(dir, item.sourceMap), 'utf8'); + loadItem.sourceMapRoot = item.sourceMapRoot; + } + ret.jsToLoad.push(loadItem); }); return ret; @@ -1144,7 +1234,9 @@ _.extend(JsImageTarget.prototype, { targetPath: file.targetPath, source: file.contents().toString('utf8'), nodeModulesDirectory: file.nodeModulesDirectory, - staticDirectory: file.staticDirectory + staticDirectory: file.staticDirectory, + sourceMap: file.sourceMap, + sourceMapRoot: file.sourceMapRoot }); }); @@ -1380,7 +1472,7 @@ var writeSiteArchive = function (targets, outputPath, options) { // Affordances for standalone use if (targets.server) { // add program.json as the first argument after "node main.js" to the boot script. - var stub = new Buffer("process.argv.splice(2, 0, 'program.json');\nrequire('./programs/server/boot.js');\n", 'utf8'); + var stub = new Buffer("process.argv.splice(2, 0, 'program.json');\nprocess.chdir(require('path').join(__dirname, 'programs', 'server'));\nrequire('./programs/server/boot.js');\n", 'utf8'); builder.write('main.js', { data: stub }); builder.write('README', { data: new Buffer( @@ -1526,9 +1618,8 @@ exports.bundle = function (appDir, outputPath, options) { assetDirs = assetDirs || []; var clientAssetDirs = getValidAssetDirs(assetDirs, { exclude: ignoreFiles, - setUrl: true - // No need to set targetPath when the asset dir is added; - // the target path will be set later in assignTargetPaths. + setUrl: true, + setTargetPath: true }); client.make({ @@ -1560,9 +1651,11 @@ exports.bundle = function (appDir, outputPath, options) { assetDirs = assetDirs || []; var serverAssetDirs = getValidAssetDirs(assetDirs, { exclude: ignoreFiles, - setTargetPath: true // We need to set the target path when the asset dir is added, // because the target path comes from the asset's path. + setTargetPath: true, + // XXX this is a hack, re-assess how the subdirs are named + useSubDirectory: true }); var targetOptions = { library: library, diff --git a/tools/files.js b/tools/files.js index 93f5ef26e4..5fec9d691d 100644 --- a/tools/files.js +++ b/tools/files.js @@ -10,10 +10,31 @@ var os = require('os'); var util = require('util'); var _ = require('underscore'); var Future = require('fibers/future'); +var sourcemap = require('source-map'); +var sourcemap_support = require('source-map-support'); var cleanup = require('./cleanup.js'); var buildmessage = require('./buildmessage.js'); +var parsedSourceMaps = {}; +var nextStackFilenameCounter = 1; +var retrieveSourceMap = function (pathForSourceMap) { + if (_.has(parsedSourceMaps, pathForSourceMap)) + return {map: parsedSourceMaps[pathForSourceMap]}; + return null; +}; + +sourcemap_support.install({ + // Use the source maps specified to runJavaScript instead of parsing source + // code for them. + retrieveSourceMap: retrieveSourceMap, + // For now, don't fix the source line in uncaught exceptions, because we + // haven't fixed handleUncaughtExceptions in source-map-support to properly + // locate the source files. + handleUncaughtExceptions: false +}); + + var files = exports; _.extend(exports, { // A sort comparator to order files into load order. @@ -595,36 +616,76 @@ _.extend(exports, { return future.wait(); }, - // Return the result of evaluating `code` using - // `runInThisContext`. `code` will be wrapped in a closure. You can - // pass additional values to bind in the closure in `env`, the keys - // being the symbols to bind and the values being their - // values. `filename` is the filename to use in exceptions that come - // from inside this code. + // Return the result of evaluating `code` using `runInThisContext`. `code` + // will be wrapped in a closure. You can pass additional values to bind in the + // closure in `options.symbols`, the keys being the symbols to bind and the + // values being their values. `options.filename` is the filename to use in + // exceptions that come from inside this code. `options.sourceMap` is an + // optional source map that represents the file. // - // The really special thing about this function is that if a parse - // error occurs, we will raise an exception of type - // files.FancySyntaxError, from which you may read 'message', 'file', - // 'line', 'column', and 'columnEnd' attributes ... v8 is - // normally reluctant to reveal this information but will write it - // to stderr if you pass it an undocumented flag. Unforunately - // though node doesn't have dup2 so we can't intercept the write. So - // instead -- only if the parse does in fact fail, to determine the - // error we start a subprocess, redirect its stderr, grab the output - // and parse it. - runJavaScript: function (code, filename, env) { + // The really special thing about this function is that if a parse error + // occurs, we will raise an exception of type files.FancySyntaxError, from + // which you may read 'message', 'file', 'line', and 'column' attributes + // ... v8 is normally reluctant to reveal this information but will write it + // to stderr if you pass it an undocumented flag. Unforunately though node + // doesn't have dup2 so we can't intercept the write. So instead we use a + // completely different parser with a better error handling API. Ah well. + runJavaScript: function (code, options) { + if (typeof code !== 'string') + throw new Error("code must be a string"); + + options = options || {}; + var filename = options.filename || "<anonymous>"; var keys = [], values = []; // don't assume that _.keys and _.values are guaranteed to // enumerate in the same order - for (var k in env) { - keys.push(k); - values.push(env[k]); + _.each(options.symbols, function (value, name) { + keys.push(name); + values.push(value); + }); + + var stackFilename = filename; + if (options.sourceMap) { + // We want to generate an arbitrary filename that we use to associate the + // file with its source map. + stackFilename = "<runJavaScript-" + nextStackFilenameCounter++ + ">"; } + var chunks = []; var header = "(function(" + keys.join(',') + "){"; + chunks.push(header); + if (options.sourceMap) { + var consumer = new sourcemap.SourceMapConsumer(options.sourceMap); + chunks.push(sourcemap.SourceNode.fromStringWithSourceMap( + code, consumer)); + } else { + chunks.push(code); + } // \n is necessary in case final line is a //-comment - var footer = "\n})"; - var wrapped = header + code + footer; + chunks.push("\n})"); + + var wrapped; + var parsedSourceMap = null; + if (options.sourceMap) { + var node = new sourcemap.SourceNode(null, null, null, chunks); + var results = node.toStringWithSourceMap({ + file: stackFilename + }); + wrapped = results.code; + parsedSourceMap = results.map.toJSON(); + if (options.sourceMapRoot) { + // Add the specified root to any root that may be in the file. + parsedSourceMap.sourceRoot = path.join( + options.sourceMapRoot, parsedSourceMap.sourceRoot || ''); + } + // source-map-support doesn't ever look at the sourcesContent field, so + // there's no point in keeping it in memory. + delete parsedSourceMap.sourcesContent; + parsedSourceMaps[stackFilename] = parsedSourceMap; + } else { + + wrapped = chunks.join(''); + }; try { // See #runInThisContext @@ -636,84 +697,75 @@ _.extend(exports, { // // Pass 'true' as third argument if we want the parse error on // stderr (which we don't.) - var func = require('vm').runInThisContext(wrapped, filename); - } catch (e) { - // Got, presumably, a parse error. OK, we're going to start - // another copy of node and feed it the offending code on - // stdin. It should give us the error on stderr. + var script = require('vm').createScript(wrapped, stackFilename); + } catch (nodeParseError) { + if (!(nodeParseError instanceof SyntaxError)) + throw nodeParseError; + // Got a parse error. Unfortunately, we can't actually get the location of + // the parse error from the SyntaxError; Node has some hacky support for + // displaying it over stderr if you pass an undocumented third argument to + // stackFilename, but that's not what we want. See + // https://github.com/joyent/node/issues/3452 + // for more information. One thing to try (and in fact, what an early + // version of this function did) is to actually fork a new node + // to run the code and parse its output. We instead run an entirely + // different JS parser, from the esprima project, but which at least + // has a nice API for reporting errors. + var esprima = require('esprima'); + try { + esprima.parse(wrapped); + } catch (esprimaParseError) { + // Is this actually an Esprima syntax error? + if (!('index' in esprimaParseError && + 'lineNumber' in esprimaParseError && + 'column' in esprimaParseError && + 'description' in esprimaParseError)) { + throw esprimaParseError; + } + var err = new files.FancySyntaxError; - var Future = require('fibers/future'); - var future = new Future; + err.message = esprimaParseError.description; - var child_process = require("child_process"); - var proc = child_process.execFile( - process.argv[0], [], { - stdio: ['pipe'] - }, function (error, stdout, stderr) { - if (! error || error.code === 0) - future.return(null); // huh? didn't fail? - else - future.return(stderr); - }); - proc.stdin.write(wrapped); - proc.stdin.end(); - var stderr = future.wait(); + if (parsedSourceMap) { + // XXX this duplicates code in computeGlobalReferences + var consumer2 = new sourcemap.SourceMapConsumer(parsedSourceMap); + var original = consumer2.originalPositionFor({ + line: esprimaParseError.lineNumber, + column: esprimaParseError.column - 1 + }); + if (original.source) { + err.file = original.source; + err.line = original.line; + err.column = original.column + 1; + throw err; + } + } - if (stderr === null) - throw new Error("subprocess parsed bad code successfully?"); - -/* stderr will look something like this (note leading blank line:) -" -[stdin]:1 -couaoeua aaaaaa nsolexloeuaoeuao - ^^^^^^ -SyntaxError: Unexpected identifier - at Object.<anonymous> ([stdin]-wrapper:6:22) - at Module._compile (module.js:449:26) - at evalScript (node.js:282:25) - at Socket.<anonymous> (node.js:152:11) - at Socket.EventEmitter.emit (events.js:93:17) - at Pipe.onread (net.js:418:51)" -*/ - var err = new files.FancySyntaxError; - err.file = filename; - var lines = stderr.split('\n'); - - // line number - var m = lines[1].match(/:(\d+)\s*$/); - if (! m) - throw new Error("can't parse line number from '" + lines[1] + "'"); - err.line = +m[1]; - - // column range - m = lines[3].match(/^(\s*)(\^+)\s*$/) - if (! m) - throw new Error("can't parse column indicator from '" + lines[3] + "'"); - err.column = m[1].length + 1; - err.columnEnd = err.column + m[2].length - 1; - - // message - m = lines[4].match(/^SyntaxError:\s*(.*)$/); - if (! m) - throw new Error("can't parse error message from '" + lines[4] + "'"); - err.message = m[1]; - - // adjust errors on line 1 to account for our header - if (err.line === 1) { - err.column -= header.length; - err.columnEnd -= header.length; + err.file = filename; // *not* stackFilename + err.line = esprimaParseError.lineNumber; + err.column = esprimaParseError.column; + // adjust errors on line 1 to account for our header + if (err.line === 1) { + err.column -= header.length; + } + throw err; } - throw err; + // What? Node thought that this was a parse error and esprima didn't? Eh, + // just throw Node's error and don't care too much about the line numbers + // being right. + throw nodeParseError; } + var func = script.runInThisContext(); + return (buildmessage.markBoundary(func)).apply(null, values); }, // - message: an error message from the parser + // - file: filename // - line: 1-based // - column: 1-based - // - columnEnd: 1-based FancySyntaxError: function () {}, OfflineError: function (error) { diff --git a/tools/library.js b/tools/library.js index c6083d791d..040cbf645e 100644 --- a/tools/library.js +++ b/tools/library.js @@ -7,7 +7,7 @@ var bundler = require('./bundler.js'); var buildmessage = require('./buildmessage.js'); var fs = require('fs'); -// Under the hood, packages in the library (/package/foo), and user +// Under the hood, packages in the library (/packages/foo), and user // applications, are both Packages -- they are just represented // differently on disk. @@ -374,4 +374,4 @@ _.extend(exports, { return out; } -}); \ No newline at end of file +}); diff --git a/tools/linker.js b/tools/linker.js index 5cddc8b6e6..b6eaf7eab7 100644 --- a/tools/linker.js +++ b/tools/linker.js @@ -1,5 +1,6 @@ var fs = require('fs'); var _ = require('underscore'); +var sourcemap = require('source-map'); var buildmessage = require('./buildmessage'); var packageDot = function (name) { @@ -9,21 +10,6 @@ var packageDot = function (name) { return "Package['" + name + "']"; }; -var generateBoundary = function () { - // In a perfect world we would call Packages.random.Random.id(). - // But we can't do that this is part of the code that is used to - // compile and load packages. So let it slide for now and provide a - // version based on (the completely non-cryptographic) Math.random, - // which is good enough for this particular application. - var alphabet = "23456789ABCDEFGHJKLMNPQRSTWXYZabcdefghijkmnopqrstuvwxyz"; - var digits = []; - for (var i = 0; i < 17; i++) { - var index = Math.floor(Math.random() * alphabet.length); - digits[i] = alphabet.substr(index, 1); - } - return "__imports_" + digits.join("") + "__"; -}; - /////////////////////////////////////////////////////////////////////////////// // Module /////////////////////////////////////////////////////////////////////////////// @@ -40,9 +26,6 @@ var Module = function (options) { // files in the module. array of File self.files = []; - // boundary to use to mark where import should go in final phase - self.boundary = generateBoundary(); - // options self.forceExport = options.forceExport || []; self.useGlobalNamespace = options.useGlobalNamespace; @@ -77,6 +60,13 @@ _.extend(Module.prototype, { return _.max(maxInFile); }, + runLinkerFileTransforms: function (exports) { + var self = this; + _.each(self.files, function (f) { + f.runLinkerFileTransform(exports); + }); + }, + // Figure out which vars need to be specifically put in the module // scope. // @@ -87,15 +77,15 @@ _.extend(Module.prototype, { // and see what your globals are. Probably this means we need to // move the emission of the Package-scope Variables section (but not // the actual static analysis) to the final phase. - computeModuleScopedVars: function () { + computeModuleScopeVars: function (exports) { var self = this; if (!self.jsAnalyze) { // We don't have access to static analysis, probably because we *are* the // js-analyze package. Let's do a stupid heuristic: any exports that have - // no dots are module scoped vars. (This works for + // no dots are module scope vars. (This works for // js-analyze.JSAnalyze...) - return _.filter(self.getExports(), function (e) { + return _.filter(exports, function (e) { return e.indexOf('.') === -1; }); } @@ -107,68 +97,68 @@ _.extend(Module.prototype, { }); globalReferences = _.uniq(globalReferences); - return globalReferences; + return _.isEmpty(globalReferences) ? undefined : globalReferences; }, - // Output is a list of objects with keys 'source' and 'servePath'. - getLinkedFiles: function () { + // Output is a list of objects with keys 'source', 'servePath', 'sourceMap', + // 'sourcePath' + getPrelinkedFiles: function (moduleExports) { var self = this; - if (! self.files.length && ! self.useGlobalNamespace) + if (! self.files.length) return []; - var moduleExports = self.getExports(); - // If we don't want to create a separate scope for this module, // then our job is much simpler. And we can get away with // preserving the line numbers. if (self.useGlobalNamespace) { - var ret = [{ - source: self.boundary, - servePath: self.importStubServePath - }]; + return _.map(self.files, function (file) { + var node = file.getPrelinkedOutput({ preserveLineNumbers: true, + exports: moduleExports }); + var results = node.toStringWithSourceMap({ + file: file.servePath + }); // results has 'code' and 'map' attributes + + var sourceMap = results.map.toJSON(); + // No use generating empty source maps. + if (_.isEmpty(sourceMap.sources)) + sourceMap = null; + else + sourceMap = JSON.stringify(sourceMap); - return ret.concat(_.map(self.files, function (file) { return { - source: file.getLinkedOutput({ preserveLineNumbers: true, - exports: moduleExports }), - servePath: file.servePath + source: results.code, + servePath: file.servePath, + sourceMap: sourceMap }; - })); + }); } // Otherwise.. - // Find the maximum line length. The extra three are for the - // comments that will be emitted when we skip a unit. - var sourceWidth = _.max([68, self.maxLineLength(120 - 2)]) + 3; - - // Figure out which variables are module scope - var moduleScopedVars = self.computeModuleScopedVars(); + // Find the maximum line length. + var sourceWidth = _.max([68, self.maxLineLength(120 - 2)]); // Prologue - var combined = "(function () {\n\n"; - combined += self.boundary; - - if (moduleScopedVars.length) { - combined += "/* Package-scope variables */\n"; - combined += "var " + moduleScopedVars.join(', ') + ";\n\n"; - } + var chunks = []; // Emit each file _.each(self.files, function (file) { - combined += file.getLinkedOutput({ sourceWidth: sourceWidth, - exports: moduleExports }); - combined += "\n"; + if (!_.isEmpty(chunks)) + chunks.push("\n\n\n\n\n\n"); + chunks.push(file.getPrelinkedOutput({ sourceWidth: sourceWidth, + exports: moduleExports })); }); - // Epilogue - combined += self.getExportCode(); - combined += "\n})();"; + var node = new sourcemap.SourceNode(null, null, null, chunks); + var results = node.toStringWithSourceMap({ + file: self.combinedServePath + }); // results has 'code' and 'map' attributes return [{ - source: combined, - servePath: self.combinedServePath + source: results.code, + servePath: self.combinedServePath, + sourceMap: results.map.toString() }]; }, @@ -181,51 +171,11 @@ _.extend(Module.prototype, { var exports = {}; _.each(self.files, function (file) { - _.each(file.units, function (unit) { - _.extend(exports, unit.exports); - }); + _.extend(exports, file.exports); }); return _.union(_.keys(exports), self.forceExport); - }, - - // Return code that saves our exports to Package.packagename.foo.bar - getExportCode: function () { - var self = this; - if (! self.name) - return ""; - // If we're a no-exports module, then we have no export code (not even - // creating Package.foo). - if (self.noExports) - return ""; - if (self.useGlobalNamespace) - // Haven't thought about this case. When would this happen? - throw new Error("Not implemented: exports from global namespace"); - - var buf = "/* Exports */\n"; - buf += "if (typeof Package === 'undefined') Package = {};\n"; - buf += packageDot(self.name) + " = "; - - var exports = self.getExports(); - // Even if there are no exports, we need to define Package.foo, because the - // existence of Package.foo is how another package (eg, one that weakly - // depends on foo) can tell if foo is loaded. - if (exports.length === 0) - return buf + "{};\n"; - - // Given exports like Foo, Bar.Baz, Bar.Quux.A, and Bar.Quux.B, - // construct an expression like - // {Foo: Foo, Bar: {Baz: Bar.Baz, Quux: {A: Bar.Quux.A, B: Bar.Quux.B}}} - var scratch = {}; - _.each(self.getExports(), function (symbol) { - scratch[symbol] = symbol; - }); - var exportTree = buildSymbolTree(scratch); - buf += writeSymbolTree(exportTree, 0); - buf += ";\n"; - return buf; } - }); // Given 'symbolMap' like {Foo: 's1', 'Bar.Baz': 's2', 'Bar.Quux.A': 's3', 'Bar.Quux.B': 's4'} @@ -285,44 +235,103 @@ var File = function (inputFile, module) { // the path where this file would prefer to be served if possible self.servePath = inputFile.servePath; - // the path to use for error message + // The relative path of this input file in its source tree (eg, + // package or app.) Used for source maps, error messages.. self.sourcePath = inputFile.sourcePath; - // should line and column be included in errors? - self.includePositionInErrors = inputFile.includePositionInErrors; - - // The individual @units in the file. Array of Unit. Concatenating - // the source of each unit, in order, will give self.source. - self.units = []; - // A function which transforms the source code once all exports are // known. (eg, for CoffeeScript.) - self.linkerUnitTransform = - inputFile.linkerUnitTransform || function (source, exports) { - return source; + self.linkerFileTransform = + inputFile.linkerFileTransform || function (sourceWithMap, exports) { + return sourceWithMap; }; // If true, don't wrap this individual file in a closure. self.bare = !!inputFile.bare; + // A source map (generated by something like CoffeeScript) for the input file. + self.sourceMap = inputFile.sourceMap; + // The Module containing this file. self.module = module; - self._unitize(); + // symbols mentioned in @export, @require, @provide, or @weak + // directives. each is a map from the symbol (given as a string) to + // true. (only @export is actually implemented) + self.exports = {}; + self.requires = {}; + self.provides = {}; + self.weaks = {}; + + self._scanForComments(); }; _.extend(File.prototype, { - // Return the union of the global references in all of the units in - // this file that we are actually planning to use. Array of string. + // Return the globals in this file as an array of symbol names. For + // example: if the code references 'Foo.bar.baz' and 'Quux', and + // neither are declared in a scope enclosing the point where they're + // referenced, then globalReferences would include ["Foo", "Quux"]. computeGlobalReferences: function () { var self = this; - var globalReferences = []; - _.each(self.units, function (unit) { - if (unit.include) - globalReferences = globalReferences.concat(unit.computeGlobalReferences()); - }); - return globalReferences; + var jsAnalyze = self.module.jsAnalyze; + // If we don't have a JSAnalyze object, we probably are the js-analyze + // package itself. Assume we have no global references. At the module level, + // we'll assume that exports are global references. + if (!jsAnalyze) + return []; + + try { + return _.keys(jsAnalyze.findAssignedGlobals(self.source)); + } catch (e) { + if (!e.$ParseError) + throw e; + + var errorOptions = { + file: self.sourcePath, + line: e.lineNumber, + column: e.column + }; + if (self.sourceMap) { + var parsed = new sourcemap.SourceMapConsumer(self.sourceMap); + var original = parsed.originalPositionFor( + {line: e.lineNumber, column: e.column - 1}); + if (original.source) { + errorOptions.file = original.source; + errorOptions.line = original.line; + errorOptions.column = original.column + 1; + } + } + + buildmessage.error(e.description, errorOptions); + + // Recover by pretending that this file is empty (which + // includes replacing its source code with '' in the output) + self.source = ""; + self.sourceMap = null; + return []; + } + }, + + + // Relative path to use in source maps to indicate this file. No + // leading slash. + _pathForSourceMap: function () { + var self = this; + + if (self.module.name) + return self.module.name + "/" + self.sourcePath; + else + return require('path').basename(self.sourcePath); + }, + + runLinkerFileTransform: function (exports) { + var self = this; + var sourceAndMap = self.linkerFileTransform( + {source: self.source, sourceMap: self.sourceMap}, + exports); + self.source = sourceAndMap.source; + self.sourceMap = sourceAndMap.sourceMap; }, // Options: @@ -331,73 +340,121 @@ _.extend(File.prototype, { // sourceWidth is ignored. // - sourceWidth: width in columns to use for the source code // - exports: the module's exports - getLinkedOutput: function (options) { + // + // Returns a SourceNode. + getPrelinkedOutput: function (options) { var self = this; - // XXX XXX if a unit is not going to be used, prepend each line with '//' - // The newline after the source closes a '//' comment. if (options.preserveLineNumbers) { // Ugly version - var body = self.linkerUnitTransform(self.source, options.exports); - if (body.length && body[body.length - 1] !== '\n') - body += '\n'; - return self.bare ? body : ("(function(){" + body + "})();\n"); + var mapNode; + if (self.sourceMap) { + mapNode = sourcemap.SourceNode.fromStringWithSourceMap( + self.source, new sourcemap.SourceMapConsumer(self.sourceMap)); + } else { + // This is an app file that was always JS. The output file here is going + // to be the same name as the input file (because _pathForSourceMap in + // apps is the basename of the source file), and having a JS file + // pointing to a source map pointing to a JS file of the same name will + // (a) be confusing (b) be unnecessary since we aren't renumbering + // anything and (c) confuse at least Chrome. + mapNode = self.source; + } + + return new sourcemap.SourceNode(null, null, null, [ + self.bare ? "" : "(function(){", + mapNode, + (self.source.length && self.source[self.source.length - 1] !== '\n' + ? "\n" : ""), + self.bare ? "" : "\n})();\n" + ]); } // Pretty version - var buf = ""; + var chunks = []; // Prologue - if (!self.bare) - buf += "(function () {\n\n"; + if (! self.bare) + chunks.push("(function () {\n\n"); // Banner + var bannerLines = [self.servePath.slice(1)]; + if (self.bare) { + bannerLines.push( + "This file is in bare mode and is not in its own closure."); + } var width = options.sourceWidth || 70; var bannerWidth = width + 3; - var divider = new Array(bannerWidth + 1).join('/') + "\n"; - var spacer = "// " + new Array(bannerWidth - 6 + 1).join(' ') + " //\n"; - var padding = new Array(bannerWidth + 1).join(' '); + var padding = bannerPadding(bannerWidth); + chunks.push(banner(bannerLines, bannerWidth)); var blankLine = new Array(width + 1).join(' ') + " //\n"; - buf += divider + spacer; - buf += "// " + (self.servePath.slice(1) + padding).slice(0, bannerWidth - 6) + - " //\n"; - if (self.bare) { - var bareText = "This file is in bare mode and is not in its own closure."; - buf += "// " + (bareText + padding).slice(0, bannerWidth - 6) + " //\n"; - } - buf += spacer + divider + blankLine; + chunks.push(blankLine); // Code, with line numbers // You might prefer your line numbers at the beginning of the // line, with /* .. */. Well, that requires parsing the source for // comments, because you have to do something different if you're // already inside a comment. - var num = 1; - _.each(self.units, function (unit) { - var unitSource = self.linkerUnitTransform(unit.source, options.exports); - var lines = unitSource.split('\n'); + var numberifyLines = function (f) { + var num = 1; + var lines = self.source.split('\n'); _.each(lines, function (line) { - if (! unit.include) - line = "// " + line; - if (line.length > width) - buf += line + "\n"; - else - buf += (line + padding).slice(0, width) + " // " + num + "\n"; + var suffix = "\n"; + + if (line.length <= width) { + suffix = padding.slice(line.length, width) + " // " + num + "\n"; + } + f(line, suffix, num); num++; }); - }); + }; + + var lines = self.source.split('\n'); + + if (self.sourceMap) { + var buf = ""; + numberifyLines(function (line, suffix) { + buf += line; + buf += suffix; + }); + // The existing source map is valid because all we're doing is adding + // things to the end of lines, which doesn't affect the source map. (If + // we wanted to be picky, we could add some explicitly non-mapped regions + // to the source map to cover the suffixes, which would make this + // equivalent to the "no source map coming in" case, but this doesn't seem + // that important.) + chunks.push(sourcemap.SourceNode.fromStringWithSourceMap( + self.source, + new sourcemap.SourceMapConsumer(self.sourceMap))); + } else { + // There are probably ways to make a more compact source map. For example, + // the only change we make is to append a comment, so we can probably emit + // one mapping for the whole file. For the moment, we'll do it by the book + // just to see how it goes. + numberifyLines(function (line, suffix, num) { + chunks.push(new sourcemap.SourceNode(num, 0, self._pathForSourceMap(), + line)); + chunks.push(suffix); + }); + } // Footer - buf += divider; + if (! self.bare) + chunks.push(dividerLine(bannerWidth) + "\n}).call(this);\n"); - // Epilogue - if (!self.bare) - buf += "\n}).call(this);\n"; - buf += "\n\n\n\n\n"; + var node = new sourcemap.SourceNode(null, null, null, chunks); - return buf; + // If we're working directly from the original source here (and not from the + // output of a transformation that had a source map), include the original + // source in the source map. (If we are working on generated code, the + // source map we received should have already contained the original + // source.) + if (!self.sourceMap) + node.setSourceContent(self._pathForSourceMap(), self.source); + + return node; }, // If "line" contains nothing but a comment (of either syntax), return the @@ -418,159 +475,73 @@ _.extend(File.prototype, { return null; }, - // Split file and populate self.units - // XXX it is an error to declare a @unit not at toplevel (eg, inside a - // function or object..) We don't detect this but we might have to to - // give an acceptable user experience.. - _unitize: function () { + // Scan for @export, etc. + _scanForComments: function () { var self = this; var lines = self.source.split("\n"); - var buf = ""; - var unit = new Unit( - null, true, self, self.includePositionInErrors ? 0 : null); - self.units.push(unit); - var lineCount = 0; _.each(lines, function (line) { var commentBody = self._getSingleLineCommentBody(line); + if (!commentBody) + return; - if (commentBody) { - // XXX overly permissive. should detect errors - var match = /^@unit(?:\s+(\S+))?$/.exec(commentBody); - if (match) { - unit.source = buf; - buf = line; - unit = new Unit(match[1] || null, false, self, - self.includePositionInErrors ? lineCount : null); - self.units.push(unit); - lineCount++; - return; - } + // XXX overly permissive. should detect errors + var match = /^@(export|require|provide|weak)(\s+.*)$/.exec(commentBody); + if (match) { + var what = match[1]; + var symbols = _.map(match[2].split(/,/), function (s) { + return s.trim(); + }); - // XXX overly permissive. should detect errors - match = /^@(export|require|provide|weak)(\s+.*)$/.exec(commentBody); - if (match) { - var what = match[1]; - var symbols = _.map(match[2].split(/,/), function (s) { - return s.trim(); + var badSymbols = _.reject(symbols, function (s) { + // XXX should be unicode-friendlier + return s.match(/^([_$a-zA-Z][_$a-zA-Z0-9]*)(\.[_$a-zA-Z][_$a-zA-Z0-9]*)*$/); + }); + if (!_.isEmpty(badSymbols)) { + buildmessage.error("bad symbols for @" + what + ": " + + JSON.stringify(badSymbols), + { file: self.sourcePath }); + // recover by ignoring + } else if (self.module.noExports && what === "export") { + buildmessage.error("@export not allowed in this slice", + { file: self.sourcePath }); + // recover by ignoring + } else { + _.each(symbols, function (s) { + if (s.length) + self[what + "s"][s] = true; }); - - var badSymbols = _.reject(symbols, function (s) { - // XXX should be unicode-friendlier - return s.match(/^([_$a-zA-Z][_$a-zA-Z0-9]*)(\.[_$a-zA-Z][_$a-zA-Z0-9]*)*$/); - }); - if (!_.isEmpty(badSymbols)) { - buildmessage.error("bad symbols for @" + what + ": " + - JSON.stringify(badSymbols), - { file: self.sourcePath }); - // recover by ignoring - } else if (self.module.noExports && what === "export") { - buildmessage.error("@export not allowed in this slice", - { file: self.sourcePath }); - // recover by ignoring - } else { - _.each(symbols, function (s) { - if (s.length) - unit[what + "s"][s] = true; - }); - } - - /* fall through */ } } - - if (lineCount !== 0) - buf += "\n"; - lineCount++; - buf += line; }); - unit.source = buf; } }); -/////////////////////////////////////////////////////////////////////////////// -// Unit -/////////////////////////////////////////////////////////////////////////////// +// Given a list of lines (not newline-terminated), returns a string placing them +// in a pretty banner of width bannerWidth. All lines must have length at most +// (bannerWidth - 6); if bannerWidth is not provided, the smallest width that +// fits is used. +var banner = function (lines, bannerWidth) { + if (!bannerWidth) + bannerWidth = 6 + _.max(lines, function (x) { return x.length; }).length; -var Unit = function (name, mandatory, file, lineOffset) { - var self = this; + var divider = dividerLine(bannerWidth); + var spacer = "// " + new Array(bannerWidth - 6 + 1).join(' ') + " //\n"; + var padding = bannerPadding(bannerWidth); - // name of the unit, or null if none provided - self.name = name; - - // source code for this unit (a string) - self.source = null; - - // true if this unit is to always be included - self.mandatory = !! mandatory; - - // true if we should include this unit in the linked output - self.include = self.mandatory; - - // The File containing the unit. - self.file = file; - - // offset of 'self.source' in the original input file, in whole - // lines (partial lines are not supported.) Used to generate correct - // line number information in error messages. Set to null to omit - // line/column information (you'll need to do this, for, eg, - // coffeescript output, given that we don't have sourcemaps here - // yet.) - self.lineOffset = lineOffset; - - // symbols mentioned in @export, @require, @provide, or @weak - // directives. each is a map from the symbol (given as a string) to - // true. - self.exports = {}; - self.requires = {}; - self.provides = {}; - self.weaks = {}; + var buf = divider + spacer; + _.each(lines, function (line) { + buf += "// " + (line + padding).slice(0, bannerWidth - 6) + " //\n"; + }); + buf += spacer + divider; + return buf; +}; +var dividerLine = function (bannerWidth) { + return new Array(bannerWidth + 1).join('/') + "\n"; +}; +var bannerPadding = function (bannerWidth) { + return new Array(bannerWidth + 1).join(' '); }; - -_.extend(Unit.prototype, { - // Return the globals in unit file as an array of symbol names. For - // example: if the code references 'Foo.bar.baz' and 'Quux', and - // neither are declared in a scope enclosing the point where they're - // referenced, then globalReferences would include ["Foo", "Quux"]. - // - // XXX Doing this at the unit level means that we need to also look - // for var declarations in various units, and use them to create - // a graph of unit dependencies such that in: - // // @unit X - // var A; - // // @unit Y - // A = 5; - // including Y requires including X. Since we don't do that, @unit - // is currently broken. It's also unused and undocumented :) - computeGlobalReferences: function () { - var self = this; - - var jsAnalyze = self.file.module.jsAnalyze; - // If we don't have a JSAnalyze object, we probably are the js-analyze - // package itself. Assume we have no global references. At the module level, - // we'll assume that exports are global references. - if (!jsAnalyze) - return []; - - try { - return _.keys(jsAnalyze.findAssignedGlobals(self.source)); - } catch (e) { - if (!e.$ParseError) - throw e; - buildmessage.error(e.description, { - file: self.file.sourcePath, - line: self.lineOffset === null ? null : e.lineNumber + self.lineOffset, - column: self.lineOffset === null ? null : e.column, - downcase: true - }); - - // Recover by pretending that this unit is empty (which - // includes replacing its source code with '' in the output) - self.source = ""; - return []; - } - } -}); /////////////////////////////////////////////////////////////////////////////// // Top-level entry point @@ -596,12 +567,10 @@ _.extend(Unit.prototype, { // bundle, not in error messages, but it's still nice to make it // look good) // - sourcePath: path to use in error messages -// - includePositionInErrors: true to include line and column -// information in errors. set to false if, eg, this is the output -// of coffeescript. (XXX replace with real sourcemaps) -// - linkerUnitTransform: if given, this function will be called -// when the module is being linked with the source of the unit -// and an array of the exports of the module; the unit's source will +// - sourceMap: an optional source map (as string) for the input file +// - linkerFileTransform: if given, this function will be called +// when the module is being linked with the source of the file +// and an array of the exports of the module; the file's source will // be replaced by what the function returns. // // forceExport: an array of symbols (as dotted strings) to force the @@ -628,8 +597,9 @@ _.extend(Unit.prototype, { // // Output is an object with keys: // - files: is an array of output files in the same format as inputFiles +// - EXCEPT THAT, for now, sourcePath is omitted and is replaced with +// sourceMap (a string) (XXX) // - exports: the exports, as a list of string ('Foo', 'Thing.Stuff', etc) -// - boundary: an opaque value that must be passed along with 'files' to link() var prelink = function (options) { if (options.noExports && options.forceExport && ! _.isEmpty(options.forceExport)) { @@ -650,16 +620,39 @@ var prelink = function (options) { module.addFile(inputFile); }); - var files = module.getLinkedFiles(); + // 1) Figure out what this entire module exports. + // 2) Run the linkerFileTransforms, which depend on the exports. (This is, eg, + // CoffeeScript arranging to not close over the exports.) + // 3) Do static analysis to compute module-scoped variables; this has to be + // done based on the *output* of the transforms. Error recovery from the + // static analysis mutates the sources, so this has to be done before + // concatenation. + // 4) Finally, concatenate. var exports = module.getExports(); + module.runLinkerFileTransforms(exports); + var packageScopeVariables = module.computeModuleScopeVars(exports); + var files = module.getPrelinkedFiles(exports); return { files: files, exports: exports, - boundary: module.boundary + packageScopeVariables: packageScopeVariables }; }; +var SOURCE_MAP_INSTRUCTIONS_COMMENT = banner([ + "This is a generated file. You can view the original", + "source in your browser if your browser supports source maps.", + "", + "If you are using Chrome, open the Developer Tools and click the gear", + "icon in its lower right corner. In the General Settings panel, turn", + "on 'Enable source maps'.", + "", + "If you are using Firefox 23, go to `about:config` and set the", + "`devtools.debugger.source-maps-enabled` preference to true.", + "(The preference should be on by default in Firefox 24; versions", + "older than 23 do not support source maps.)" +]); // Finish the linking. // @@ -673,35 +666,71 @@ var prelink = function (options) { // // prelinkFiles: the 'files' output from prelink() // -// boundary: the 'boundary' output from prelink() -// // Output is an array of final output files in the same format as the // 'inputFiles' argument to prelink(). var link = function (options) { - var importCode = options.useGlobalNamespace ? - getImportCode(options.imports, "/* Imports for global scope */\n\n", true) : - getImportCode(options.imports, "/* Imports */\n"); + if (options.useGlobalNamespace) { + var ret = []; + if (!_.isEmpty(options.imports)) { + ret.push({ + source: getImportCode(options.imports, + "/* Imports for global scope */\n\n", true), + servePath: options.importStubServePath + }); + } + return ret.concat(options.prelinkFiles); + } + + var header = getHeader({ + imports: options.imports, + packageScopeVariables: options.packageScopeVariables + }); + var footer = getFooter({ + exports: options.exports, + name: options.name + }); var ret = []; _.each(options.prelinkFiles, function (file) { - var source = file.source; - var parts = source.split(options.boundary); - if (parts.length > 2) - throw new Error("Boundary appears more than once?"); - if (parts.length === 2) { - source = parts[0] + importCode + parts[1]; - if (source.length === 0) - return; // empty global-imports file -- elide + if (file.sourceMap) { + var chunks = [header]; + if (options.includeSourceMapInstructions) + chunks.push("\n" + SOURCE_MAP_INSTRUCTIONS_COMMENT + "\n\n"); + chunks.push(sourcemap.SourceNode.fromStringWithSourceMap( + file.source, new sourcemap.SourceMapConsumer(file.sourceMap))); + chunks.push(footer); + var node = new sourcemap.SourceNode(null, null, null, chunks); + var results = node.toStringWithSourceMap({ + file: file.servePath + }); + ret.push({ + source: results.code, + servePath: file.servePath, + sourceMap: results.map.toString() + }); + } else { + ret.push({ + source: header + file.source + footer, + servePath: file.servePath + }); } - ret.push({ - source: source, - servePath: file.servePath - }); }); return ret; }; +var getHeader = function (options) { + var chunks = []; + chunks.push("(function () {\n\n" ); + chunks.push(getImportCode(options.imports, "/* Imports */\n")); + if (options.packageScopeVariables + && !_.isEmpty(options.packageScopeVariables)) { + chunks.push("/* Package-scope variables */\n"); + chunks.push("var " + options.packageScopeVariables.join(', ') + ";\n\n"); + } + return chunks.join(''); +}; + var getImportCode = function (imports, header, omitvar) { var self = this; @@ -712,10 +741,10 @@ var getImportCode = function (imports, header, omitvar) { _.each(imports, function (name, symbol) { scratch[symbol] = packageDot(name) + "." + symbol; }); - var imports = buildSymbolTree(scratch); + var tree = buildSymbolTree(scratch); var buf = header; - _.each(imports, function (node, key) { + _.each(tree, function (node, key) { buf += (omitvar ? "" : "var " ) + key + " = " + writeSymbolTree(node) + ";\n"; }); @@ -725,6 +754,37 @@ var getImportCode = function (imports, header, omitvar) { return buf; }; +var getFooter = function (options) { + var chunks = []; + + if (options.name && options.exports && !_.isEmpty(options.exports)) { + chunks.push("/* Exports */\n"); + chunks.push("if (typeof Package === 'undefined') Package = {};\n"); + chunks.push(packageDot(options.name), " = "); + + // Even if there are no exports, we need to define Package.foo, because the + // existence of Package.foo is how another package (eg, one that weakly + // depends on foo) can tell if foo is loaded. + if (_.isEmpty(options.exports)) { + chunks.push("{};\n"); + } else { + // Given exports like Foo, Bar.Baz, Bar.Quux.A, and Bar.Quux.B, + // construct an expression like + // {Foo: Foo, Bar: {Baz: Bar.Baz, Quux: {A: Bar.Quux.A, B: Bar.Quux.B}}} + var scratch = {}; + _.each(options.exports, function (symbol) { + scratch[symbol] = symbol; + }); + var exportTree = buildSymbolTree(scratch); + chunks.push(writeSymbolTree(exportTree, 0)); + chunks.push(";\n"); + } + } + + chunks.push("\n})();\n"); + return chunks.join(''); +}; + var linker = module.exports = { prelink: prelink, link: link diff --git a/tools/packages.js b/tools/packages.js index b1c4a3840d..46ac0f3bc8 100644 --- a/tools/packages.js +++ b/tools/packages.js @@ -12,6 +12,7 @@ var archinfo = require(path.join(__dirname, 'archinfo.js')); var linker = require(path.join(__dirname, 'linker.js')); var unipackage = require('./unipackage.js'); var fs = require('fs'); +var sourcemap = require('source-map'); // Find all files under `rootPath` that have an extension in // `extensions` (an array of extensions without leading dot), and @@ -61,6 +62,12 @@ var scanForSources = function (rootPath, extensions, ignoreFiles) { }); }; +var rejectBadPath = function (p) { + if (p.match(/\.\./)) + throw new Error("bad path: " + p); +}; + + /////////////////////////////////////////////////////////////////////////////// // Slice /////////////////////////////////////////////////////////////////////////////// @@ -147,14 +154,15 @@ var Slice = function (pkg, options) { // Are we allowed to have exports? (eg, test slices don't export.) self.noExports = !!options.noExports; - // Prelink output. 'boundary' is a magic cookie used for inserting - // imports. 'prelinkFiles' is the partially linked JavaScript code - // (an array of objects with keys 'source' and 'servePath', both - // strings -- see prelink() in linker.js) Both of these are inputs - // into the final link phase, which inserts the final JavaScript - // resources into 'resources'. Set only when isBuilt is true. - self.boundary = null; + // Prelink output. 'prelinkFiles' is the partially linked JavaScript code (an + // array of objects with keys 'source' and 'servePath', both strings -- see + // prelink() in linker.js) 'packageScopeVariables' are are variables that are + // syntactically globals in our input files and which we capture with a + // package-scope closure. Both of these are inputs into the final link phase, + // which inserts the final JavaScript resources into 'resources'. Set only + // when isBuilt is true. self.prelinkFiles = null; + self.packageScopeVariables = null; // All of the data provided for eventual inclusion in the bundle, // other than JavaScript that still needs to be fed through the @@ -173,6 +181,8 @@ var Slice = function (pkg, options) { // honored for "static", ignored for "head" and "body", sometimes // honored for CSS but ignored if we are concatenating. // + // sourceMap: Allowed only for "js". If present, a string. + // // Set only when isBuilt is true. self.resources = null; @@ -191,7 +201,7 @@ _.extend(Slice.prototype, { // through the appropriate handlers and run the prelink phase on any // resulting JavaScript. Also add all provided source files to the // package dependencies. Sets fields such as dependencies, exports, - // boundary, prelinkFiles, and resources. + // prelinkFiles, packageScopeVariables, and resources. build: function () { var self = this; var isApp = ! self.pkg.name; @@ -258,6 +268,8 @@ _.extend(Slice.prototype, { // can ensure that the version of the file that you use is // exactly the one that is recorded in the dependency // information. + // - pathForSourceMap: If this file is to be included in a source map, + // this is the name you should use for it in the map. // - rootOutputPath: on browser targets, for resources such as // stylesheet and static assets, this is the root URL that // will get prepended to the paths you pick for your output @@ -282,7 +294,7 @@ _.extend(Slice.prototype, { // effect, such as minification.) // - addJavaScript({ path: "my/program.js", data: "my code", // sourcePath: "src/my/program.js", - // lineForLine: true, bare: true}) + // bare: true }) // Add JavaScript code, which will be namespaced into this // package's environment (eg, it will see only the exports of // this package's imports), and which will be subject to @@ -291,11 +303,7 @@ _.extend(Slice.prototype, { // that will be used in any error messages generated (eg, // "foo.js:4:1: syntax error"). It must be present and should // be relative to the project root. Typically 'inputPath' will - // do handsomely. Set the misleadingly named lineForLine - // option to true if line X, column Y in the input corresponds - // to line X, column Y in the output. This will enable line - // and column reporting in error messages. (XXX replace this - // with source maps) "bare" means to not wrap the file in + // do handsomely. "bare" means to not wrap the file in // a closure, so that its vars are shared with other files // in the module. // - addAsset({ path: "my/image.png", data: Buffer }) @@ -308,11 +316,11 @@ _.extend(Slice.prototype, { // Assets.getText or Assets.getBinary. // - error({ message: "There's a problem in your source file", // sourcePath: "src/my/program.ext", line: 12, - // column: 20, columnEnd: 25, func: "doStuff" }) + // column: 20, func: "doStuff" }) // Flag an error -- at a particular location in a source // file, if you like (you can even indicate a function name // to show in the error, like in stack traces.) sourcePath, - // line, column, columnEnd, and func are all optional. + // line, column, and func are all optional. // // XXX for now, these handlers must only generate portable code // (code that isn't dependent on the arch, other than 'browser' @@ -350,6 +358,14 @@ _.extend(Slice.prototype, { inputSize: contents.length, inputPath: relPath, _fullInputPath: absPath, // avoid, see above.. + // XXX duplicates _pathForSourceMap() in linker + pathForSourceMap: ( + self.pkg.name + ? self.pkg.name + "/" + relPath + : path.basename(relPath)), + // null if this is an app. intended to be used for the sources + // dictionary for source maps. + packageName: self.pkg.name, rootOutputPath: self.pkg.serveRoot, arch: self.arch, fileOptions: fileOptions, @@ -396,9 +412,9 @@ _.extend(Slice.prototype, { source: options.data, sourcePath: options.sourcePath, servePath: path.join(self.pkg.serveRoot, options.path), - includePositionInErrors: options.lineForLine, - linkerUnitTransform: options.linkerUnitTransform, - bare: !!options.bare + linkerFileTransform: options.linkerFileTransform, + bare: !!options.bare, + sourceMap: options.sourceMap }); }, addAsset: function (options) { @@ -450,8 +466,6 @@ _.extend(Slice.prototype, { combinedServePath: isApp ? null : "/packages/" + self.pkg.name + (self.sliceName === "main" ? "" : ("." + self.sliceName)) + ".js", - // XXX report an error if there is a package called global-imports - importStubServePath: '/packages/global-imports.js', name: self.pkg.name || null, forceExport: self.forceExport, noExports: self.noExports, @@ -475,8 +489,8 @@ _.extend(Slice.prototype, { }); self.prelinkFiles = results.files; - self.boundary = results.boundary; self.exports = results.exports; + self.packageScopeVariables = results.packageScopeVariables; self.resources = resources; self.isBuilt = true; }, @@ -530,17 +544,23 @@ _.extend(Slice.prototype, { var files = linker.link({ imports: imports, useGlobalNamespace: isApp, + // XXX report an error if there is a package called global-imports + importStubServePath: isApp && '/packages/global-imports.js', prelinkFiles: self.prelinkFiles, - boundary: self.boundary + exports: self.exports, + packageScopeVariables: self.packageScopeVariables, + includeSourceMapInstructions: archinfo.matches(self.arch, "browser"), + name: self.pkg.name || null }); // Add each output as a resource var jsResources = _.map(files, function (file) { return { type: "js", - data: new Buffer(file.source, 'utf8'), + data: new Buffer(file.source, 'utf8'), // XXX encoding servePath: file.servePath, - staticDirectory: self.staticDirectory + staticDirectory: self.staticDirectory, + sourceMap: file.sourceMap }; }); @@ -590,7 +610,6 @@ _.extend(Slice.prototype, { data: compileStep.read().toString('utf8'), path: compileStep.inputPath, sourcePath: compileStep.inputPath, - lineForLine: true, // XXX eventually get rid of backward-compatibility "raw" name bare: compileStep.fileOptions.bare || compileStep.fileOptions.raw }); @@ -1298,8 +1317,10 @@ _.extend(Package.prototype, { }; try { - files.runJavaScript(code.toString('utf8'), 'package.js', - { Package: Package, Npm: Npm }); + files.runJavaScript(code.toString('utf8'), { + filename: 'package.js', + symbols: { Package: Package, Npm: Npm } + }); } catch (e) { buildmessage.exception(e); @@ -1793,8 +1814,8 @@ _.extend(Package.prototype, { self.testSlices = mainJson.testSlices; _.each(mainJson.plugins, function (pluginMeta) { - if (pluginMeta.path.match(/\.\./)) - throw new Error("bad path in unipackage"); + rejectBadPath(pluginMeta.path); + var plugin = bundler.readJsImage(path.join(dir, pluginMeta.path)); if (! archinfo.matches(archinfo.host(), plugin.arch)) { @@ -1820,8 +1841,7 @@ _.extend(Package.prototype, { _.each(mainJson.slices, function (sliceMeta) { // aggressively sanitize path (don't let it escape to parent // directory) - if (sliceMeta.path.match(/\.\./)) - throw new Error("bad path in unipackage"); + rejectBadPath(sliceMeta.path); var sliceJson = JSON.parse( fs.readFileSync(path.join(dir, sliceMeta.path))); var sliceBasePath = path.dirname(path.join(dir, sliceMeta.path)); @@ -1832,8 +1852,7 @@ _.extend(Package.prototype, { var nodeModulesPath = null; if (sliceJson.node_modules) { - if (sliceJson.node_modules.match(/\.\./)) - throw new Error("bad node_modules path in unipackage"); + rejectBadPath(sliceJson.node_modules); nodeModulesPath = path.join(sliceBasePath, sliceJson.node_modules); } @@ -1858,13 +1877,12 @@ _.extend(Package.prototype, { slice.isBuilt = true; slice.exports = sliceJson.exports || []; - slice.boundary = sliceJson.boundary; + slice.packageScopeVariables = sliceJson.packageScopeVariables || []; slice.prelinkFiles = []; slice.resources = []; _.each(sliceJson.resources, function (resource) { - if (resource.file.match(/\.\./)) - throw new Error("bad resource file path in unipackage"); + rejectBadPath(resource.file); var fd = fs.openSync(path.join(sliceBasePath, resource.file), "r"); try { @@ -1878,10 +1896,16 @@ _.extend(Package.prototype, { throw new Error("couldn't read entire resource"); if (resource.type === "prelink") { - slice.prelinkFiles.push({ + var prelinkFile = { source: data.toString('utf8'), servePath: resource.servePath - }); + }; + if (resource.sourceMap) { + rejectBadPath(resource.sourceMap); + prelinkFile.sourceMap = fs.readFileSync( + path.join(sliceBasePath, resource.sourceMap), 'utf8'); + } + slice.prelinkFiles.push(prelinkFile); } else if (_.contains(["head", "body", "css", "js", "static"], resource.type)) { slice.resources.push({ @@ -1934,7 +1958,7 @@ _.extend(Package.prototype, { var buildInfoJson = { dependencies: { files: {}, directories: {} }, - source: options.buildOfPath || undefined, + source: options.buildOfPath || undefined }; builder.reserve("unipackage.json"); @@ -1977,6 +2001,7 @@ _.extend(Package.prototype, { var sliceJson = { format: "unipackage-slice-pre1", exports: slice.exports, + packageScopeVariables: slice.packageScopeVariables, uses: _.map(slice.uses, function (u) { var specParts = u.spec.split('.'); if (specParts.length > 2) @@ -1990,7 +2015,6 @@ _.extend(Package.prototype, { }), node_modules: slice.nodeModulesPath ? 'npm/node_modules' : undefined, resources: [], - boundary: slice.boundary, staticDirectory: path.join(sliceDir, self.serveRoot) }; @@ -2028,13 +2052,11 @@ _.extend(Package.prototype, { if (_.contains(["head", "body"], resource.type)) return; // already did this one - var resourcePath = builder.generateFilename( - path.join(sliceDir, resource.servePath)); - - builder.write(resourcePath, { data: resource.data }); sliceJson.resources.push({ type: resource.type, - file: resourcePath, + file: builder.writeToGeneratedFilename( + path.join(sliceDir, resource.servePath), + { data: resource.data }), length: resource.data.length, offset: 0, servePath: resource.servePath || undefined @@ -2043,21 +2065,26 @@ _.extend(Package.prototype, { // Output prelink resources _.each(slice.prelinkFiles, function (file) { - var resourcePath = builder.generateFilename( - path.join(sliceDir, file.servePath)); var data = new Buffer(file.source, 'utf8'); - - builder.write(resourcePath, { - data: data - }); - - sliceJson.resources.push({ + var resource = { type: 'prelink', - file: resourcePath, + file: builder.writeToGeneratedFilename( + path.join(sliceDir, file.servePath), + { data: data }), length: data.length, offset: 0, servePath: file.servePath || undefined - }); + }; + + if (file.sourceMap) { + // Write the source map. + resource.sourceMap = builder.writeToGeneratedFilename( + path.join(sliceDir, file.servePath + '.map'), + { data: new Buffer(file.sourceMap, 'utf8') } + ); + } + + sliceJson.resources.push(resource); }); // If slice has included node_modules, copy them in diff --git a/tools/server/boot.js b/tools/server/boot.js index c4dce95b1a..1170ca9ff0 100644 --- a/tools/server/boot.js +++ b/tools/server/boot.js @@ -3,6 +3,7 @@ var fs = require("fs"); var path = require("path"); var Future = require(path.join("fibers", "future")); var _ = require('underscore'); +var sourcemap_support = require('source-map-support'); // This code is duplicated in tools/server/server.js. var MIN_NODE_VERSION = 'v0.8.24'; @@ -13,15 +14,16 @@ if (require('semver').lt(process.version, MIN_NODE_VERSION)) { } // read our control files -var serverJson = - JSON.parse(fs.readFileSync(path.join(__dirname, process.argv[2]), 'utf8')); +var serverJsonPath = path.resolve(process.argv[2]); +var serverDir = path.dirname(serverJsonPath); +var serverJson = JSON.parse(fs.readFileSync(serverJsonPath, 'utf8')); var configJson = - JSON.parse(fs.readFileSync(path.join(__dirname, 'config.json'), 'utf8')); + JSON.parse(fs.readFileSync(path.resolve(serverDir, 'config.json'), 'utf8')); // Set up environment __meteor_bootstrap__ = { startup_hooks: [], - serverDir: __dirname, + serverDir: serverDir, configJson: configJson }; __meteor_runtime_config__ = { meteorRelease: configJson.release }; @@ -34,10 +36,49 @@ __meteor_runtime_config__ = { meteorRelease: configJson.release }; if (!process.env.NODE_ENV) process.env.NODE_ENV = 'production'; +// Map from load path to its source map. +var parsedSourceMaps = {}; + +// Read all the source maps into memory once. +_.each(serverJson.load, function (fileInfo) { + if (fileInfo.sourceMap) { + var rawSourceMap = fs.readFileSync( + path.resolve(serverDir, fileInfo.sourceMap), 'utf8'); + // Parse the source map only once, not each time it's needed. Also remove + // the anti-XSSI header if it's there. + var parsedSourceMap = JSON.parse(rawSourceMap.replace(/^\)\]\}'/, '')); + // source-map-support doesn't ever look at the sourcesContent field, so + // there's no point in keeping it in memory. + delete parsedSourceMap.sourcesContent; + var url; + if (fileInfo.sourceMapRoot) { + // Add the specified root to any root that may be in the file. + parsedSourceMap.sourceRoot = path.join( + fileInfo.sourceMapRoot, parsedSourceMap.sourceRoot || ''); + } + parsedSourceMaps[fileInfo.path] = parsedSourceMap; + } +}); + +var retrieveSourceMap = function (pathForSourceMap) { + if (_.has(parsedSourceMaps, pathForSourceMap)) + return { map: parsedSourceMaps[pathForSourceMap] }; + return null; +}; + +sourcemap_support.install({ + // Use the source maps specified in program.json instead of parsing source + // code for them. + retrieveSourceMap: retrieveSourceMap, + // For now, don't fix the source line in uncaught exceptions, because we + // haven't fixed handleUncaughtExceptions in source-map-support to properly + // locate the source files. + handleUncaughtExceptions: false +}); Fiber(function () { _.each(serverJson.load, function (fileInfo) { - var code = fs.readFileSync(path.join(__dirname, fileInfo.path)); + var code = fs.readFileSync(path.resolve(serverDir, fileInfo.path)); var Npm = { require: function (name) { @@ -46,7 +87,7 @@ Fiber(function () { } var nodeModuleDir = - path.join(__dirname, fileInfo.node_modules, name); + path.resolve(serverDir, fileInfo.node_modules, name); if (fs.existsSync(nodeModuleDir)) { return require(nodeModuleDir); @@ -67,7 +108,7 @@ Fiber(function () { } } }; - var staticDirectory = path.join(__dirname, fileInfo.staticDirectory); + var staticDirectory = path.resolve(serverDir, fileInfo.staticDirectory); var getAsset = function (assetPath, encoding, callback) { var fut; if (! callback) { diff --git a/tools/tests/test_bundler.js b/tools/tests/test_bundler.js index 434c028fad..0e9b24e3c9 100644 --- a/tools/tests/test_bundler.js +++ b/tools/tests/test_bundler.js @@ -12,7 +12,7 @@ /*global*/ Fiber = require('fibers'); /*global*/ Future = require('fibers/future'); -/*global*/ mainJSContents = "process.argv.splice(2, 0, 'program.json');\nrequire('./programs/server/boot.js');\n"; +/*global*/ mainJSContents = "process.argv.splice(2, 0, 'program.json');\nprocess.chdir(require('path').join(__dirname, 'programs', 'server'));\nrequire('./programs/server/boot.js');\n"; var tmpBaseDir = files.mkdtemp('test_bundler'); var tmpCounter = 1;