diff --git a/tools/builder.js b/tools/builder.js index 92db3af3db..c906ec84c3 100644 --- a/tools/builder.js +++ b/tools/builder.js @@ -1,4 +1,4 @@ -import {WatchSet, readAndWatchFile} from './watch.js'; +import {WatchSet, readAndWatchFile, sha1} from './watch.js'; import files from './files.js'; import NpmDiscards from './npm-discards.js'; import {Profile} from './profile.js'; @@ -23,8 +23,10 @@ export default class Builder { // Paths already written to. Map from canonicalized relPath (no // trailing slash) to true for a file, or false for a directory. this.usedAsFile = { '': false, '.': false }; + this.previousUsedAsFile = {}; this.writtenHashes = {}; + this.previousWrittenHashes = {}; // foo/bar => foo/.build1234.bar // Should we include a random number? The advantage is that multiple @@ -34,9 +36,30 @@ export default class Builder { this.buildPath = files.pathJoin(files.pathDirname(this.outputPath), '.build' + nonce + "." + files.pathBasename(this.outputPath)); + files.rm_recursive(this.buildPath); - files.mkdir_p(this.buildPath, 0o755); + if (previousBuilder) { + if (previousBuilder.outputPath !== outputPath) { + throw new Error( + `previousBuilder option can only be set to a builder with the same output path. +Previous builder: ${previousBuilder.outputPath}, this builder: ${outputPath}` + ); + } + + if (files.exists(previousBuilder.outputPath)) { + files.renameDirAlmostAtomically( + previousBuilder.outputPath, + this.buildPath); + + this.previousWrittenHashes = previousBuilder.writtenHashes; + this.previousUsedAsFile = previousBuilder.usedAsFile; + } else { + console.log('XXX the previous build doesnt exist', previousBuilder.outputPath); + } + } else { + files.mkdir_p(this.buildPath, 0o755); + } this.watchSet = new WatchSet(); @@ -57,8 +80,21 @@ export default class Builder { partsSoFar.push(part); const partial = partsSoFar.join(files.pathSep); if (! (partial in this.usedAsFile)) { - // It's new -- create it - files.mkdir(files.pathJoin(this.buildPath, partial), 0o755); + let needToMkdir = true; + if (partial in this.previousUsedAsFile) { + if (this.previousUsedAsFile[partial]) { + // was previously used as file, delete it, create a directory + files.unlink(partial); + } else { + // is already a directory + needToMkdir = false; + } + } + + if (needToMkdir) { + // It's new -- create it + files.mkdir(files.pathJoin(this.buildPath, partial), 0o755); + } this.usedAsFile[partial] = false; } else if (this.usedAsFile[partial]) { // Already exists and is a file. Oops. @@ -126,20 +162,22 @@ export default class Builder { // `data` and or `file` must be passed. // // Options: - // - data: a Buffer to write to relPath. + // - data: a Buffer to write to relPath. Overrides `file`. // - file: a filename to write to relPath, as a string. // - sanitize: if true, then all components of the path are stripped // of any potentially troubling characters, an exception is thrown // if any path segments consist entirely of dots (eg, '..'), and // if there is a file in the bundle with the same relPath, then // the path is changed by adding a numeric suffix. + // - hash: a sha1 string used to determine if the contents of the + // new file written is not cached. // - executable: if true, mark the file as executable. // - symlink: if set to a string, create a symlink to its value // // Returns the final canonicalize relPath that was written to. // // If `file` is used then it will be added to the builder's WatchSet. - write(relPath, {data, file, sanitize, executable, symlink}) { + write(relPath, {data, file, hash, sanitize, executable, symlink}) { // Ensure no trailing slash if (relPath.slice(-1) === files.pathSep) relPath = relPath.slice(0, -1); @@ -149,14 +187,18 @@ export default class Builder { if (sanitize) relPath = this._sanitize(relPath); + let getData = null; if (data) { if (! (data instanceof Buffer)) throw new Error("data must be a Buffer"); if (file) throw new Error("May only pass one of data and file, not both"); + getData = () => data; } else if (file) { - data = - readAndWatchFile(this.watchSet, files.pathResolve(file)); + // postpone reading the file into memory + getData = () => readAndWatchFile(this.watchSet, files.pathResolve(file)); + } else if (! symlink) { + throw new Error('Builder can not write without either data or a file path or a symlink path: ' + relPath); } this._ensureDirectory(files.pathDirname(relPath)); @@ -165,10 +207,22 @@ export default class Builder { if (symlink) { files.symlink(symlink, absPath); } else { - // Builder is used to create build products, which should be read-only; - // users shouldn't be manually editing automatically generated files and - // expecting the results to "stick". - files.writeFile(absPath, data, { mode: executable ? 0o555 : 0o444 }); + hash = hash || sha1(getData()); + + if (this.previousWrittenHashes[relPath] !== hash) { + if (files.exists(absPath)) { + // XXX what if it is a directory? + files.unlink(absPath); + } + // Builder is used to create build products, which should be read-only; + // users shouldn't be manually editing automatically generated files and + // expecting the results to "stick". + files.writeFile(absPath, getData(), { + mode: executable ? 0o555 : 0o444 + }); + } + + this.writtenHashes[relPath] = hash; } this.usedAsFile[relPath] = true; @@ -184,9 +238,17 @@ export default class Builder { relPath = relPath.slice(0, -1); this._ensureDirectory(files.pathDirname(relPath)); - files.writeFile(files.pathJoin(this.buildPath, relPath), - new Buffer(JSON.stringify(data, null, 2), 'utf8'), - {mode: 0o444}); + const absPath = files.pathJoin(this.buildPath, relPath); + + if (files.exists(absPath)) { + // XXX what if it is a directory? + files.unlink(absPath); + } + + files.writeFile( + absPath, + new Buffer(JSON.stringify(data, null, 2), 'utf8'), + {mode: 0o444}); this.usedAsFile[relPath] = true; } @@ -222,7 +284,17 @@ export default class Builder { const shouldBeDirectory = (i < parts.length - 1) || directory; if (shouldBeDirectory) { if (! (soFar in this.usedAsFile)) { - files.mkdir(files.pathJoin(this.buildPath, soFar), 0o755); + let needToMkdir = true; + if (soFar in this.previousUsedAsFile) { + if (this.previousUsedAsFile[soFar]) { + files.unlink(soFar); + } else { + needToMkdir = false; + } + } + if (needToMkdir) { + files.mkdir(files.pathJoin(this.buildPath, soFar), 0o755); + } this.usedAsFile[soFar] = false; } } else { @@ -377,6 +449,8 @@ export default class Builder { // it" goes. this.usedAsFile[thisRelTo] = true; } else { + // XXX can't really optimize this copying without reading + // the file into memory to calculate the hash. files.copyFile(thisAbsFrom, files.pathResolve(this.buildPath, thisRelTo), fileStatus.mode); diff --git a/tools/bundler.js b/tools/bundler.js index a48b6f5c1c..5e72096ebc 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -1689,32 +1689,37 @@ var writeFile = Profile("bundler..writeFile", function (file, builder) { // to wait until the server is actually driven by the manifest // (rather than just serving all of the files in a certain // directories) - builder.write(file.targetPath, { data: file.contents() }); + builder.write(file.targetPath, { data: file.contents(), hash: file.hash() }); }); // Writes a target a path in 'programs' var writeTargetToPath = Profile( - "bundler..writeTargetToPath", function (name, target, outputPath, options) { - var builder = new Builder({ - outputPath: files.pathJoin(outputPath, 'programs', name) + "bundler..writeTargetToPath", + function (name, target, outputPath, { + includeNodeModules, + getRelativeTargetPath, + previousBuilder + }) { + var builder = new Builder({ + outputPath: files.pathJoin(outputPath, 'programs', name), + previousBuilder + }); + + var targetBuild = + target.write(builder, {includeNodeModules, getRelativeTargetPath}); + + builder.complete(); + + return { + name, + arch: target.mostCompatibleArch(), + path: files.pathJoin('programs', name, targetBuild.controlFile), + nodePath: targetBuild.nodePath, + cordovaDependencies: target.cordovaDependencies || undefined, + builder + }; }); - var targetBuild = - target.write(builder, { - includeNodeModules: options.includeNodeModules, - getRelativeTargetPath: options.getRelativeTargetPath }); - - builder.complete(); - - return { - name: name, - arch: target.mostCompatibleArch(), - path: files.pathJoin('programs', name, targetBuild.controlFile), - nodePath: targetBuild.nodePath, - cordovaDependencies: target.cordovaDependencies || undefined - }; -}); - /////////////////////////////////////////////////////////////////////////////// // writeSiteArchive /////////////////////////////////////////////////////////////////////////////// @@ -1741,18 +1746,26 @@ var writeTargetToPath = Profile( // - builtBy: vanity identification string to write into metadata // - releaseName: The Meteor release version // - getRelativeTargetPath: see doc at ServerTarget.write +// - previousBuilder: previous Builder object used in previous iteration var writeSiteArchive = Profile( - "bundler..writeSiteArchive", function (targets, outputPath, options) { - var builder = new Builder({ - outputPath: outputPath - }); + "bundler..writeSiteArchive", + function (targets, outputPath, { + includeNodeModules, + builtBy, + releaseName, + getRelativeTargetPath, + previousBuilders + }) { + + const builders = {}; + const builder = new Builder({outputPath}); try { var json = { format: "site-archive-pre1", - builtBy: options.builtBy, + builtBy, programs: [], - meteorRelease: options.releaseName + meteorRelease: releaseName }; var nodePath = []; @@ -1773,28 +1786,29 @@ var writeSiteArchive = Profile( }); builder.write('README', { data: new Buffer( -"This is a Meteor application bundle. It has only one external dependency:\n" + -"Node.js 0.10.36 or newer. To run the application:\n" + -"\n" + -" $ (cd programs/server && npm install)\n" + -" $ export MONGO_URL='mongodb://user:password@host:port/databasename'\n" + -" $ export ROOT_URL='http://example.com'\n" + -" $ export MAIL_URL='smtp://user:password@mailhost:port/'\n" + -" $ node main.js\n" + -"\n" + -"Use the PORT environment variable to set the port where the\n" + -"application will listen. The default is 80, but that will require\n" + -"root on most systems.\n" + -"\n" + -"Find out more about Meteor at meteor.com.\n", +`This is a Meteor application bundle. It has only one external dependency: +Node.js 0.10.36 or newer. To run the application: + + $ (cd programs/server && npm install) + $ export MONGO_URL='mongodb://user:password@host:port/databasename' + $ export ROOT_URL='http://example.com' + $ export MAIL_URL='smtp://user:password@mailhost:port/' + $ node main.js + +Use the PORT environment variable to set the port where the +application will listen. The default is 80, but that will require +root on most systems. + +Find out more about Meteor at meteor.com. +`, 'utf8')}); } // Merge the WatchSet of everything that went into the bundle. - var clientWatchSet = new watch.WatchSet(); - var serverWatchSet = new watch.WatchSet(); - var dependencySources = [builder].concat(_.values(targets)); - _.each(dependencySources, function (s) { + const clientWatchSet = new watch.WatchSet(); + const serverWatchSet = new watch.WatchSet(); + const dependencySources = [builder].concat(_.values(targets)); + dependencySources.forEach(s => { if (s instanceof ClientTarget) { clientWatchSet.merge(s.getWatchSet()); } else { @@ -1802,16 +1816,29 @@ var writeSiteArchive = Profile( } }); - _.each(targets, function (target, name) { - var targetBuild = writeTargetToPath(name, target, builder.buildPath, { - includeNodeModules: options.includeNodeModules, - builtBy: options.builtBy, - releaseName: options.releaseName, - getRelativeTargetPath: options.getRelativeTargetPath + Object.keys(targets).forEach(name => { + const target = targets[name]; + const previousBuilder = previousBuilders[name]; + const { + arch, path, cordovaDependencies, + nodePath: targetNP, + builder: targetBuilder + } = + writeTargetToPath(name, target, builder.buildPath, { + includeNodeModules, + builtBy, + releaseName, + getRelativeTargetPath, + previousBuilder + }); + + builders[name] = targetBuilder; + + json.programs.push({ + name, arch, path, cordovaDependencies }); - json.programs.push(targetBuild); - nodePath = nodePath.concat(targetBuild.nodePath); + nodePath = nodePath.concat(targetNP); }); // Control file @@ -1820,11 +1847,19 @@ var writeSiteArchive = Profile( // We did it! builder.complete(); + // now, go and "fix up" the outputPath properties of the sub-builders + Object.keys(builders).forEach(name => { + const subBuilder = builders[name]; + subBuilder.outputPath = builder.outputPath + subBuilder.outputPath.substring(builder.buildPath.length); + console.log("fix up: ", subBuilder.outputPath); + }); + return { - clientWatchSet: clientWatchSet, - serverWatchSet: serverWatchSet, + clientWatchSet, + serverWatchSet, starManifest: json, - nodePath: nodePath + nodePath, + builders }; } catch (e) { builder.abort(); @@ -1902,14 +1937,17 @@ var writeSiteArchive = Profile( * will point somewhere else -- into the app (if any) whose packages * you are testing! */ -exports.bundle = function (options) { - var projectContext = options.projectContext; - - var outputPath = options.outputPath; - var includeNodeModules = options.includeNodeModules; - var buildOptions = options.buildOptions || {}; - buildOptions.minify = buildOptions.minify || 'development'; - var shouldLint = options.lint || false; +exports.bundle = function ({ + projectContext, + outputPath, + includeNodeModules, + buildOptions, + lint: shouldLint, + previousBuilders, + hasCachedBundle +}) { + buildOptions = buildOptions || {}; + shouldLint = shouldLint || false; var appDir = projectContext.projectDir; @@ -1929,6 +1967,7 @@ exports.bundle = function (options) { var targets = {}; var nodePath = []; var lintingMessages = null; + var builders = {}; if (! release.usingRightReleaseForApp(projectContext)) throw new Error("running wrong release for app?"); @@ -2023,7 +2062,7 @@ exports.bundle = function (options) { }); // Server - if (! options.hasCachedBundle) { + if (! hasCachedBundle) { var server = makeServerTarget(app, clientTargets); targets.server = server; } @@ -2049,27 +2088,36 @@ exports.bundle = function (options) { // Write to disk var writeOptions = { - includeNodeModules: includeNodeModules, - builtBy: builtBy, - releaseName: releaseName, - getRelativeTargetPath: getRelativeTargetPath + includeNodeModules, + builtBy, + releaseName, + getRelativeTargetPath }; if (outputPath !== null) { - if (options.hasCachedBundle) { + if (hasCachedBundle) { // If we already have a cached bundle, just recreate the new targets. // XXX This might make the contents of "star.json" out of date. + builders = _.clone(previousBuilders); _.each(targets, function (target, name) { + const previousBuilder = previousBuilders[name]; + console.log('previous builder for ', name, previousBuilder.outputPath); var targetBuild = - writeTargetToPath(name, target, outputPath, writeOptions); + writeTargetToPath(name, target, outputPath, _.extend(writeOptions, {previousBuilder})); nodePath = nodePath.concat(targetBuild.nodePath); clientWatchSet.merge(target.getWatchSet()); + builders[name] = targetBuild.builder; }); } else { - starResult = writeSiteArchive(targets, outputPath, writeOptions); + starResult = writeSiteArchive( + targets, + outputPath, + _.extend(writeOptions, {previousBuilders})); + nodePath = nodePath.concat(starResult.nodePath); serverWatchSet.merge(starResult.serverWatchSet); clientWatchSet.merge(starResult.clientWatchSet); + builders = starResult.builders; } } @@ -2082,10 +2130,11 @@ exports.bundle = function (options) { return { errors: success ? false : messages, warnings: lintingMessages, - serverWatchSet: serverWatchSet, - clientWatchSet: clientWatchSet, + serverWatchSet, + clientWatchSet, starManifest: starResult && starResult.starManifest, - nodePath: nodePath + nodePath, + builders }; }; diff --git a/tools/run-app.js b/tools/run-app.js index 1da5a29e4c..6e7295ebfe 100644 --- a/tools/run-app.js +++ b/tools/run-app.js @@ -465,6 +465,9 @@ _.extend(AppRunner.prototype, { // a single invocation of _runOnce(). var cachedServerWatchSet; + // Builders saved from previous iterations + var builders = {}; + var bundleApp = function () { if (! firstRun) { // If the build fails in a way that could be fixed by a refresh, allow @@ -553,14 +556,20 @@ _.extend(AppRunner.prototype, { includeNodeModules = 'reference-directly'; } - return bundler.bundle({ + var bundleResult = bundler.bundle({ projectContext: self.projectContext, outputPath: bundlePath, includeNodeModules: includeNodeModules, buildOptions: self.buildOptions, hasCachedBundle: !! cachedServerWatchSet, - lint: self.lint + lint: self.lint, + previousBuilders: builders }); + + // save new builders with their caches + ({builders} = bundleResult); + + return bundleResult; }); // Keep the server watch set from the initial bundle, because subsequent @@ -763,10 +772,10 @@ _.extend(AppRunner.prototype, { if (bundleResultOrRunResult.runResult) return bundleResultOrRunResult.runResult; bundleResult = bundleResultOrRunResult.bundleResult; - if (bundleResult.warnings) { + if (self.lint && bundleResult.warnings) { runLog.log( 'Linting your app.\n\n' + - bundleResult.warnings.formatMessages(), { arrow: true }) + bundleResult.warnings.formatMessages(), { arrow: true }); } var oldFuture = self.runFuture = new Future;