github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Reuse Builder's from previous iterations

Browse Source 
An optimization to avoid writing the same files over and over again.
Tested only on Mac OS X.

There is a potential problem with this approach on Windows, since the
way processes retain open files is different.

On Linux (and I assume, OS X, too), when a process has a file open, it
reads the file from a memory snapshot loaded by OS. The OS also would
increment the reference count for that file, so even if it is unlinked
from the FS, OS would still keep it available as long as the process
keeps it open.

When we want to replace a file under a running Meteor app process, we
first delete the old built file and then write a new file to the same
path. In my understanding, it would force the OS to assign a different
inode for the new file, so the old file and the new file will be
different. Then, we either restart the app process, or signal it to
reload its assets, so it reads the new files, releasing the older ones.

I need to verify this understanding with somebody who actually knows how
OS and FS work.
This commit is contained in:
Slava Kim
2015-07-05 13:22:32 -07:00
parent ff61acade9
commit a3841479eb
3 changed files with 226 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
tools/builder.js
View File

@@ -1,4 +1,4 @@
import {WatchSet, readAndWatchFile} from './watch.js';
import {WatchSet, readAndWatchFile, sha1} from './watch.js';
import files from './files.js';
import NpmDiscards from './npm-discards.js';
import {Profile} from './profile.js';
@@ -23,8 +23,10 @@ export default class Builder {
// Paths already written to. Map from canonicalized relPath (no
// trailing slash) to true for a file, or false for a directory.
this.usedAsFile = { '': false, '.': false };
this.previousUsedAsFile = {};
this.writtenHashes = {};
this.previousWrittenHashes = {};
// foo/bar => foo/.build1234.bar
// Should we include a random number? The advantage is that multiple
@@ -34,9 +36,30 @@ export default class Builder {
this.buildPath = files.pathJoin(files.pathDirname(this.outputPath),
'.build' + nonce + "." +
files.pathBasename(this.outputPath));
files.rm_recursive(this.buildPath);
files.mkdir_p(this.buildPath, 0o755);
if (previousBuilder) {
if (previousBuilder.outputPath !== outputPath) {
throw new Error(
`previousBuilder option can only be set to a builder with the same output path.
Previous builder: ${previousBuilder.outputPath}, this builder: ${outputPath}`
);
}
if (files.exists(previousBuilder.outputPath)) {
files.renameDirAlmostAtomically(
previousBuilder.outputPath,
this.buildPath);
this.previousWrittenHashes = previousBuilder.writtenHashes;
this.previousUsedAsFile = previousBuilder.usedAsFile;
} else {
console.log('XXX the previous build doesnt exist', previousBuilder.outputPath);
}
} else {
files.mkdir_p(this.buildPath, 0o755);
}
this.watchSet = new WatchSet();
@@ -57,8 +80,21 @@ export default class Builder {
partsSoFar.push(part);
const partial = partsSoFar.join(files.pathSep);
if (! (partial in this.usedAsFile)) {
// It's new -- create it
files.mkdir(files.pathJoin(this.buildPath, partial), 0o755);
let needToMkdir = true;
if (partial in this.previousUsedAsFile) {
if (this.previousUsedAsFile[partial]) {
// was previously used as file, delete it, create a directory
files.unlink(partial);
} else {
// is already a directory
needToMkdir = false;
}
}
if (needToMkdir) {
// It's new -- create it
files.mkdir(files.pathJoin(this.buildPath, partial), 0o755);
}
this.usedAsFile[partial] = false;
} else if (this.usedAsFile[partial]) {
// Already exists and is a file. Oops.
@@ -126,20 +162,22 @@ export default class Builder {
// `data` and or `file` must be passed.
//
// Options:
// - data: a Buffer to write to relPath.
// - data: a Buffer to write to relPath. Overrides `file`.
// - file: a filename to write to relPath, as a string.
// - sanitize: if true, then all components of the path are stripped
// of any potentially troubling characters, an exception is thrown
// if any path segments consist entirely of dots (eg, '..'), and
// if there is a file in the bundle with the same relPath, then
// the path is changed by adding a numeric suffix.
// - hash: a sha1 string used to determine if the contents of the
// new file written is not cached.
// - executable: if true, mark the file as executable.
// - symlink: if set to a string, create a symlink to its value
//
// Returns the final canonicalize relPath that was written to.
//
// If `file` is used then it will be added to the builder's WatchSet.
write(relPath, {data, file, sanitize, executable, symlink}) {
write(relPath, {data, file, hash, sanitize, executable, symlink}) {
// Ensure no trailing slash
if (relPath.slice(-1) === files.pathSep)
relPath = relPath.slice(0, -1);
@@ -149,14 +187,18 @@ export default class Builder {
if (sanitize)
relPath = this._sanitize(relPath);
let getData = null;
if (data) {
if (! (data instanceof Buffer))
throw new Error("data must be a Buffer");
if (file)
throw new Error("May only pass one of data and file, not both");
getData = () => data;
} else if (file) {
data =
readAndWatchFile(this.watchSet, files.pathResolve(file));
// postpone reading the file into memory
getData = () => readAndWatchFile(this.watchSet, files.pathResolve(file));
} else if (! symlink) {
throw new Error('Builder can not write without either data or a file path or a symlink path: ' + relPath);
}
this._ensureDirectory(files.pathDirname(relPath));
@@ -165,10 +207,22 @@ export default class Builder {
if (symlink) {
files.symlink(symlink, absPath);
} else {
// Builder is used to create build products, which should be read-only;
// users shouldn't be manually editing automatically generated files and
// expecting the results to "stick".
files.writeFile(absPath, data, { mode: executable ? 0o555 : 0o444 });
hash = hash || sha1(getData());
if (this.previousWrittenHashes[relPath] !== hash) {
if (files.exists(absPath)) {
// XXX what if it is a directory?
files.unlink(absPath);
}
// Builder is used to create build products, which should be read-only;
// users shouldn't be manually editing automatically generated files and
// expecting the results to "stick".
files.writeFile(absPath, getData(), {
mode: executable ? 0o555 : 0o444
});
}
this.writtenHashes[relPath] = hash;
}
this.usedAsFile[relPath] = true;
@@ -184,9 +238,17 @@ export default class Builder {
relPath = relPath.slice(0, -1);
this._ensureDirectory(files.pathDirname(relPath));
files.writeFile(files.pathJoin(this.buildPath, relPath),
new Buffer(JSON.stringify(data, null, 2), 'utf8'),
{mode: 0o444});
const absPath = files.pathJoin(this.buildPath, relPath);
if (files.exists(absPath)) {
// XXX what if it is a directory?
files.unlink(absPath);
}
files.writeFile(
absPath,
new Buffer(JSON.stringify(data, null, 2), 'utf8'),
{mode: 0o444});
this.usedAsFile[relPath] = true;
}
@@ -222,7 +284,17 @@ export default class Builder {
const shouldBeDirectory = (i < parts.length - 1) || directory;
if (shouldBeDirectory) {
if (! (soFar in this.usedAsFile)) {
files.mkdir(files.pathJoin(this.buildPath, soFar), 0o755);
let needToMkdir = true;
if (soFar in this.previousUsedAsFile) {
if (this.previousUsedAsFile[soFar]) {
files.unlink(soFar);
} else {
needToMkdir = false;
}
}
if (needToMkdir) {
files.mkdir(files.pathJoin(this.buildPath, soFar), 0o755);
}
this.usedAsFile[soFar] = false;
}
} else {
@@ -377,6 +449,8 @@ export default class Builder {
// it" goes.
this.usedAsFile[thisRelTo] = true;
} else {
// XXX can't really optimize this copying without reading
// the file into memory to calculate the hash.
files.copyFile(thisAbsFrom,
files.pathResolve(this.buildPath, thisRelTo),
fileStatus.mode);

197
tools/bundler.js
View File

@@ -1689,32 +1689,37 @@ var writeFile = Profile("bundler..writeFile", function (file, builder) {
// to wait until the server is actually driven by the manifest
// (rather than just serving all of the files in a certain
// directories)
builder.write(file.targetPath, { data: file.contents() });
builder.write(file.targetPath, { data: file.contents(), hash: file.hash() });
});
// Writes a target a path in 'programs'
var writeTargetToPath = Profile(
"bundler..writeTargetToPath", function (name, target, outputPath, options) {
var builder = new Builder({
outputPath: files.pathJoin(outputPath, 'programs', name)
"bundler..writeTargetToPath",
function (name, target, outputPath, {
includeNodeModules,
getRelativeTargetPath,
previousBuilder
}) {
var builder = new Builder({
outputPath: files.pathJoin(outputPath, 'programs', name),
previousBuilder
});
var targetBuild =
target.write(builder, {includeNodeModules, getRelativeTargetPath});
builder.complete();
return {
name,
arch: target.mostCompatibleArch(),
path: files.pathJoin('programs', name, targetBuild.controlFile),
nodePath: targetBuild.nodePath,
cordovaDependencies: target.cordovaDependencies || undefined,
builder
};
});
var targetBuild =
target.write(builder, {
includeNodeModules: options.includeNodeModules,
getRelativeTargetPath: options.getRelativeTargetPath });
builder.complete();
return {
name: name,
arch: target.mostCompatibleArch(),
path: files.pathJoin('programs', name, targetBuild.controlFile),
nodePath: targetBuild.nodePath,
cordovaDependencies: target.cordovaDependencies || undefined
};
});
///////////////////////////////////////////////////////////////////////////////
// writeSiteArchive
///////////////////////////////////////////////////////////////////////////////
@@ -1741,18 +1746,26 @@ var writeTargetToPath = Profile(
// - builtBy: vanity identification string to write into metadata
// - releaseName: The Meteor release version
// - getRelativeTargetPath: see doc at ServerTarget.write
// - previousBuilder: previous Builder object used in previous iteration
var writeSiteArchive = Profile(
"bundler..writeSiteArchive", function (targets, outputPath, options) {
var builder = new Builder({
outputPath: outputPath
});
"bundler..writeSiteArchive",
function (targets, outputPath, {
includeNodeModules,
builtBy,
releaseName,
getRelativeTargetPath,
previousBuilders
}) {
const builders = {};
const builder = new Builder({outputPath});
try {
var json = {
format: "site-archive-pre1",
builtBy: options.builtBy,
builtBy,
programs: [],
meteorRelease: options.releaseName
meteorRelease: releaseName
};
var nodePath = [];
@@ -1773,28 +1786,29 @@ var writeSiteArchive = Profile(
});
builder.write('README', { data: new Buffer(
"This is a Meteor application bundle. It has only one external dependency:\n" +
"Node.js 0.10.36 or newer. To run the application:\n" +
"\n" +
" $ (cd programs/server && npm install)\n" +
" $ export MONGO_URL='mongodb://user:password@host:port/databasename'\n" +
" $ export ROOT_URL='http://example.com'\n" +
" $ export MAIL_URL='smtp://user:password@mailhost:port/'\n" +
" $ node main.js\n" +
"\n" +
"Use the PORT environment variable to set the port where the\n" +
"application will listen. The default is 80, but that will require\n" +
"root on most systems.\n" +
"\n" +
"Find out more about Meteor at meteor.com.\n",
`This is a Meteor application bundle. It has only one external dependency:
Node.js 0.10.36 or newer. To run the application:
$ (cd programs/server && npm install)
$ export MONGO_URL='mongodb://user:password@host:port/databasename'
$ export ROOT_URL='http://example.com'
$ export MAIL_URL='smtp://user:password@mailhost:port/'
$ node main.js
Use the PORT environment variable to set the port where the
application will listen. The default is 80, but that will require
root on most systems.
Find out more about Meteor at meteor.com.
`,
'utf8')});
}
// Merge the WatchSet of everything that went into the bundle.
var clientWatchSet = new watch.WatchSet();
var serverWatchSet = new watch.WatchSet();
var dependencySources = [builder].concat(_.values(targets));
_.each(dependencySources, function (s) {
const clientWatchSet = new watch.WatchSet();
const serverWatchSet = new watch.WatchSet();
const dependencySources = [builder].concat(_.values(targets));
dependencySources.forEach(s => {
if (s instanceof ClientTarget) {
clientWatchSet.merge(s.getWatchSet());
} else {
@@ -1802,16 +1816,29 @@ var writeSiteArchive = Profile(
}
});
_.each(targets, function (target, name) {
var targetBuild = writeTargetToPath(name, target, builder.buildPath, {
includeNodeModules: options.includeNodeModules,
builtBy: options.builtBy,
releaseName: options.releaseName,
getRelativeTargetPath: options.getRelativeTargetPath
Object.keys(targets).forEach(name => {
const target = targets[name];
const previousBuilder = previousBuilders[name];
const {
arch, path, cordovaDependencies,
nodePath: targetNP,
builder: targetBuilder
} =
writeTargetToPath(name, target, builder.buildPath, {
includeNodeModules,
builtBy,
releaseName,
getRelativeTargetPath,
previousBuilder
});
builders[name] = targetBuilder;
json.programs.push({
name, arch, path, cordovaDependencies
});
json.programs.push(targetBuild);
nodePath = nodePath.concat(targetBuild.nodePath);
nodePath = nodePath.concat(targetNP);
});
// Control file
@@ -1820,11 +1847,19 @@ var writeSiteArchive = Profile(
// We did it!
builder.complete();
// now, go and "fix up" the outputPath properties of the sub-builders
Object.keys(builders).forEach(name => {
const subBuilder = builders[name];
subBuilder.outputPath = builder.outputPath + subBuilder.outputPath.substring(builder.buildPath.length);
console.log("fix up: ", subBuilder.outputPath);
});
return {
clientWatchSet: clientWatchSet,
serverWatchSet: serverWatchSet,
clientWatchSet,
serverWatchSet,
starManifest: json,
nodePath: nodePath
nodePath,
builders
};
} catch (e) {
builder.abort();
@@ -1902,14 +1937,17 @@ var writeSiteArchive = Profile(
* will point somewhere else -- into the app (if any) whose packages
* you are testing!
*/
exports.bundle = function (options) {
var projectContext = options.projectContext;
var outputPath = options.outputPath;
var includeNodeModules = options.includeNodeModules;
var buildOptions = options.buildOptions || {};
buildOptions.minify = buildOptions.minify || 'development';
var shouldLint = options.lint || false;
exports.bundle = function ({
projectContext,
outputPath,
includeNodeModules,
buildOptions,
lint: shouldLint,
previousBuilders,
hasCachedBundle
}) {
buildOptions = buildOptions || {};
shouldLint = shouldLint || false;
var appDir = projectContext.projectDir;
@@ -1929,6 +1967,7 @@ exports.bundle = function (options) {
var targets = {};
var nodePath = [];
var lintingMessages = null;
var builders = {};
if (! release.usingRightReleaseForApp(projectContext))
throw new Error("running wrong release for app?");
@@ -2023,7 +2062,7 @@ exports.bundle = function (options) {
});
// Server
if (! options.hasCachedBundle) {
if (! hasCachedBundle) {
var server = makeServerTarget(app, clientTargets);
targets.server = server;
}
@@ -2049,27 +2088,36 @@ exports.bundle = function (options) {
// Write to disk
var writeOptions = {
includeNodeModules: includeNodeModules,
builtBy: builtBy,
releaseName: releaseName,
getRelativeTargetPath: getRelativeTargetPath
includeNodeModules,
builtBy,
releaseName,
getRelativeTargetPath
};
if (outputPath !== null) {
if (options.hasCachedBundle) {
if (hasCachedBundle) {
// If we already have a cached bundle, just recreate the new targets.
// XXX This might make the contents of "star.json" out of date.
builders = _.clone(previousBuilders);
_.each(targets, function (target, name) {
const previousBuilder = previousBuilders[name];
console.log('previous builder for ', name, previousBuilder.outputPath);
var targetBuild =
writeTargetToPath(name, target, outputPath, writeOptions);
writeTargetToPath(name, target, outputPath, _.extend(writeOptions, {previousBuilder}));
nodePath = nodePath.concat(targetBuild.nodePath);
clientWatchSet.merge(target.getWatchSet());
builders[name] = targetBuild.builder;
});
} else {
starResult = writeSiteArchive(targets, outputPath, writeOptions);
starResult = writeSiteArchive(
targets,
outputPath,
_.extend(writeOptions, {previousBuilders}));
nodePath = nodePath.concat(starResult.nodePath);
serverWatchSet.merge(starResult.serverWatchSet);
clientWatchSet.merge(starResult.clientWatchSet);
builders = starResult.builders;
}
}
@@ -2082,10 +2130,11 @@ exports.bundle = function (options) {
return {
errors: success ? false : messages,
warnings: lintingMessages,
serverWatchSet: serverWatchSet,
clientWatchSet: clientWatchSet,
serverWatchSet,
clientWatchSet,
starManifest: starResult && starResult.starManifest,
nodePath: nodePath
nodePath,
builders
};
};

17
tools/run-app.js
View File

@@ -465,6 +465,9 @@ _.extend(AppRunner.prototype, {
// a single invocation of _runOnce().
var cachedServerWatchSet;
// Builders saved from previous iterations
var builders = {};
var bundleApp = function () {
if (! firstRun) {
// If the build fails in a way that could be fixed by a refresh, allow
@@ -553,14 +556,20 @@ _.extend(AppRunner.prototype, {
includeNodeModules = 'reference-directly';
}
return bundler.bundle({
var bundleResult = bundler.bundle({
projectContext: self.projectContext,
outputPath: bundlePath,
includeNodeModules: includeNodeModules,
buildOptions: self.buildOptions,
hasCachedBundle: !! cachedServerWatchSet,
lint: self.lint
lint: self.lint,
previousBuilders: builders
});
// save new builders with their caches
({builders} = bundleResult);
return bundleResult;
});
// Keep the server watch set from the initial bundle, because subsequent
@@ -763,10 +772,10 @@ _.extend(AppRunner.prototype, {
if (bundleResultOrRunResult.runResult)
return bundleResultOrRunResult.runResult;
bundleResult = bundleResultOrRunResult.bundleResult;
if (bundleResult.warnings) {
if (self.lint && bundleResult.warnings) {
runLog.log(
'Linting your app.\n\n' +
bundleResult.warnings.formatMessages(), { arrow: true })
bundleResult.warnings.formatMessages(), { arrow: true });
}
var oldFuture = self.runFuture = new Future;