github/meteor
1
1
Fork 0
mirror of https://github.com/meteor/meteor.git synced 2026-05-02 03:01:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

New Watcher implementation (for watching for changed files and determining when to rebundle the app.) Unlike the old DependencyWatcher, it uses absolute paths and contains no Meteor-specific knowledge. Also unlike the old DependencyWatcher, it has comprehensive unit tests (scripts/watch-test.sh).

Browse Source
This commit is contained in:
Geoff Schmidt
2013-04-04 03:51:08 -07:00
committed by David Glasser
parent 45217c41f3
commit 24682fd7f4
7 changed files with 802 additions and 331 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/run-tools-tests.sh
View File

@@ -40,6 +40,11 @@ unset METEOR_WAREHOUSE_DIR
###
./bundler-test.sh
###
### Watcher unit tests
###
./watch-test.sh
###
### Test the Meteor CLI from a checkout. We do this last because it is least likely to fail.
###

14
scripts/watch-test.sh Executable file
View File

@@ -0,0 +1,14 @@
#!/bin/bash
# stop on any non-zero return value from test_watch.js, and print "FAILED"
set -e
trap 'echo FAILED' EXIT
METEOR_DIR=$(pwd)/..
# run tests
./node.sh $METEOR_DIR/tools/tests/test_watch.js
# cleanup trap, and print "SUCCESS"
trap - EXIT
echo PASSED

132
tools/bundler.js
View File

@@ -20,16 +20,6 @@
// "size": size in bytes
// "hash": sha1 hash of the contents
// }
// /dependencies.json: files to monitor for changes in development mode
// - extensions [list of extensions registered for user code, with dots]
// - packages [map from package name to list of paths relative to the package]
// - core [paths relative to 'app' in meteor tree]
// - app [paths relative to top of app tree]
// - exclude [list of regexps for files to ignore (everywhere)]
// (for 'core' and 'apps', if a directory is given, you should
// monitor everything in the subtree under it minus the stuff that
// matches exclude, and if it doesn't exist yet, you should watch for
// it to appear)
//
// The application launcher is expected to execute /main.js with node, setting
// various environment variables (such as PORT and MONGO_URL). The enclosed node
@@ -39,7 +29,6 @@ var path = require('path');
var files = require(path.join(__dirname, 'files.js'));
var packages = require(path.join(__dirname, 'packages.js'));
var linker = require(path.join(__dirname, 'linker.js'));
var warehouse = require(path.join(__dirname, 'warehouse.js'));
var crypto = require('crypto');
var fs = require('fs');
var uglify = require('uglify-js');
@@ -437,6 +426,18 @@ _.extend(Bundle.prototype, {
return ret;
},
_addDependency: function (filePath, hash, onlyIfExists) {
var self = this;
filePath = path.resolve(filePath);
if (onlyIfExists && ! fs.existsSync(filePath))
return;
if (! hash)
hash = sha1(fs.readFileSync(filePath));
self.dependencyInfo.files[filePath] = hash;
},
// nodeModulesMode should be "skip", "symlink", or "copy"
// computes self.dependencyInfo
writeToDirectory: function (outputPath, projectDir, nodeModulesMode) {
@@ -444,18 +445,20 @@ _.extend(Bundle.prototype, {
var appJson = {};
var isApp = files.is_app_dir(projectDir);
self.dependencyInfo = {core: [], app: [], packages: {}, hashes: {}};
self.dependencyInfo = {files: {}, directories: {}};
if (isApp) {
self.dependencyInfo.app.push(path.join('.meteor', 'packages'));
self.dependencyInfo.app.push(path.join('.meteor', 'release'));
self._addDependency(path.join(projectDir, '.meteor', 'packages'));
// Not sure why 'release' doesn't exist in a test run, but roll with it
self._addDependency(path.join(projectDir, '.meteor', 'release'), null,
true);
}
// --- Set up build area ---
// foo/bar => foo/.build.bar
var buildPath = path.join(path.dirname(outputPath),
'.build.' + path.basename(outputPath));
'.build.' + path.basename(outputPath));
// XXX cleaner error handling. don't make the humans read an
// exception (and, make suitable for use in automated systems)
@@ -464,10 +467,16 @@ _.extend(Bundle.prototype, {
// --- Core runner code ---
files.cp_r(path.join(__dirname, 'server'),
path.join(buildPath, 'server'), {ignore: ignoreFiles});
// XXX we don't do content-based dependency watching for these files
self.dependencyInfo.core.push('server');
var serverPath = path.join(__dirname, 'server');
var copied = files.cp_r(serverPath, path.join(buildPath, 'server'),
{ignore: ignoreFiles});
_.each(copied, function (relPath) {
self._addDependency(path.join(serverPath, relPath));
});
self.dependencyInfo.directories[serverPath] = {
include: [/.?/],
exclude: ignoreFiles
};
// --- Third party dependencies ---
@@ -508,22 +517,28 @@ _.extend(Bundle.prototype, {
};
if (isApp) {
if (fs.existsSync(path.join(projectDir, 'public'))) {
var copied =
files.cp_r(path.join(projectDir, 'public'),
path.join(buildPath, 'static'), {ignore: ignoreFiles});
var publicDir = path.join(projectDir, 'public');
_.each(copied, function (fsRelativePath) {
var filepath = path.join(buildPath, 'static', fsRelativePath);
if (fs.existsSync(publicDir)) {
var copied =
files.cp_r(publicDir, path.join(buildPath, 'static'),
{ignore: ignoreFiles});
_.each(copied, function (relPath) {
var filepath = path.join(publicDir, relPath);
var contents = fs.readFileSync(filepath);
var hash = sha1(contents);
self.dependencyInfo.hashes[
path.join(projectDir, 'public', fsRelativePath)] = hash;
addClientFileToManifest(fsRelativePath, contents, 'static', false,
self._addDependency(filepath, hash);
addClientFileToManifest(relPath, contents, 'static', false,
undefined, hash);
});
}
self.dependencyInfo.app.push('public');
self.dependencyInfo.directories[publicDir] = {
include: [/.?/],
exclude: ignoreFiles
};
}
// Add cache busting query param if needed, and
@@ -603,7 +618,7 @@ _.extend(Bundle.prototype, {
where: 'internal',
hash: sha1(appHtml)
});
self.dependencyInfo.core.push(path.join('tools', 'app.html.in'));
self._addDependency(path.join(__dirname, 'app.html.in'));
// --- Documentation, and running from the command line ---
@@ -626,30 +641,35 @@ _.extend(Bundle.prototype, {
"\n" +
"Find out more about Meteor at meteor.com.\n");
// --- Source file dependencies ---
_.each(_.values(self.slices), function (slice) {
_.extend(self.dependencyInfo.files, slice.pkg.dependencyFileShas);
});
if (isApp) {
// Include files in the app that match any file extension
// handled by any package that the app uses
self.dependencyInfo.directories[path.resolve(projectDir)] = {
include: _.map(self._appExtensions(), function (ext) {
return new RegExp('\\.' + ext.slice(1) + "$");
}),
exclude: ignoreFiles
};
// Exclude the packages directory in an app, since packages
// explicitly call out the files they use
self.dependencyInfo.directories[path.resolve(projectDir, 'packages')] = {
exclude: [/.?/]
};
// Exclude .meteor/local and everything under it
self.dependencyInfo.directories[
path.resolve(projectDir, '.meteor', 'local')] = { exclude: [/.?/] };
}
// --- Metadata ---
appJson.manifest = self.manifest;
self.dependencyInfo.extensions = self._appExtensions();
self.dependencyInfo.exclude = _.pluck(ignoreFiles, 'source');
self.dependencyInfo.packages = {};
_.each(_.values(self.slices), function (slice) {
// Data for the mtime dependency watcher. We only record data here for
// packages, not apps, since apps watch the whole directory for added
// files.
if (slice.pkg.name) {
self.dependencyInfo.packages[slice.pkg.name] = _.union(
self.dependencyInfo.packages[slice.pkg.name] || [],
_.keys(slice.pkg.dependencyFileShas)
);
}
// Data for the contents dependency watcher check.
_.each(slice.pkg.dependencyFileShas, function (sha, relPath) {
self.dependencyInfo.hashes[
path.join(slice.pkg.sourceRoot, relPath)] = sha;
});
});
if (self.releaseStamp && self.releaseStamp !== 'none')
appJson.release = self.releaseStamp;
@@ -680,16 +700,8 @@ _.extend(Bundle.prototype, {
* - errors: An array of strings, or falsy if bundling succeeded.
* - dependencyInfo: Information about files and paths that were
* inputs into the bundle and that we may wish to monitor for
* changes if we are developing interactively.
* - extensions: list of extensions registered for user code, with dots]
* - packages: map from package name to list of paths relative to the package
* - core: paths relative to 'app' in meteor tree
* - app: paths relative to top of app tree
* - exclude: list of regexps for files to ignore (everywhere)
* (for 'core' and 'apps', if a directory is given, you should
* monitor everything in the subtree under it minus the stuff that
* matches exclude, and if it doesn't exist yet, you should watch
* for it to appear)
* changes when developing interactively. It has two keys, 'files'
* and 'directories', in the format expected by watch.Watcher.
*
* On failure ('errors' is truthy), no bundle will be output (in fact,
* outputPath will have been removed if it existed.)

24
tools/packages.js
View File

@@ -188,6 +188,18 @@ var Package = function (library) {
};
_.extend(Package.prototype, {
// Add a dependency (in the sense of dependencyFileShas) on a
// file. If hash is supplied it should be the sha1 of the file
// contents. If omitted it will be computed. relPath will be
// resolved to an absolute path (relative to self.sourceRoot.)
_addDependency: function (relPath, contents) {
var self = this;
var absPath = path.resolve(self.sourceRoot, relPath);
if (! contents)
contents = fs.readFileSync(absPath);
self.dependencyFileShas[absPath] = bundler.sha1(contents);
},
// loads a package's package.js file into memory, using
// runInThisContext. Wraps the contents of package.js in a closure,
// supplying pseudo-globals 'Package' and 'Npm'.
@@ -218,8 +230,7 @@ _.extend(Package.prototype, {
// steer clear
var func = require('vm').runInThisContext(wrapped, fullpath, true);
func(self.packageFacade, self.npmFacade);
self.dependencyFileShas['package.js'] = bundler.sha1(code);
self._addDependency("package.js", code);
// source files used
var sources = {use: {client: [], server: []},
@@ -402,9 +413,7 @@ _.extend(Package.prototype, {
'spark', 'templating', 'startup', 'past'];
packages = _.union(packages, project.get_packages(appDir));
// XXX this read has a race with the actual read that is used
var packagesFile = path.join(appDir, '.meteor', 'packages');
self.dependencyFileShas[path.join('.meteor', 'packages')] =
bundler.sha1(fs.readFileSync(packagesFile));
self._addDependency(path.join(appDir, '.meteor', 'packages'));
_.each(["use", "test"], function (role) {
_.each(["client", "server"], function (where) {
@@ -502,10 +511,11 @@ _.extend(Package.prototype, {
};
_.each(self.sources[role][where], function (relPath) {
var absPath = path.resolve(self.sourceRoot, relPath);
var ext = path.extname(relPath).substr(1);
var handler = self._getSourceHandler(role, where, ext);
var contents = fs.readFileSync(path.join(self.sourceRoot, relPath));
self.dependencyFileShas[relPath] = bundler.sha1(contents);
var contents = fs.readFileSync(absPath);
self._addDependency(absPath, contents);
if (! handler) {
// If we don't have an extension handler, serve this file

328
tools/run.js
View File

@@ -8,6 +8,7 @@ var httpProxy = require('http-proxy');
var files = require('./files.js');
var library = require('./library.js');
var watch = require('./watch.js');
var project = require('./project.js');
var updater = require('./updater.js');
var bundler = require('./bundler.js');
@@ -218,7 +219,7 @@ var logToClients = function (msg) {
// onExit
// [onListen]
// [nodeOptions]
// [settingsFile]
// [settings]
var startServer = function (options) {
// environment
@@ -233,12 +234,10 @@ var startServer = function (options) {
env.PORT = options.innerPort;
env.MONGO_URL = options.mongoUrl;
env.ROOT_URL = env.ROOT_URL || ('http://localhost:' + options.outerPort);
if (options.settingsFile) {
// Re-read the settings file each time we call startServer.
var settings = exports.getSettings(options.settingsFile);
if (settings)
env.METEOR_SETTINGS = settings;
}
if (options.settings)
env.METEOR_SETTINGS = options.settings;
else
delete env.METEOR_SETTINGS;
var nodeOptions = _.clone(options.nodeOptions);
nodeOptions.push(path.join(options.bundlePath, 'main.js'));
@@ -311,243 +310,6 @@ var killServer = function (handle) {
clearInterval(handle.timer);
};
////////// Watching dependencies //////////
// deps is the data from dependencies.json in the bundle
// appDir is the root of the app
// relativeFiles are any other files to watch, relative to the current
// directory (eg, the --settings file)
// onChange is only fired once
var DependencyWatcher = function (
deps, appDir, relativeFiles, library, onChange) {
var self = this;
self.appDir = appDir;
self.onChange = onChange;
self.watches = {}; // path => unwatch function with no arguments
self.lastContents = {}; // path => last contents (array of filenames)
self.mtimes = {}; // path => last seen mtime
// If a file is under a sourceDir, and has one of the
// sourceExtensions, then it's interesting.
self.sourceDirs = [self.appDir];
self.sourceExtensions = deps.extensions || [];
// Any file under a bulkDir is interesting. (bulkDirs may also
// contain individual files)
self.bulkDirs = [];
// If we're running from a git checkout, we reload when "core" files like
// server.js change.
if (!files.usesWarehouse()) {
_.each(deps.core || [], function (filepath) {
self.bulkDirs.push(path.join(files.getCurrentToolsDir(), filepath));
});
}
_.each(deps.app || [], function (filepath) {
self.bulkDirs.push(path.join(self.appDir, filepath));
});
// Additional list of specific files that are interesting.
self.specificFiles = {};
for (var pkg in (deps.packages || {})) {
// We only watch for changes in local packages, rather than ones in the
// warehouse, since only changes to local ones need to cause an app to
// reload. Notably, the app will *not* reload the first time a local package
// is created which overrides an installed package.
var localPackageDir = library.directoryForLocalPackage(pkg);
if (localPackageDir) {
_.each(deps.packages[pkg], function (file) {
self.specificFiles[path.join(localPackageDir, file)] = true;
});
}
};
_.each(relativeFiles, function (file) {
self.specificFiles[file] = true;
});
// Things that are never interesting.
self.excludePatterns = _.map((deps.exclude || []), function (pattern) {
return new RegExp(pattern);
});
self.excludePaths = [
path.join(appDir, '.meteor', 'local'),
// For app packages, we only watch files explicitly used by the package (in
// specificFiles)
path.join(appDir, 'packages')
];
// Start monitoring
_.each(_.union(self.sourceDirs, self.bulkDirs, _.keys(self.specificFiles)),
_.bind(self._scan, self, true));
// mtime scans are great and relatively efficient, but they have a couple of
// issues. One is that they only detect changes in mtimes from the start of
// dependency watching, not from the actual bundled file, so if bundling is
// slow and somebody edits a file after it's used by the bundler but before
// the DependencyWatcher is created, we'll miss it. An even worse problem is
// that on OSX HFS+, mtime resolution is only one second, so if a file is
// written twice in a second the bundler might get the first version and never
// notice the second change! So, in a second, we'll do a one-time scan to
// check the hash of each file against what the bundler told us we should see.
//
// This will still miss files that are newly added during bundling, and there
// are also race conditions where the bundler may calculate some hashes via a
// separate read than the read that actually was used in bundling... but it's
// close.
setTimeout(function() {
_.each(deps.hashes, function (hash, filepath) {
fs.readFile(filepath, function (error, contents) {
// Fire if the file was deleted or changed contents.
if (error || bundler.sha1(contents) !== hash)
self._fire();
});
});
}, 1000);
};
_.extend(DependencyWatcher.prototype, {
// stop monitoring
destroy: function () {
var self = this;
self.onChange = null;
for (var filepath in self.watches)
self.watches[filepath](); // unwatch
self.watches = {};
},
_fire: function () {
var self = this;
if (self.onChange) {
var f = self.onChange;
self.onChange = null;
f();
self.destroy();
}
},
// initial is true on the inital scan, to suppress notifications
_scan: function (initial, filepath) {
var self = this;
if (self._isExcluded(filepath))
return false;
try {
var stats = fs.lstatSync(filepath);
} catch (e) {
// doesn't exist -- leave stats undefined
}
// '+' is necessary to coerce the mtimes from date objects to ints
// (unix times) so they can be conveniently tested for equality
if (stats && +stats.mtime === +self.mtimes[filepath])
// We already know about this file and it hasn't actually
// changed. Probably its atime changed.
return;
// If an interesting file has changed, fire!
var isInteresting = self._isInteresting(filepath);
if (!initial && isInteresting) {
self._fire();
return;
}
if (!stats) {
// A directory (or an uninteresting file) was removed
var unwatch = self.watches[filepath];
unwatch && unwatch();
delete self.watches[filepath];
delete self.lastContents[filepath];
delete self.mtimes[filepath];
return;
}
// If we're seeing this file or directory for the first time,
// monitor it if necessary
if (!(filepath in self.watches) &&
(isInteresting || stats.isDirectory())) {
if (!stats.isDirectory()) {
// Intentionally not using fs.watch since it doesn't play well with
// vim (https://github.com/joyent/node/issues/3172)
fs.watchFile(filepath, {interval: 500}, // poll a lot!
_.bind(self._scan, self, false, filepath));
self.watches[filepath] = function() { fs.unwatchFile(filepath); };
} else {
// fs.watchFile doesn't work for directories (as tested on ubuntu)
var watch = fs.watch(filepath, {interval: 500}, // poll a lot!
_.bind(self._scan, self, false, filepath));
self.watches[filepath] = function() { watch.close(); };
}
self.mtimes[filepath] = stats.mtime;
}
// If a directory, recurse into any new files it contains. (We
// don't need to check for removed files here, since if we care
// about a file, we'll already be monitoring it)
if (stats.isDirectory()) {
var oldContents = self.lastContents[filepath] || [];
var newContents = fs.readdirSync(filepath);
var added = _.difference(newContents, oldContents);
self.lastContents[filepath] = newContents;
_.each(added, function (child) {
self._scan(initial, path.join(filepath, child));
});
}
},
// Should we even bother to scan/recurse into this file?
_isExcluded: function (filepath) {
var self = this;
// Files we're specifically being asked to scan are never excluded. For
// example, files from app packages (that are actually pulled in by their
// package.js) are not excluded, but the app packages directory itself is
// (so that other files in package directories aren't watched).
if (filepath in self.specificFiles)
return false;
if (_.indexOf(self.excludePaths, filepath) !== -1)
return true;
var excludedByPattern = _.any(self.excludePatterns, function (regexp) {
return path.basename(filepath).match(regexp);
});
return excludedByPattern;
},
// Should we fire if this file changes?
_isInteresting: function (filepath) {
var self = this;
if (self._isExcluded(filepath))
return false;
var inAnyDir = function (dirs) {
return _.any(dirs, function (dir) {
return filepath.slice(0, dir.length) === dir;
});
};
// Specific, individual files that we want to monitor
if (filepath in self.specificFiles)
return true;
// Source files
if (inAnyDir(self.sourceDirs) &&
files.findExtension(self.sourceExtensions, filepath))
return true;
// Other directories and files that are included
if (inAnyDir(self.bulkDirs))
return true;
return false;
}
});
///////////////////////////////////////////////////////////////////////////////
// Also used by "meteor deploy" in meteor.js.
@@ -576,7 +338,7 @@ exports.getSettings = function (filename) {
// This function never returns and will call process.exit() if it
// can't continue. If you change this, remember to call
// watcher.destroy() as appropriate.
// watcher.stop() as appropriate.
//
// context is as created in meteor.js.
// options include: port, minify, once, settingsFile, testPackages
@@ -638,22 +400,18 @@ exports.run = function (context, options) {
return;
if (watcher)
watcher.destroy();
watcher.stop();
var relativeFiles;
if (options.settingsFile) {
relativeFiles = [options.settingsFile];
}
var onChange = function () {
if (Status.crashing)
logToClients({'system': "=> Modified -- restarting."});
Status.reset();
restartServer();
};
watcher = new DependencyWatcher(dependencyInfo, context.appDir,
relativeFiles, context.library, onChange);
watcher = new watch.Watcher({
files: dependencyInfo.files,
directories: dependencyInfo.directories,
onChange: function () {
if (Status.crashing)
logToClients({'system': "=> Modified -- restarting."});
Status.reset();
restartServer();
}
});
};
// Using `inFiber` since bundling can yield when loading a manifest
@@ -690,9 +448,9 @@ exports.run = function (context, options) {
// Make the library reload packages, in case they've changed
context.library.flush();
// Bundle up the app
var bundleResult = bundler.bundle(context.appDir, bundlePath, bundleOpts);
startWatching(bundleResult.dependencyInfo);
var dependencyInfo = bundleResult.dependencyInfo;
if (bundleResult.errors) {
logToClients({stdout: "=> Errors prevented startup:\n"});
_.each(bundleResult.errors, function (e) {
@@ -703,10 +461,52 @@ exports.run = function (context, options) {
return;
}
// Read the settings file, if any
var settings = null;
if (options.settingsFile) {
settings = exports.getSettings(options.settingsFile);
// 'getSettings' will collapse any amount of whitespace down to
// the empty string, so to get the sha1 for change monitoring,
// we need to reread the file, which creates a tiny race
// condition (not a big enough deal to care about right now.)
var settingsHash =
bundler.sha1(fs.readFileSync(options.settingsFile, "utf8"));
// Reload if the setting file changes
dependencyInfo.files[path.resolve(options.settingsFile)] =
settingsHash;
}
// If using a warehouse, don't do dependency monitoring on any of
// the files that are in the warehouse. You should not be editing
// those files directly.
if (files.usesWarehouse()) {
var warehouseDir = path.resolve(warehouse.getWarehouseDir());
var filterKeys = function (obj) {
_.each(_.keys(obj), function (k) {
k = path.resolve(k);
if (warehouseDir.length <= k.length &&
k.substr(0, warehouseDir.length) === warehouseDir)
delete obj[k];
});
};
filterKeys(dependencyInfo.files);
filterKeys(dependencyInfo.directories);
}
// Start watching for changes for files. There's no hurry to call
// this, since dependencyInfo contains a snapshot of the state of
// the world at the time of bundling, in the form of hashes and
// lists of matching files in each directory.
startWatching(dependencyInfo);
// Start the server
Status.running = true;
if (firstRun) {
process.stdout.write("=> Meteor server running on: http://localhost:" + outerPort + "/\n");
process.stdout.write("=> Meteor server running on: http://localhost:" +
outerPort + "/\n");
firstRun = false;
lastThingThatPrintedWasRestartMessage = false;
} else {
@@ -745,7 +545,7 @@ exports.run = function (context, options) {
requestQueue = [];
},
nodeOptions: getNodeOptionsFromEnvironment(),
settingsFile: options.settingsFile
settings: settings
});
});

322
tools/tests/test_watch.js Normal file
View File

@@ -0,0 +1,322 @@
var path = require('path');
var fs = require('fs');
var _ = require('underscore');
var assert = require('assert');
var crypto = require('crypto');
var Fiber = require('fibers');
var Future = require('fibers/future');
var watch = require(path.join(__dirname, '..', 'watch.js'));
var files = require(path.join(__dirname, '..', 'files.js'));
var tmp = files.mkdtemp('test_watch');
var serial = 0;
var touchFile = function (filePath, contents) {
filePath = path.join(tmp, filePath);
files.mkdir_p(path.dirname(filePath));
fs.writeFileSync(filePath, contents || ('' + serial));
serial++;
};
var touchDir = function (dirPath) {
dirPath = path.join(tmp, dirPath);
files.mkdir_p(dirPath);
};
var remove = function (fileOrDirPath) {
fileOrDirPath = path.join(tmp, fileOrDirPath);
files.rm_recursive(fileOrDirPath);
};
var theWatcher;
var fired;
var firedFuture;
var lastOptions;
var go = function (options) {
options = options || lastOptions;
lastOptions = options;
if (theWatcher) {
theWatcher.stop();
theWatcher = null;
}
fired = false;
var files = {};
_.each(options.files, function (value, file) {
file = path.join(tmp, file);
if (typeof value !== "string") {
if (fs.existsSync(file)) {
var hash = crypto.createHash('sha1');
hash.update(fs.readFileSync(file));
value = hash.digest('hex');
} else {
value = 'dummyhash';
}
}
files[file] = value;
});
var directories = {};
_.each(options.directories, function (options, dir) {
dir = path.join(tmp, dir);
directories[dir] = options;
});
theWatcher = new watch.Watcher({
files: files,
directories: directories,
onChange: function () {
fired = true;
if (firedFuture)
firedFuture.return(true);
}
});
}
var fires = function (waitDuration) {
if (! theWatcher)
throw new Error("No watcher?");
if (fired)
return true;
// Wait up to a second for it to fire
var timeout = setTimeout(function () {
firedFuture.return(false);
}, waitDuration || 1000);
if (firedFuture)
throw new Error("Already have a future");
firedFuture = new Future;
var ret = firedFuture.wait();
clearTimeout(timeout);
firedFuture = null;
return ret;
};
var waitForTopOfSecond = function () {
while (true) {
var msPastSecond = +(new Date) % 1000;
if (msPastSecond < 100) {
return;
}
var f = new Future;
setTimeout(function () {
f.return();
}, 25);
f.wait();
}
};
var delay = function (duration) {
var f = new Future;
setTimeout(function () {
f.return();
}, duration);
f.wait();
};
Fiber(function () {
console.log("Test Watcher");
console.log("... one file");
touchFile('/aa/b', 'kitten');
go({
files: { '/aa/b': true }
});
assert(!fires());
touchFile('/aa/b', 'kitten');
assert(!fires());
touchFile('/aa/b', 'puppy');
assert(fires());
go();
touchFile('/aa/b', 'puppy');
assert(!fires());
touchFile('/aa/b', 'kitten');
assert(fires());
go();
remove('/aa/b');
assert(fires());
touchFile('/aa/b');
go({
files: { '/aa/b': true, '/aa/c': true }
});
assert(fires()); // look like /aa/c was removed
console.log("... directories");
go({
files: {'/aa/b': true },
directories: {'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
}}
});
touchFile('/aa/c');
assert(!fires());
touchFile('/aa/maybe-not');
assert(!fires());
touchFile('/aa/never-yes');
assert(!fires());
touchFile('/aa/never');
assert(!fires());
touchFile('/aa/yes-for-sure');
assert(fires());
go();
touchFile('/aa/nope');
assert(fires()); // because yes-for-sure isn't in the file list
remove('/aa/yes-for-sure');
go();
assert(!fires());
touchFile('/aa/maybe-this-time');
assert(fires());
go();
assert(fires()); // maybe-this-time is still there
go({
files: {'/aa/b': true, '/aa/maybe-this-time': true },
directories: {'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
}}
});
go();
assert(!fires()); // maybe-this-time is now in the expected file list
touchFile('/aa/maybe-yes');
assert(fires());
remove('/aa/maybe-yes');
remove('/aa/maybe-this-time');
go();
assert(fires()); // maybe-this-time is missing
console.log("... recursive directories");
touchFile('/aa/b');
go({
files: {'/aa/b': true },
directories: {'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
}}
});
touchDir('/aa/yess');
assert(!fires());
remove('/aa/yess');
assert(!fires());
touchFile('/aa/yess/kitten');
assert(!fires());
touchFile('/aa/yess/maybe');
assert(fires());
remove('/aa/yess');
go();
touchFile('/aa/whatever/kitten');
assert(!fires());
touchFile('/aa/whatever/maybe');
assert(fires());
remove('/aa/whatever');
go();
touchDir('/aa/i/love/subdirectories');
assert(!fires());
touchFile('/aa/i/love/subdirectories/yessir');
assert(fires());
remove('/aa/i/love/subdirectories/yessir');
go();
touchFile('/aa/i/love/subdirectories/every/day');
assert(!fires());
remove('/aa/i/love/subdirectories');
assert(!fires());
touchFile('/aa/i/love/not/nothing/yes');
assert(!fires());
touchFile('/aa/i/love/not/nothing/maybe/yes');
assert(!fires());
touchFile('/aa/i/love/maybe');
assert(fires());
remove('/aa/i');
remove('/aa/whatever');
remove('/aa');
touchFile('/aa/b');
console.log("... nested directories");
go({
files: {'/aa/b': true },
directories: {
'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
},
'/aa/x': {
include: [/kitten/],
exclude: [/puppy/]
}
}
});
touchFile('/aa/kitten');
assert(!fires());
touchFile('/aa/maybe.puppy');
assert(fires());
remove('/aa/maybe.puppy');
go();
touchFile('/aa/x/kitten');
assert(fires());
remove('/aa/x/kitten');
go();
touchFile('/aa/x/yes');
assert(!fires());
touchFile('/aa/x/kitten.not');
assert(fires());
remove('/aa');
console.log("... rapid changes to file");
touchFile('/aa/x');
waitForTopOfSecond();
go({
files: {'/aa/x': true }});
touchFile('/aa/x');
assert(fires(2000));
go({
directories: {
'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
}
}
});
waitForTopOfSecond();
touchFile('/aa/thing1/whatever');
delay(100);
touchFile('/aa/thing2/yes');
assert(fires(2000));
remove('/aa');
console.log("... rapid changes to directory");
touchDir('/aa');
waitForTopOfSecond();
go({
directories: {'/aa': {
include: [/yes/, /maybe/, /aa/],
exclude: [/not/, /never/]
}}
});
touchFile('/aa/x/yes');
assert(fires(2000));
remove('/aa/x');
waitForTopOfSecond();
go();
delay(600);
touchFile('/aa/x/not');
delay(600);
touchFile('/aa/x/yes');
assert(fires(2000));
remove('/aa/x');
touchDir('/aa/x');
go();
delay(2000);
waitForTopOfSecond();
touchFile('/aa/x/no');
delay(600);
touchFile('/aa/x/yes');
assert(fires(2000));
console.log("Watcher test passed");
theWatcher.stop();
}).run();

308
tools/watch.js Normal file
View File

@@ -0,0 +1,308 @@
var fs = require("fs");
var path = require("path");
var crypto = require('crypto');
var _ = require('underscore');
// Watch for changes to a set of files, and the first time that any of
// the files change, call a user-provided callback. (If you want a
// second callback, you'll need to create a second Watcher.)
//
// You can set up two kinds of watches, file and directory watches.
//
// In a file watch, you provide an absolute path to a file and a SHA1
// (encoded as hex) of the contents of that file. If the file ever
// changes so that its contents no longer match that SHA1, the
// callback triggers.
//
// In a directory watch, you provide an absolute path to a directory
// and two lists of regular expressions specifying the files to
// include or exclude. If there is ever a file in the directory or its
// children that matches the criteria set up by the regular
// expressions, but that IS NOT present as a file watch, then the
// callback triggers.
//
// For directory watches, the regular expressions work as follows. You
// provide two arrays of regular expressions, an include list and an
// exclude list. A file in the directory matches if it matches at
// least one regular expression in the include list, and doesn't match
// any regular expressions in the exclude list. Subdirectories are
// included recursively, as long as their names do not match any
// regular expression in the exclude list.
//
// When multiple directory watches are set up, say on a directory A
// and its subdirectory B, the most specific watch takes precedence in
// each directory. So only B's include/exclude lists will be checked
// in B.
//
// Regular expressions are checked only against individual path
// components (the actual name of the file or the subdirectory), not
// against the entire path.
//
// You can call stop() to stop watching and tear down the
// watcher. Calling stop() guarantees that you will not receive a
// callback (if you have not already.) Calling stop() is unnecessary
// if you've received a callback.
//
// A limitation of the current implementation is that if you set up a
// directory watch on a directory A, and A does not exist at the time
// the Watcher is created but is then created later, then A will not
// be monitored. (Of course, this limitation only applies to the roots
// of the directory watches. If A exists at the time the watch is
// created, and a subdirectory B is later created, it will be properly
// detected. Likewise if A exists and is then deleted it will be
// detected.)
//
// Options may include
// - files: see self.files comment below
// - directories: see self.directories comment below
// - onChange: the function to call when the first change is detected
//
var Watcher = function (options) {
var self = this;
// Map from the absolute path to a file, to a sha1 hash. Fire when
// the file changes from that sha.
self.files = options.files || {};
// Map from an absolute path to a directory, to an object with keys
// 'include' and 'exclude', both lists of regular expressions. Fire
// when a file is added to that directory whose name matches at
// least one regular expression in 'include' and no regular
// expressions in 'exclude'. Subdirectories are included
// recursively, but not subdirectories that match 'exclude'. The
// most specific rule wins, so you can change the parameters in
// effect in subdirectories simply by specifying additional rules.
self.directories = options.directories || {};
// Function to call when a change is detected according to one of
// the above.
self.onChange = options.onChange;
if (! self.onChange)
throw new Error("onChange option is required");
// self.directories in a different form. It's an array of objects,
// each with keys 'dir', 'include', 'options', where path is
// guaranteed to not contain a trailing slash (unless it is the root
// directory) and the objects are sorted from longest path to
// shortest (that is, most specific rule to least specific.)
self.rules = _.map(self.directories, function (options, dir) {
return {
dir: path.resolve(dir),
include: options.include || [],
exclude: options.exclude || []
};
});
self.rules = self.rules.sort(function (a, b) {
return a.dir.length < b.dir.length ? 1 : -1;
});
self.stopped = false;
self.fileWatches = []; // array of paths
self.directoryWatches = {}; // map from path to watch object
self._startFileWatches();
_.each(self.rules, function (rule) {
self._watchDirectory(rule.dir);
});
};
_.extend(Watcher.prototype, {
_checkFileChanged: function (absPath) {
var self = this;
if (! fs.existsSync(absPath))
return true;
var hasher = crypto.createHash('sha1');
hasher.update(fs.readFileSync(absPath));
var hash = hasher.digest('hex');
return (self.files[absPath] !== hash);
},
_startFileWatches: function () {
var self = this;
// Set up a watch for each file
_.each(self.files, function (hash, absPath) {
// Intentionally not using fs.watch since it doesn't play well with
// vim (https://github.com/joyent/node/issues/3172)
// Note that we poll very frequently (500 ms)
fs.watchFile(absPath, {interval: 500}, function () {
// Fire only if the contents of the file actually changed (eg,
// maybe just its atime changed)
if (self._checkFileChanged(absPath))
self._fire();
});
self.fileWatches.push(absPath);
// Check for the case where by the time we created the watch,
// the file had already changed from the sha we were provided.
if (self._checkFileChanged(absPath))
self._fire();
});
// One second later, check the files again, because fs.watchFile
// is actually implemented by polling the file's mtime, and some
// filesystems (OSX HFS+) only keep mtimes to a resolution of one
// second. This handles the case where we check the hash and set
// up the watch, but then the file change before the clock rolls
// over to the next second, and fs.watchFile doesn't notice and
// doesn't call us back. #WorkAroundLowPrecisionMtimes
setTimeout(function () {
_.each(self.files, function (hash, absPath) {
if (self._checkFileChanged(absPath))
self._fire();
});
}, 1000);
},
// Pass true for `include` to include everything (and process only
// excludes)
_matches: function (filename, include, exclude) {
var self = this;
if (include === true)
include = [/.?/];
for (var i = 0; i < include.length; i++)
if (include[i].test(filename))
break;
if (i === include.length) {
return false; // didn't match any includes
}
for (var i = 0; i < exclude.length; i++) {
if (exclude[i].test(filename)) {
return false; // matched an exclude
}
}
// Matched an include and didn't match any excludes
return true;
},
_watchDirectory: function (absPath) {
var self = this;
if (absPath in self.directoryWatches)
// Already being taken care of
return;
// Determine the options that apply to this directory by finding
// the most specific rule.
absPath = path.resolve(absPath); // ensure no trailing slash
for (var i = 0; i < self.rules.length; i++) {
var rule = self.rules[i];
if (absPath.length >= rule.dir.length &&
absPath.substr(0, rule.dir.length) === rule.dir)
break; // found a match
rule = null;
}
if (! rule)
// Huh, doesn't appear that we're supposed to be watching this
// directory.
return;
var contents = [];
var scanDirectory = function (isDoubleCheck) {
if (self.stopped)
return;
if (! fs.existsSync(absPath)) {
// Directory was removed. Stop watching.
var watch = self.directoryWatches[absPath];
watch && watch.close();
delete self.directoryWatches[absPath];
return;
}
// Find previously unknown files and subdirectories. (We don't
// care about removed subdirectories because the logic
// immediately above handles them, and we don't care about
// removed files because the ones we care about will already
// have file watches on them.)
var newContents = fs.readdirSync(absPath);
var added = _.difference(newContents, contents);
contents = newContents;
// Look at each newly added item
_.each(added, function (addedItem) {
var addedPath = path.join(absPath, addedItem);
// Is it a directory?
try {
var stats = fs.lstatSync(addedPath);
} catch (e) {
// Can't be found? That's weird. Ignore.
return;
}
var isDirectory = stats.isDirectory();
// Does it match the rule?
if (! self._matches(addedItem,
isDirectory ? true : rule.include,
rule.exclude))
return; // No
if (! isDirectory) {
if (! (addedPath in self.files))
// Found a newly added file that we care about.
self._fire();
} else {
// Found a subdirectory that we care to monitor.
self._watchDirectory(addedPath);
}
});
if (! isDoubleCheck) {
// Whenever a directory changes, scan it soon as we notice,
// but then scan it again one secord later just to make sure
// that we haven't missed any changes. See commentary at
// #WorkAroundLowPrecisionMtimes
setTimeout(function () {
scanDirectory(true);
}, 1000);
}
};
// fs.watchFile doesn't work for directories (as tested on ubuntu)
// Notice that we poll very frequently (500 ms)
try {
self.directoryWatches[absPath] =
fs.watch(absPath, {interval: 500}, scanDirectory);
scanDirectory();
} catch (e) {
// Can happen if the directory doesn't exist, say because a
// nonexistent path was included in self.directories
}
},
_fire: function () {
var self = this;
if (self.stopped)
return;
self.stop();
self.onChange();
},
stop: function () {
var self = this;
self.stopped = true;
// Clean up file watches
_.each(self.fileWatches, function (absPath) {
fs.unwatchFile(absPath);
});
self.fileWatches = [];
// Clean up directory watches
_.each(self.directoryWatches, function (watch) {
watch.close();
});
self.directoryWatches = {};
}
});
exports.Watcher = Watcher;