From 24682fd7f44fd48c3a0744fa2ee2deded038901d Mon Sep 17 00:00:00 2001 From: Geoff Schmidt Date: Thu, 4 Apr 2013 03:51:08 -0700 Subject: [PATCH] New Watcher implementation (for watching for changed files and determining when to rebundle the app.) Unlike the old DependencyWatcher, it uses absolute paths and contains no Meteor-specific knowledge. Also unlike the old DependencyWatcher, it has comprehensive unit tests (scripts/watch-test.sh). --- scripts/run-tools-tests.sh | 5 + scripts/watch-test.sh | 14 ++ tools/bundler.js | 132 ++++++++------- tools/packages.js | 24 ++- tools/run.js | 328 ++++++++----------------------------- tools/tests/test_watch.js | 322 ++++++++++++++++++++++++++++++++++++ tools/watch.js | 308 ++++++++++++++++++++++++++++++++++ 7 files changed, 802 insertions(+), 331 deletions(-) create mode 100755 scripts/watch-test.sh create mode 100644 tools/tests/test_watch.js create mode 100644 tools/watch.js diff --git a/scripts/run-tools-tests.sh b/scripts/run-tools-tests.sh index d6e002d500..d7fe24604f 100755 --- a/scripts/run-tools-tests.sh +++ b/scripts/run-tools-tests.sh @@ -40,6 +40,11 @@ unset METEOR_WAREHOUSE_DIR ### ./bundler-test.sh +### +### Watcher unit tests +### +./watch-test.sh + ### ### Test the Meteor CLI from a checkout. We do this last because it is least likely to fail. ### diff --git a/scripts/watch-test.sh b/scripts/watch-test.sh new file mode 100755 index 0000000000..ee9074641e --- /dev/null +++ b/scripts/watch-test.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# stop on any non-zero return value from test_watch.js, and print "FAILED" +set -e +trap 'echo FAILED' EXIT + +METEOR_DIR=$(pwd)/.. + +# run tests +./node.sh $METEOR_DIR/tools/tests/test_watch.js + +# cleanup trap, and print "SUCCESS" +trap - EXIT +echo PASSED diff --git a/tools/bundler.js b/tools/bundler.js index b0ff4d743a..8d175cfc5f 100644 --- a/tools/bundler.js +++ b/tools/bundler.js @@ -20,16 +20,6 @@ // "size": size in bytes // "hash": sha1 hash of the contents // } -// /dependencies.json: files to monitor for changes in development mode -// - extensions [list of extensions registered for user code, with dots] -// - packages [map from package name to list of paths relative to the package] -// - core [paths relative to 'app' in meteor tree] -// - app [paths relative to top of app tree] -// - exclude [list of regexps for files to ignore (everywhere)] -// (for 'core' and 'apps', if a directory is given, you should -// monitor everything in the subtree under it minus the stuff that -// matches exclude, and if it doesn't exist yet, you should watch for -// it to appear) // // The application launcher is expected to execute /main.js with node, setting // various environment variables (such as PORT and MONGO_URL). The enclosed node @@ -39,7 +29,6 @@ var path = require('path'); var files = require(path.join(__dirname, 'files.js')); var packages = require(path.join(__dirname, 'packages.js')); var linker = require(path.join(__dirname, 'linker.js')); -var warehouse = require(path.join(__dirname, 'warehouse.js')); var crypto = require('crypto'); var fs = require('fs'); var uglify = require('uglify-js'); @@ -437,6 +426,18 @@ _.extend(Bundle.prototype, { return ret; }, + _addDependency: function (filePath, hash, onlyIfExists) { + var self = this; + filePath = path.resolve(filePath); + + if (onlyIfExists && ! fs.existsSync(filePath)) + return; + + if (! hash) + hash = sha1(fs.readFileSync(filePath)); + self.dependencyInfo.files[filePath] = hash; + }, + // nodeModulesMode should be "skip", "symlink", or "copy" // computes self.dependencyInfo writeToDirectory: function (outputPath, projectDir, nodeModulesMode) { @@ -444,18 +445,20 @@ _.extend(Bundle.prototype, { var appJson = {}; var isApp = files.is_app_dir(projectDir); - self.dependencyInfo = {core: [], app: [], packages: {}, hashes: {}}; + self.dependencyInfo = {files: {}, directories: {}}; if (isApp) { - self.dependencyInfo.app.push(path.join('.meteor', 'packages')); - self.dependencyInfo.app.push(path.join('.meteor', 'release')); + self._addDependency(path.join(projectDir, '.meteor', 'packages')); + // Not sure why 'release' doesn't exist in a test run, but roll with it + self._addDependency(path.join(projectDir, '.meteor', 'release'), null, + true); } // --- Set up build area --- // foo/bar => foo/.build.bar var buildPath = path.join(path.dirname(outputPath), - '.build.' + path.basename(outputPath)); + '.build.' + path.basename(outputPath)); // XXX cleaner error handling. don't make the humans read an // exception (and, make suitable for use in automated systems) @@ -464,10 +467,16 @@ _.extend(Bundle.prototype, { // --- Core runner code --- - files.cp_r(path.join(__dirname, 'server'), - path.join(buildPath, 'server'), {ignore: ignoreFiles}); - // XXX we don't do content-based dependency watching for these files - self.dependencyInfo.core.push('server'); + var serverPath = path.join(__dirname, 'server'); + var copied = files.cp_r(serverPath, path.join(buildPath, 'server'), + {ignore: ignoreFiles}); + _.each(copied, function (relPath) { + self._addDependency(path.join(serverPath, relPath)); + }); + self.dependencyInfo.directories[serverPath] = { + include: [/.?/], + exclude: ignoreFiles + }; // --- Third party dependencies --- @@ -508,22 +517,28 @@ _.extend(Bundle.prototype, { }; if (isApp) { - if (fs.existsSync(path.join(projectDir, 'public'))) { - var copied = - files.cp_r(path.join(projectDir, 'public'), - path.join(buildPath, 'static'), {ignore: ignoreFiles}); + var publicDir = path.join(projectDir, 'public'); - _.each(copied, function (fsRelativePath) { - var filepath = path.join(buildPath, 'static', fsRelativePath); + if (fs.existsSync(publicDir)) { + var copied = + files.cp_r(publicDir, path.join(buildPath, 'static'), + {ignore: ignoreFiles}); + + _.each(copied, function (relPath) { + var filepath = path.join(publicDir, relPath); var contents = fs.readFileSync(filepath); var hash = sha1(contents); - self.dependencyInfo.hashes[ - path.join(projectDir, 'public', fsRelativePath)] = hash; - addClientFileToManifest(fsRelativePath, contents, 'static', false, + + self._addDependency(filepath, hash); + addClientFileToManifest(relPath, contents, 'static', false, undefined, hash); }); } - self.dependencyInfo.app.push('public'); + + self.dependencyInfo.directories[publicDir] = { + include: [/.?/], + exclude: ignoreFiles + }; } // Add cache busting query param if needed, and @@ -603,7 +618,7 @@ _.extend(Bundle.prototype, { where: 'internal', hash: sha1(appHtml) }); - self.dependencyInfo.core.push(path.join('tools', 'app.html.in')); + self._addDependency(path.join(__dirname, 'app.html.in')); // --- Documentation, and running from the command line --- @@ -626,30 +641,35 @@ _.extend(Bundle.prototype, { "\n" + "Find out more about Meteor at meteor.com.\n"); + // --- Source file dependencies --- + + _.each(_.values(self.slices), function (slice) { + _.extend(self.dependencyInfo.files, slice.pkg.dependencyFileShas); + }); + + if (isApp) { + // Include files in the app that match any file extension + // handled by any package that the app uses + self.dependencyInfo.directories[path.resolve(projectDir)] = { + include: _.map(self._appExtensions(), function (ext) { + return new RegExp('\\.' + ext.slice(1) + "$"); + }), + exclude: ignoreFiles + }; + // Exclude the packages directory in an app, since packages + // explicitly call out the files they use + self.dependencyInfo.directories[path.resolve(projectDir, 'packages')] = { + exclude: [/.?/] + }; + // Exclude .meteor/local and everything under it + self.dependencyInfo.directories[ + path.resolve(projectDir, '.meteor', 'local')] = { exclude: [/.?/] }; + } + // --- Metadata --- appJson.manifest = self.manifest; - self.dependencyInfo.extensions = self._appExtensions(); - self.dependencyInfo.exclude = _.pluck(ignoreFiles, 'source'); - self.dependencyInfo.packages = {}; - _.each(_.values(self.slices), function (slice) { - // Data for the mtime dependency watcher. We only record data here for - // packages, not apps, since apps watch the whole directory for added - // files. - if (slice.pkg.name) { - self.dependencyInfo.packages[slice.pkg.name] = _.union( - self.dependencyInfo.packages[slice.pkg.name] || [], - _.keys(slice.pkg.dependencyFileShas) - ); - } - // Data for the contents dependency watcher check. - _.each(slice.pkg.dependencyFileShas, function (sha, relPath) { - self.dependencyInfo.hashes[ - path.join(slice.pkg.sourceRoot, relPath)] = sha; - }); - }); - if (self.releaseStamp && self.releaseStamp !== 'none') appJson.release = self.releaseStamp; @@ -680,16 +700,8 @@ _.extend(Bundle.prototype, { * - errors: An array of strings, or falsy if bundling succeeded. * - dependencyInfo: Information about files and paths that were * inputs into the bundle and that we may wish to monitor for - * changes if we are developing interactively. - * - extensions: list of extensions registered for user code, with dots] - * - packages: map from package name to list of paths relative to the package - * - core: paths relative to 'app' in meteor tree - * - app: paths relative to top of app tree - * - exclude: list of regexps for files to ignore (everywhere) - * (for 'core' and 'apps', if a directory is given, you should - * monitor everything in the subtree under it minus the stuff that - * matches exclude, and if it doesn't exist yet, you should watch - * for it to appear) + * changes when developing interactively. It has two keys, 'files' + * and 'directories', in the format expected by watch.Watcher. * * On failure ('errors' is truthy), no bundle will be output (in fact, * outputPath will have been removed if it existed.) diff --git a/tools/packages.js b/tools/packages.js index 418cd12672..a960cdd7fa 100644 --- a/tools/packages.js +++ b/tools/packages.js @@ -188,6 +188,18 @@ var Package = function (library) { }; _.extend(Package.prototype, { + // Add a dependency (in the sense of dependencyFileShas) on a + // file. If hash is supplied it should be the sha1 of the file + // contents. If omitted it will be computed. relPath will be + // resolved to an absolute path (relative to self.sourceRoot.) + _addDependency: function (relPath, contents) { + var self = this; + var absPath = path.resolve(self.sourceRoot, relPath); + if (! contents) + contents = fs.readFileSync(absPath); + self.dependencyFileShas[absPath] = bundler.sha1(contents); + }, + // loads a package's package.js file into memory, using // runInThisContext. Wraps the contents of package.js in a closure, // supplying pseudo-globals 'Package' and 'Npm'. @@ -218,8 +230,7 @@ _.extend(Package.prototype, { // steer clear var func = require('vm').runInThisContext(wrapped, fullpath, true); func(self.packageFacade, self.npmFacade); - - self.dependencyFileShas['package.js'] = bundler.sha1(code); + self._addDependency("package.js", code); // source files used var sources = {use: {client: [], server: []}, @@ -402,9 +413,7 @@ _.extend(Package.prototype, { 'spark', 'templating', 'startup', 'past']; packages = _.union(packages, project.get_packages(appDir)); // XXX this read has a race with the actual read that is used - var packagesFile = path.join(appDir, '.meteor', 'packages'); - self.dependencyFileShas[path.join('.meteor', 'packages')] = - bundler.sha1(fs.readFileSync(packagesFile)); + self._addDependency(path.join(appDir, '.meteor', 'packages')); _.each(["use", "test"], function (role) { _.each(["client", "server"], function (where) { @@ -502,10 +511,11 @@ _.extend(Package.prototype, { }; _.each(self.sources[role][where], function (relPath) { + var absPath = path.resolve(self.sourceRoot, relPath); var ext = path.extname(relPath).substr(1); var handler = self._getSourceHandler(role, where, ext); - var contents = fs.readFileSync(path.join(self.sourceRoot, relPath)); - self.dependencyFileShas[relPath] = bundler.sha1(contents); + var contents = fs.readFileSync(absPath); + self._addDependency(absPath, contents); if (! handler) { // If we don't have an extension handler, serve this file diff --git a/tools/run.js b/tools/run.js index 927664ac4a..31096d5b3d 100644 --- a/tools/run.js +++ b/tools/run.js @@ -8,6 +8,7 @@ var httpProxy = require('http-proxy'); var files = require('./files.js'); var library = require('./library.js'); +var watch = require('./watch.js'); var project = require('./project.js'); var updater = require('./updater.js'); var bundler = require('./bundler.js'); @@ -218,7 +219,7 @@ var logToClients = function (msg) { // onExit // [onListen] // [nodeOptions] -// [settingsFile] +// [settings] var startServer = function (options) { // environment @@ -233,12 +234,10 @@ var startServer = function (options) { env.PORT = options.innerPort; env.MONGO_URL = options.mongoUrl; env.ROOT_URL = env.ROOT_URL || ('http://localhost:' + options.outerPort); - if (options.settingsFile) { - // Re-read the settings file each time we call startServer. - var settings = exports.getSettings(options.settingsFile); - if (settings) - env.METEOR_SETTINGS = settings; - } + if (options.settings) + env.METEOR_SETTINGS = options.settings; + else + delete env.METEOR_SETTINGS; var nodeOptions = _.clone(options.nodeOptions); nodeOptions.push(path.join(options.bundlePath, 'main.js')); @@ -311,243 +310,6 @@ var killServer = function (handle) { clearInterval(handle.timer); }; -////////// Watching dependencies ////////// - -// deps is the data from dependencies.json in the bundle -// appDir is the root of the app -// relativeFiles are any other files to watch, relative to the current -// directory (eg, the --settings file) -// onChange is only fired once -var DependencyWatcher = function ( - deps, appDir, relativeFiles, library, onChange) { - var self = this; - - self.appDir = appDir; - self.onChange = onChange; - self.watches = {}; // path => unwatch function with no arguments - self.lastContents = {}; // path => last contents (array of filenames) - self.mtimes = {}; // path => last seen mtime - - // If a file is under a sourceDir, and has one of the - // sourceExtensions, then it's interesting. - self.sourceDirs = [self.appDir]; - self.sourceExtensions = deps.extensions || []; - - // Any file under a bulkDir is interesting. (bulkDirs may also - // contain individual files) - self.bulkDirs = []; - // If we're running from a git checkout, we reload when "core" files like - // server.js change. - if (!files.usesWarehouse()) { - _.each(deps.core || [], function (filepath) { - self.bulkDirs.push(path.join(files.getCurrentToolsDir(), filepath)); - }); - } - _.each(deps.app || [], function (filepath) { - self.bulkDirs.push(path.join(self.appDir, filepath)); - }); - - // Additional list of specific files that are interesting. - self.specificFiles = {}; - for (var pkg in (deps.packages || {})) { - // We only watch for changes in local packages, rather than ones in the - // warehouse, since only changes to local ones need to cause an app to - // reload. Notably, the app will *not* reload the first time a local package - // is created which overrides an installed package. - var localPackageDir = library.directoryForLocalPackage(pkg); - if (localPackageDir) { - _.each(deps.packages[pkg], function (file) { - self.specificFiles[path.join(localPackageDir, file)] = true; - }); - } - }; - - _.each(relativeFiles, function (file) { - self.specificFiles[file] = true; - }); - - // Things that are never interesting. - self.excludePatterns = _.map((deps.exclude || []), function (pattern) { - return new RegExp(pattern); - }); - self.excludePaths = [ - path.join(appDir, '.meteor', 'local'), - // For app packages, we only watch files explicitly used by the package (in - // specificFiles) - path.join(appDir, 'packages') - ]; - - // Start monitoring - _.each(_.union(self.sourceDirs, self.bulkDirs, _.keys(self.specificFiles)), - _.bind(self._scan, self, true)); - - // mtime scans are great and relatively efficient, but they have a couple of - // issues. One is that they only detect changes in mtimes from the start of - // dependency watching, not from the actual bundled file, so if bundling is - // slow and somebody edits a file after it's used by the bundler but before - // the DependencyWatcher is created, we'll miss it. An even worse problem is - // that on OSX HFS+, mtime resolution is only one second, so if a file is - // written twice in a second the bundler might get the first version and never - // notice the second change! So, in a second, we'll do a one-time scan to - // check the hash of each file against what the bundler told us we should see. - // - // This will still miss files that are newly added during bundling, and there - // are also race conditions where the bundler may calculate some hashes via a - // separate read than the read that actually was used in bundling... but it's - // close. - setTimeout(function() { - _.each(deps.hashes, function (hash, filepath) { - fs.readFile(filepath, function (error, contents) { - // Fire if the file was deleted or changed contents. - if (error || bundler.sha1(contents) !== hash) - self._fire(); - }); - }); - }, 1000); -}; - -_.extend(DependencyWatcher.prototype, { - // stop monitoring - destroy: function () { - var self = this; - self.onChange = null; - for (var filepath in self.watches) - self.watches[filepath](); // unwatch - self.watches = {}; - }, - - _fire: function () { - var self = this; - if (self.onChange) { - var f = self.onChange; - self.onChange = null; - f(); - self.destroy(); - } - }, - - // initial is true on the inital scan, to suppress notifications - _scan: function (initial, filepath) { - var self = this; - - if (self._isExcluded(filepath)) - return false; - - try { - var stats = fs.lstatSync(filepath); - } catch (e) { - // doesn't exist -- leave stats undefined - } - - // '+' is necessary to coerce the mtimes from date objects to ints - // (unix times) so they can be conveniently tested for equality - if (stats && +stats.mtime === +self.mtimes[filepath]) - // We already know about this file and it hasn't actually - // changed. Probably its atime changed. - return; - - // If an interesting file has changed, fire! - var isInteresting = self._isInteresting(filepath); - if (!initial && isInteresting) { - self._fire(); - return; - } - - if (!stats) { - // A directory (or an uninteresting file) was removed - var unwatch = self.watches[filepath]; - unwatch && unwatch(); - delete self.watches[filepath]; - delete self.lastContents[filepath]; - delete self.mtimes[filepath]; - return; - } - - // If we're seeing this file or directory for the first time, - // monitor it if necessary - if (!(filepath in self.watches) && - (isInteresting || stats.isDirectory())) { - if (!stats.isDirectory()) { - // Intentionally not using fs.watch since it doesn't play well with - // vim (https://github.com/joyent/node/issues/3172) - fs.watchFile(filepath, {interval: 500}, // poll a lot! - _.bind(self._scan, self, false, filepath)); - self.watches[filepath] = function() { fs.unwatchFile(filepath); }; - } else { - // fs.watchFile doesn't work for directories (as tested on ubuntu) - var watch = fs.watch(filepath, {interval: 500}, // poll a lot! - _.bind(self._scan, self, false, filepath)); - self.watches[filepath] = function() { watch.close(); }; - } - self.mtimes[filepath] = stats.mtime; - } - - // If a directory, recurse into any new files it contains. (We - // don't need to check for removed files here, since if we care - // about a file, we'll already be monitoring it) - if (stats.isDirectory()) { - var oldContents = self.lastContents[filepath] || []; - var newContents = fs.readdirSync(filepath); - var added = _.difference(newContents, oldContents); - - self.lastContents[filepath] = newContents; - _.each(added, function (child) { - self._scan(initial, path.join(filepath, child)); - }); - } - }, - - // Should we even bother to scan/recurse into this file? - _isExcluded: function (filepath) { - var self = this; - - // Files we're specifically being asked to scan are never excluded. For - // example, files from app packages (that are actually pulled in by their - // package.js) are not excluded, but the app packages directory itself is - // (so that other files in package directories aren't watched). - if (filepath in self.specificFiles) - return false; - - if (_.indexOf(self.excludePaths, filepath) !== -1) - return true; - - var excludedByPattern = _.any(self.excludePatterns, function (regexp) { - return path.basename(filepath).match(regexp); - }); - - return excludedByPattern; - }, - - // Should we fire if this file changes? - _isInteresting: function (filepath) { - var self = this; - - if (self._isExcluded(filepath)) - return false; - - var inAnyDir = function (dirs) { - return _.any(dirs, function (dir) { - return filepath.slice(0, dir.length) === dir; - }); - }; - - // Specific, individual files that we want to monitor - if (filepath in self.specificFiles) - return true; - - // Source files - if (inAnyDir(self.sourceDirs) && - files.findExtension(self.sourceExtensions, filepath)) - return true; - - // Other directories and files that are included - if (inAnyDir(self.bulkDirs)) - return true; - - return false; - } -}); - /////////////////////////////////////////////////////////////////////////////// // Also used by "meteor deploy" in meteor.js. @@ -576,7 +338,7 @@ exports.getSettings = function (filename) { // This function never returns and will call process.exit() if it // can't continue. If you change this, remember to call -// watcher.destroy() as appropriate. +// watcher.stop() as appropriate. // // context is as created in meteor.js. // options include: port, minify, once, settingsFile, testPackages @@ -638,22 +400,18 @@ exports.run = function (context, options) { return; if (watcher) - watcher.destroy(); + watcher.stop(); - var relativeFiles; - if (options.settingsFile) { - relativeFiles = [options.settingsFile]; - } - - var onChange = function () { - if (Status.crashing) - logToClients({'system': "=> Modified -- restarting."}); - Status.reset(); - restartServer(); - }; - - watcher = new DependencyWatcher(dependencyInfo, context.appDir, - relativeFiles, context.library, onChange); + watcher = new watch.Watcher({ + files: dependencyInfo.files, + directories: dependencyInfo.directories, + onChange: function () { + if (Status.crashing) + logToClients({'system': "=> Modified -- restarting."}); + Status.reset(); + restartServer(); + } + }); }; // Using `inFiber` since bundling can yield when loading a manifest @@ -690,9 +448,9 @@ exports.run = function (context, options) { // Make the library reload packages, in case they've changed context.library.flush(); + // Bundle up the app var bundleResult = bundler.bundle(context.appDir, bundlePath, bundleOpts); - startWatching(bundleResult.dependencyInfo); - + var dependencyInfo = bundleResult.dependencyInfo; if (bundleResult.errors) { logToClients({stdout: "=> Errors prevented startup:\n"}); _.each(bundleResult.errors, function (e) { @@ -703,10 +461,52 @@ exports.run = function (context, options) { return; } + // Read the settings file, if any + var settings = null; + if (options.settingsFile) { + settings = exports.getSettings(options.settingsFile); + + // 'getSettings' will collapse any amount of whitespace down to + // the empty string, so to get the sha1 for change monitoring, + // we need to reread the file, which creates a tiny race + // condition (not a big enough deal to care about right now.) + var settingsHash = + bundler.sha1(fs.readFileSync(options.settingsFile, "utf8")); + + // Reload if the setting file changes + dependencyInfo.files[path.resolve(options.settingsFile)] = + settingsHash; + } + + // If using a warehouse, don't do dependency monitoring on any of + // the files that are in the warehouse. You should not be editing + // those files directly. + if (files.usesWarehouse()) { + var warehouseDir = path.resolve(warehouse.getWarehouseDir()); + var filterKeys = function (obj) { + _.each(_.keys(obj), function (k) { + k = path.resolve(k); + if (warehouseDir.length <= k.length && + k.substr(0, warehouseDir.length) === warehouseDir) + delete obj[k]; + }); + }; + filterKeys(dependencyInfo.files); + filterKeys(dependencyInfo.directories); + } + + // Start watching for changes for files. There's no hurry to call + // this, since dependencyInfo contains a snapshot of the state of + // the world at the time of bundling, in the form of hashes and + // lists of matching files in each directory. + startWatching(dependencyInfo); + + // Start the server Status.running = true; if (firstRun) { - process.stdout.write("=> Meteor server running on: http://localhost:" + outerPort + "/\n"); + process.stdout.write("=> Meteor server running on: http://localhost:" + + outerPort + "/\n"); firstRun = false; lastThingThatPrintedWasRestartMessage = false; } else { @@ -745,7 +545,7 @@ exports.run = function (context, options) { requestQueue = []; }, nodeOptions: getNodeOptionsFromEnvironment(), - settingsFile: options.settingsFile + settings: settings }); }); diff --git a/tools/tests/test_watch.js b/tools/tests/test_watch.js new file mode 100644 index 0000000000..19f042d2b3 --- /dev/null +++ b/tools/tests/test_watch.js @@ -0,0 +1,322 @@ +var path = require('path'); +var fs = require('fs'); +var _ = require('underscore'); +var assert = require('assert'); +var crypto = require('crypto'); +var Fiber = require('fibers'); +var Future = require('fibers/future'); +var watch = require(path.join(__dirname, '..', 'watch.js')); +var files = require(path.join(__dirname, '..', 'files.js')); + +var tmp = files.mkdtemp('test_watch'); +var serial = 0; + +var touchFile = function (filePath, contents) { + filePath = path.join(tmp, filePath); + files.mkdir_p(path.dirname(filePath)); + fs.writeFileSync(filePath, contents || ('' + serial)); + serial++; +}; + +var touchDir = function (dirPath) { + dirPath = path.join(tmp, dirPath); + files.mkdir_p(dirPath); +}; + +var remove = function (fileOrDirPath) { + fileOrDirPath = path.join(tmp, fileOrDirPath); + files.rm_recursive(fileOrDirPath); +}; + +var theWatcher; +var fired; +var firedFuture; +var lastOptions; +var go = function (options) { + options = options || lastOptions; + lastOptions = options; + if (theWatcher) { + theWatcher.stop(); + theWatcher = null; + } + fired = false; + + var files = {}; + _.each(options.files, function (value, file) { + file = path.join(tmp, file); + if (typeof value !== "string") { + if (fs.existsSync(file)) { + var hash = crypto.createHash('sha1'); + hash.update(fs.readFileSync(file)); + value = hash.digest('hex'); + } else { + value = 'dummyhash'; + } + } + files[file] = value; + }); + + var directories = {}; + _.each(options.directories, function (options, dir) { + dir = path.join(tmp, dir); + directories[dir] = options; + }); + + theWatcher = new watch.Watcher({ + files: files, + directories: directories, + onChange: function () { + fired = true; + if (firedFuture) + firedFuture.return(true); + } + }); +} + +var fires = function (waitDuration) { + if (! theWatcher) + throw new Error("No watcher?"); + if (fired) + return true; + // Wait up to a second for it to fire + var timeout = setTimeout(function () { + firedFuture.return(false); + }, waitDuration || 1000); + if (firedFuture) + throw new Error("Already have a future"); + firedFuture = new Future; + var ret = firedFuture.wait(); + clearTimeout(timeout); + firedFuture = null; + return ret; +}; + +var waitForTopOfSecond = function () { + while (true) { + var msPastSecond = +(new Date) % 1000; + if (msPastSecond < 100) { + return; + } + var f = new Future; + setTimeout(function () { + f.return(); + }, 25); + f.wait(); + } +}; + +var delay = function (duration) { + var f = new Future; + setTimeout(function () { + f.return(); + }, duration); + f.wait(); +}; + +Fiber(function () { + console.log("Test Watcher"); + + console.log("... one file"); + touchFile('/aa/b', 'kitten'); + go({ + files: { '/aa/b': true } + }); + assert(!fires()); + touchFile('/aa/b', 'kitten'); + assert(!fires()); + touchFile('/aa/b', 'puppy'); + assert(fires()); + go(); + touchFile('/aa/b', 'puppy'); + assert(!fires()); + touchFile('/aa/b', 'kitten'); + assert(fires()); + go(); + remove('/aa/b'); + assert(fires()); + touchFile('/aa/b'); + go({ + files: { '/aa/b': true, '/aa/c': true } + }); + assert(fires()); // look like /aa/c was removed + + console.log("... directories"); + go({ + files: {'/aa/b': true }, + directories: {'/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + }} + }); + touchFile('/aa/c'); + assert(!fires()); + touchFile('/aa/maybe-not'); + assert(!fires()); + touchFile('/aa/never-yes'); + assert(!fires()); + touchFile('/aa/never'); + assert(!fires()); + touchFile('/aa/yes-for-sure'); + assert(fires()); + go(); + touchFile('/aa/nope'); + assert(fires()); // because yes-for-sure isn't in the file list + remove('/aa/yes-for-sure'); + go(); + assert(!fires()); + touchFile('/aa/maybe-this-time'); + assert(fires()); + go(); + assert(fires()); // maybe-this-time is still there + go({ + files: {'/aa/b': true, '/aa/maybe-this-time': true }, + directories: {'/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + }} + }); + go(); + assert(!fires()); // maybe-this-time is now in the expected file list + touchFile('/aa/maybe-yes'); + assert(fires()); + remove('/aa/maybe-yes'); + remove('/aa/maybe-this-time'); + go(); + assert(fires()); // maybe-this-time is missing + + console.log("... recursive directories"); + touchFile('/aa/b'); + go({ + files: {'/aa/b': true }, + directories: {'/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + }} + }); + touchDir('/aa/yess'); + assert(!fires()); + remove('/aa/yess'); + assert(!fires()); + touchFile('/aa/yess/kitten'); + assert(!fires()); + touchFile('/aa/yess/maybe'); + assert(fires()); + remove('/aa/yess'); + go(); + touchFile('/aa/whatever/kitten'); + assert(!fires()); + touchFile('/aa/whatever/maybe'); + assert(fires()); + + remove('/aa/whatever'); + go(); + touchDir('/aa/i/love/subdirectories'); + assert(!fires()); + touchFile('/aa/i/love/subdirectories/yessir'); + assert(fires()); + remove('/aa/i/love/subdirectories/yessir'); + go(); + touchFile('/aa/i/love/subdirectories/every/day'); + assert(!fires()); + remove('/aa/i/love/subdirectories'); + assert(!fires()); + touchFile('/aa/i/love/not/nothing/yes'); + assert(!fires()); + touchFile('/aa/i/love/not/nothing/maybe/yes'); + assert(!fires()); + touchFile('/aa/i/love/maybe'); + assert(fires()); + remove('/aa/i'); + remove('/aa/whatever'); + + remove('/aa'); + touchFile('/aa/b'); + console.log("... nested directories"); + go({ + files: {'/aa/b': true }, + directories: { + '/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + }, + '/aa/x': { + include: [/kitten/], + exclude: [/puppy/] + } + } + }); + touchFile('/aa/kitten'); + assert(!fires()); + touchFile('/aa/maybe.puppy'); + assert(fires()); + remove('/aa/maybe.puppy'); + go(); + touchFile('/aa/x/kitten'); + assert(fires()); + remove('/aa/x/kitten'); + go(); + touchFile('/aa/x/yes'); + assert(!fires()); + touchFile('/aa/x/kitten.not'); + assert(fires()); + remove('/aa'); + + console.log("... rapid changes to file"); + touchFile('/aa/x'); + waitForTopOfSecond(); + go({ + files: {'/aa/x': true }}); + touchFile('/aa/x'); + assert(fires(2000)); + go({ + directories: { + '/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + } + } + }); + waitForTopOfSecond(); + touchFile('/aa/thing1/whatever'); + delay(100); + touchFile('/aa/thing2/yes'); + assert(fires(2000)); + remove('/aa'); + + console.log("... rapid changes to directory"); + touchDir('/aa'); + waitForTopOfSecond(); + go({ + directories: {'/aa': { + include: [/yes/, /maybe/, /aa/], + exclude: [/not/, /never/] + }} + }); + touchFile('/aa/x/yes'); + assert(fires(2000)); + remove('/aa/x'); + + waitForTopOfSecond(); + go(); + delay(600); + touchFile('/aa/x/not'); + delay(600); + touchFile('/aa/x/yes'); + assert(fires(2000)); + remove('/aa/x'); + + touchDir('/aa/x'); + go(); + delay(2000); + waitForTopOfSecond(); + touchFile('/aa/x/no'); + delay(600); + touchFile('/aa/x/yes'); + assert(fires(2000)); + + console.log("Watcher test passed"); + theWatcher.stop(); + +}).run(); + + diff --git a/tools/watch.js b/tools/watch.js new file mode 100644 index 0000000000..b1d628e0fc --- /dev/null +++ b/tools/watch.js @@ -0,0 +1,308 @@ +var fs = require("fs"); +var path = require("path"); +var crypto = require('crypto'); +var _ = require('underscore'); + +// Watch for changes to a set of files, and the first time that any of +// the files change, call a user-provided callback. (If you want a +// second callback, you'll need to create a second Watcher.) +// +// You can set up two kinds of watches, file and directory watches. +// +// In a file watch, you provide an absolute path to a file and a SHA1 +// (encoded as hex) of the contents of that file. If the file ever +// changes so that its contents no longer match that SHA1, the +// callback triggers. +// +// In a directory watch, you provide an absolute path to a directory +// and two lists of regular expressions specifying the files to +// include or exclude. If there is ever a file in the directory or its +// children that matches the criteria set up by the regular +// expressions, but that IS NOT present as a file watch, then the +// callback triggers. +// +// For directory watches, the regular expressions work as follows. You +// provide two arrays of regular expressions, an include list and an +// exclude list. A file in the directory matches if it matches at +// least one regular expression in the include list, and doesn't match +// any regular expressions in the exclude list. Subdirectories are +// included recursively, as long as their names do not match any +// regular expression in the exclude list. +// +// When multiple directory watches are set up, say on a directory A +// and its subdirectory B, the most specific watch takes precedence in +// each directory. So only B's include/exclude lists will be checked +// in B. +// +// Regular expressions are checked only against individual path +// components (the actual name of the file or the subdirectory), not +// against the entire path. +// +// You can call stop() to stop watching and tear down the +// watcher. Calling stop() guarantees that you will not receive a +// callback (if you have not already.) Calling stop() is unnecessary +// if you've received a callback. +// +// A limitation of the current implementation is that if you set up a +// directory watch on a directory A, and A does not exist at the time +// the Watcher is created but is then created later, then A will not +// be monitored. (Of course, this limitation only applies to the roots +// of the directory watches. If A exists at the time the watch is +// created, and a subdirectory B is later created, it will be properly +// detected. Likewise if A exists and is then deleted it will be +// detected.) +// +// Options may include +// - files: see self.files comment below +// - directories: see self.directories comment below +// - onChange: the function to call when the first change is detected +// +var Watcher = function (options) { + var self = this; + + // Map from the absolute path to a file, to a sha1 hash. Fire when + // the file changes from that sha. + self.files = options.files || {}; + + // Map from an absolute path to a directory, to an object with keys + // 'include' and 'exclude', both lists of regular expressions. Fire + // when a file is added to that directory whose name matches at + // least one regular expression in 'include' and no regular + // expressions in 'exclude'. Subdirectories are included + // recursively, but not subdirectories that match 'exclude'. The + // most specific rule wins, so you can change the parameters in + // effect in subdirectories simply by specifying additional rules. + self.directories = options.directories || {}; + + // Function to call when a change is detected according to one of + // the above. + self.onChange = options.onChange; + if (! self.onChange) + throw new Error("onChange option is required"); + + // self.directories in a different form. It's an array of objects, + // each with keys 'dir', 'include', 'options', where path is + // guaranteed to not contain a trailing slash (unless it is the root + // directory) and the objects are sorted from longest path to + // shortest (that is, most specific rule to least specific.) + self.rules = _.map(self.directories, function (options, dir) { + return { + dir: path.resolve(dir), + include: options.include || [], + exclude: options.exclude || [] + }; + }); + self.rules = self.rules.sort(function (a, b) { + return a.dir.length < b.dir.length ? 1 : -1; + }); + + self.stopped = false; + self.fileWatches = []; // array of paths + self.directoryWatches = {}; // map from path to watch object + + self._startFileWatches(); + _.each(self.rules, function (rule) { + self._watchDirectory(rule.dir); + }); +}; + +_.extend(Watcher.prototype, { + _checkFileChanged: function (absPath) { + var self = this; + + if (! fs.existsSync(absPath)) + return true; + + var hasher = crypto.createHash('sha1'); + hasher.update(fs.readFileSync(absPath)); + var hash = hasher.digest('hex'); + + return (self.files[absPath] !== hash); + }, + + _startFileWatches: function () { + var self = this; + + // Set up a watch for each file + _.each(self.files, function (hash, absPath) { + // Intentionally not using fs.watch since it doesn't play well with + // vim (https://github.com/joyent/node/issues/3172) + // Note that we poll very frequently (500 ms) + fs.watchFile(absPath, {interval: 500}, function () { + // Fire only if the contents of the file actually changed (eg, + // maybe just its atime changed) + if (self._checkFileChanged(absPath)) + self._fire(); + }); + self.fileWatches.push(absPath); + + // Check for the case where by the time we created the watch, + // the file had already changed from the sha we were provided. + if (self._checkFileChanged(absPath)) + self._fire(); + }); + + // One second later, check the files again, because fs.watchFile + // is actually implemented by polling the file's mtime, and some + // filesystems (OSX HFS+) only keep mtimes to a resolution of one + // second. This handles the case where we check the hash and set + // up the watch, but then the file change before the clock rolls + // over to the next second, and fs.watchFile doesn't notice and + // doesn't call us back. #WorkAroundLowPrecisionMtimes + setTimeout(function () { + _.each(self.files, function (hash, absPath) { + if (self._checkFileChanged(absPath)) + self._fire(); + }); + }, 1000); + }, + + // Pass true for `include` to include everything (and process only + // excludes) + _matches: function (filename, include, exclude) { + var self = this; + + if (include === true) + include = [/.?/]; + for (var i = 0; i < include.length; i++) + if (include[i].test(filename)) + break; + if (i === include.length) { + return false; // didn't match any includes + } + + for (var i = 0; i < exclude.length; i++) { + if (exclude[i].test(filename)) { + return false; // matched an exclude + } + } + + // Matched an include and didn't match any excludes + return true; + }, + + _watchDirectory: function (absPath) { + var self = this; + + if (absPath in self.directoryWatches) + // Already being taken care of + return; + + // Determine the options that apply to this directory by finding + // the most specific rule. + absPath = path.resolve(absPath); // ensure no trailing slash + for (var i = 0; i < self.rules.length; i++) { + var rule = self.rules[i]; + if (absPath.length >= rule.dir.length && + absPath.substr(0, rule.dir.length) === rule.dir) + break; // found a match + rule = null; + } + if (! rule) + // Huh, doesn't appear that we're supposed to be watching this + // directory. + return; + + var contents = []; + var scanDirectory = function (isDoubleCheck) { + if (self.stopped) + return; + + if (! fs.existsSync(absPath)) { + // Directory was removed. Stop watching. + var watch = self.directoryWatches[absPath]; + watch && watch.close(); + delete self.directoryWatches[absPath]; + return; + } + + // Find previously unknown files and subdirectories. (We don't + // care about removed subdirectories because the logic + // immediately above handles them, and we don't care about + // removed files because the ones we care about will already + // have file watches on them.) + var newContents = fs.readdirSync(absPath); + var added = _.difference(newContents, contents); + contents = newContents; + + // Look at each newly added item + _.each(added, function (addedItem) { + var addedPath = path.join(absPath, addedItem); + + // Is it a directory? + try { + var stats = fs.lstatSync(addedPath); + } catch (e) { + // Can't be found? That's weird. Ignore. + return; + } + var isDirectory = stats.isDirectory(); + + // Does it match the rule? + if (! self._matches(addedItem, + isDirectory ? true : rule.include, + rule.exclude)) + return; // No + + if (! isDirectory) { + if (! (addedPath in self.files)) + // Found a newly added file that we care about. + self._fire(); + } else { + // Found a subdirectory that we care to monitor. + self._watchDirectory(addedPath); + } + }); + + if (! isDoubleCheck) { + // Whenever a directory changes, scan it soon as we notice, + // but then scan it again one secord later just to make sure + // that we haven't missed any changes. See commentary at + // #WorkAroundLowPrecisionMtimes + setTimeout(function () { + scanDirectory(true); + }, 1000); + } + }; + + // fs.watchFile doesn't work for directories (as tested on ubuntu) + // Notice that we poll very frequently (500 ms) + try { + self.directoryWatches[absPath] = + fs.watch(absPath, {interval: 500}, scanDirectory); + scanDirectory(); + } catch (e) { + // Can happen if the directory doesn't exist, say because a + // nonexistent path was included in self.directories + } + }, + + _fire: function () { + var self = this; + + if (self.stopped) + return; + + self.stop(); + self.onChange(); + }, + + stop: function () { + var self = this; + self.stopped = true; + + // Clean up file watches + _.each(self.fileWatches, function (absPath) { + fs.unwatchFile(absPath); + }); + self.fileWatches = []; + + // Clean up directory watches + _.each(self.directoryWatches, function (watch) { + watch.close(); + }); + self.directoryWatches = {}; + } +}); + +exports.Watcher = Watcher;