Merge pull request #11689 from meteor/feature/recursive-file-watchers

Recursive file watchers

meteor

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-BUNDLE_VERSION=14.17.6.2
+BUNDLE_VERSION=14.17.6.5
 
 # OS Check. Put here because here is where we download the precompiled
 # bundles that are arch specific.

scripts/dev-bundle-tool-package.js

@@ -58,6 +58,7 @@ var packageJson = {
     split2: "3.2.2",
     multipipe: "2.0.1",
     pathwatcher: "8.1.0",
+    "vscode-nsfw": "2.1.8",
     // The @wry/context package version must be compatible with the
     // version constraint imposed by optimism/package.json.
     optimism: "0.16.1",

tools/cli/commands-packages.js

@@ -28,7 +28,6 @@ import {
   newPluginId,
   splitPluginsAndPackages,
 } from '../cordova/index.js';
-import { disableNativeWatcher } from '../fs/safe-watcher';
 import { updateMeteorToolSymlink } from "../packaging/updater.js";
 
 // For each release (or package), we store a meta-record with its name,
@@ -96,7 +95,6 @@ main.registerCommand({
     'allow-incompatible-update': { type: Boolean }
   }
 }, function (options) {
-  disableNativeWatcher();
 
   // If we're in an app, make sure that we can build the current app. Otherwise
   // just make sure that we can build some fake app.
@@ -1160,8 +1158,6 @@ main.registerCommand({
   },
   catalogRefresh: new catalog.Refresh.OnceAtStart({ ignoreErrors: true })
 }, function (options) {
-  disableNativeWatcher();
-
   var projectContext = new projectContextModule.ProjectContext({
     projectDir: options.appDir,
     allowIncompatibleUpdate: options['allow-incompatible-update']
@@ -1773,8 +1769,6 @@ main.registerCommand({
   maxArgs: Infinity,
   catalogRefresh: new catalog.Refresh.OnceAtStart({ ignoreErrors: true })
 }, function (options) {
-  disableNativeWatcher();
-
   // If you are specifying packages individually, you probably don't want to
   // update the release.
   if (options.args.length > 0) {
@@ -2137,8 +2131,6 @@ main.registerCommand({
   requiresApp: true,
   catalogRefresh: new catalog.Refresh.OnceAtStart({ ignoreErrors: true })
 }, function (options) {
-  disableNativeWatcher();
-
   var projectContext = new projectContextModule.ProjectContext({
     projectDir: options.appDir,
     allowIncompatibleUpdate: options["allow-incompatible-update"]
@@ -2347,8 +2339,6 @@ main.registerCommand({
   requiresApp: true,
   catalogRefresh: new catalog.Refresh.Never()
 }, function (options) {
-  disableNativeWatcher();
-
   var projectContext = new projectContextModule.ProjectContext({
     projectDir: options.appDir,
     allowIncompatibleUpdate: options["allow-incompatible-update"]

tools/cli/commands.js

@@ -17,7 +17,6 @@ var release = require('../packaging/release.js');
 
 const { Profile } = require("../tool-env/profile");
 
-import { disableNativeWatcher } from '../fs/safe-watcher';
 import { ensureDevBundleDependencies } from '../cordova/index.js';
 import { CordovaRunner } from '../cordova/runner.js';
 import { iOSRunTarget, AndroidRunTarget } from '../cordova/run-targets.js';
@@ -537,8 +536,6 @@ main.registerCommand({
   },
   catalogRefresh: new catalog.Refresh.Never()
 }, function (options) {
-  disableNativeWatcher();
-
   // Creating a package is much easier than creating an app, so if that's what
   // we are doing, do that first. (For example, we don't springboard to the
   // latest release to create a package if we are inside an app)
@@ -944,7 +941,6 @@ main.registerCommand({
   name: "build",
   ...buildCommands,
 }, async function (options) {
-  disableNativeWatcher();
   return Profile.run(
     "meteor build",
     () => Promise.await(buildCommand(options))

tools/fs/safe-watcher.ts

@@ -5,9 +5,15 @@ import {
   convertToOSPath,
   watchFile,
   unwatchFile,
+  toPosixPath,
+  pathRelative
 } from "./files";
+import {
+  join as nativeJoin
+} from 'path';
+import nsfw from 'vscode-nsfw';
 
-const watchLibrary = require("pathwatcher");
+const pathwatcher = require('pathwatcher');
 
 // Default to prioritizing changed files, but disable that behavior (and
 // thus prioritize all files equally) if METEOR_WATCH_PRIORITIZE_CHANGED
@@ -28,11 +34,17 @@ var NO_WATCHER_POLLING_INTERVAL =
 // file watchers, but it's to our advantage if they survive restarts.
 const WATCHER_CLEANUP_DELAY_MS = 30000;
 
+// Since linux doesn't have recursive file watching, nsfw has to walk the
+// watched folder and create a separate watcher for each subfolder. Until it has a
+// way for us to filter which folders it walks we will continue to use
+// pathwatcher to avoid having too many watchers.
+let watcherLibrary = process.env.METEOR_WATCHER_LIBRARY || 
+  (process.platform === 'linux' ? 'pathwatcher' : 'nsfw');
+
 // Pathwatcher complains (using console.error, ugh) if you try to watch
 // two files with the same stat.ino number but different paths on linux, so we have
 // to deduplicate files by ino.
-const DEDUPLICATE_BY_INO = process.platform !== "win32";
-
+const DEDUPLICATE_BY_INO = watcherLibrary === 'pathwatcher';
 // Set METEOR_WATCH_FORCE_POLLING environment variable to a truthy value to
 // force the use of files.watchFile instead of watchLibrary.watch.
 let watcherEnabled = ! JSON.parse(
@@ -41,7 +53,6 @@ let watcherEnabled = ! JSON.parse(
 
 const entriesByIno = new Map;
 
-
 export type SafeWatcher = {
   close: () => void;
 }
@@ -52,10 +63,14 @@ interface Entry extends SafeWatcher {
   callbacks: Set<EntryCallback>;
   rewatch: () => void;
   release: (callback: EntryCallback) => void;
+  _fire: (event: string) => void;
 }
 
 const entries: Record<string, Entry | null> = Object.create(null);
 
+// Folders that are watched recursively
+let watchRoots = new Set<string>();
+
 // Set of paths for which a change event has been fired, watched with
 // watchLibrary.watch if available. This could be an LRU cache, but in
 // practice it should never grow large enough for that to matter.
@@ -263,7 +278,8 @@ function startNewWatcher(absPath: string): Entry {
       safeUnwatch();
 
       unwatchFile(absPath, watchFileWrapper);
-    }
+    },
+    _fire: fire
   };
 
   if (stat && stat.ino > 0) {
@@ -341,9 +357,9 @@ function statWatch(
 }
 
 function watchLibraryWatch(absPath: string, callback: EntryCallback) {
-  if (watcherEnabled) {
+  if (watcherEnabled && watcherLibrary === 'pathwatcher') {
     try {
-      return watchLibrary.watch(convertToOSPath(absPath), callback);
+      return pathwatcher.watch(convertToOSPath(absPath), callback);
     } catch (e) {
       maybeSuggestRaisingWatchLimit(e);
       // ... ignore the error.  We'll still have watchFile, which is good
@@ -402,9 +418,61 @@ export const watch = Profile(
   }
 );
 
-// On Windows, pathwatcher can sometimes cause Meteor to get stuck. If we
-// don't need native watching for a command, we can disable it.
-// This is a temporary fix until pathwatcher is fixed or we replace it.
-export function disableNativeWatcher () {
-  watcherEnabled = false;
-} 
+const fireNames = {
+  [nsfw.actions.CREATED]: 'change',
+  [nsfw.actions.MODIFIED]: 'change',
+  [nsfw.actions.DELETED]: 'delete'
+}
+
+export function addWatchRoot(absPath: string) {
+  if (watchRoots.has(absPath) || watcherLibrary !== 'nsfw' || !watcherEnabled) {
+    return;
+  }
+
+  watchRoots.add(absPath);
+  
+  // If there already is a watcher for a parent directory, there is no need
+  // to create this watcher.
+  for (const path of watchRoots) {
+    let relativePath = pathRelative(path, absPath);
+    if (
+      path !== absPath &&
+      !relativePath.startsWith('..') &&
+      !relativePath.startsWith('/')
+    ) {
+      return;
+    }
+  }
+
+  // TODO: check if there are any existing watchers that are children of this
+  // watcher and stop them
+
+  nsfw(
+    convertToOSPath(absPath),
+    (events) => {
+      events.forEach(event => {
+        if(event.action === nsfw.actions.RENAMED) {
+          let oldPath = nativeJoin(event.directory, event.oldFile);
+          let oldEntry = entries[toPosixPath(oldPath)];
+          if (oldEntry) {
+            oldEntry._fire('rename');
+          }
+
+          let path = nativeJoin(event.newDirectory, event.newFile);
+          let newEntry = entries[toPosixPath(path)];
+          if (newEntry) {
+            newEntry._fire('change');
+          }
+        } else {
+            let path = nativeJoin(event.directory, event.file);
+            let entry = entries[toPosixPath(path)];
+            if (entry) {
+              entry._fire(fireNames[event.action]);
+            }
+        }
+      })
+    }
+  ).then(watcher => {
+    watcher.start()
+  });
+}

tools/project-context.js

@@ -89,6 +89,7 @@ import {
 } from "./utils/archinfo";
 
 import Resolver from "./isobuild/resolver";
+import { addWatchRoot } from './fs/safe-watcher';
 
 const CAN_DELAY_LEGACY_BUILD = ! JSON.parse(
   process.env.METEOR_DISALLOW_DELAYED_LEGACY_BUILD || "false"
@@ -175,6 +176,8 @@ Object.assign(ProjectContext.prototype, {
       : (options.projectLocalDir ||
         files.pathJoin(self.projectDir, '.meteor', 'local'));
 
+    addWatchRoot(self.projectDir);
+
     // Used by 'meteor rebuild'; true to rebuild all packages, or a list of
     // package names.  Deletes the isopacks and their plugin caches.
     self._forceRebuildPackages = options.forceRebuildPackages;
@@ -929,6 +932,13 @@ Object.assign(ProjectContext.prototype, {
     var self = this;
     buildmessage.assertInCapture();
 
+
+    self.packageMap.eachPackage((name, packageInfo) => {
+      if (packageInfo.kind === 'local') {
+        addWatchRoot(packageInfo.packageSource.sourceRoot)
+      }
+    });
+
     self.isopackCache = new isopackCacheModule.IsopackCache({
       packageMap: self.packageMap,
       includeCordovaUnibuild: (self._forceIncludeCordovaUnibuild