enable legacy watcher by config

scripts/dev-bundle-tool-package.js

@@ -57,6 +57,8 @@ var packageJson = {
     escope: "3.6.0",
     split2: "3.2.2",
     multipipe: "2.0.1",
+    pathwatcher: "8.1.2",
+    "vscode-nsfw": "2.1.8",
     "@parcel/watcher": "2.5.1",
     // The @wry/context package version must be compatible with the
     // version constraint imposed by optimism/package.json.

tools/cli/commands.js

@@ -261,14 +261,28 @@ export function parseRunTargets(targets) {
   });
 };
 
-function isModernArchsOnlyEnabled(appDir) {
+let meteorConfig;
+
+function getMeteorConfig(appDir) {
+  if (meteorConfig) return meteorConfig;
   const packageJsonPath = files.pathJoin(appDir, 'package.json');
   if (!files.exists(packageJsonPath)) {
     return false;
   }
   const packageJsonFile = files.readFile(packageJsonPath, 'utf8');
   const packageJson = JSON.parse(packageJsonFile);
-  return packageJson?.meteor?.modern?.webArchsOnly === true || packageJson?.meteor?.modern === true;
+  meteorConfig = packageJson?.meteor;
+  return meteorConfig;
+}
+
+function isModernArchsOnlyEnabled(appDir) {
+  const meteorConfig = getMeteorConfig(appDir);
+  return meteorConfig?.modern?.webArchsOnly === true || meteorConfig?.modern === true;
+}
+
+export function isModernWatcherEnabled(appDir) {
+  const meteorConfig = getMeteorConfig(appDir);
+  return meteorConfig?.modern?.watcher === true || meteorConfig?.modern === true;
 }
 
 function filterWebArchs(webArchs, excludeArchsOption, appDir, options) {
@@ -547,6 +561,7 @@ async function doRunCommand(options) {
   webArchs = filterWebArchs(webArchs, options['exclude-archs'], options.appDir, options);
   // Set the webArchs to include for compilation later
   global.includedWebArchs = webArchs;
+  global.modernWatcher = isModernWatcherEnabled(options.appDir);
 
   const buildMode = options.production ? 'production' : 'development';
 
@@ -1447,6 +1462,7 @@ on an OS X system.");
       ...options,
       platforms: projectContext.platformList.getPlatforms(),
     });
+    global.modernWatcher = isModernWatcherEnabled(options.appDir);
   }
 
   var buildDir = projectContext.getProjectLocalDirectory('build_tar');
@@ -2517,6 +2533,7 @@ var runTestAppForPackages = async function (projectContext, options) {
   buildOptions.webArchs = filterWebArchs(webArchs, options['exclude-archs'], projectContext.appDirectory, options);
   // Set the webArchs to include for compilation later
   global.includedWebArchs = buildOptions.webArchs;
+  global.modernWatcher = isModernWatcherEnabled(projectContext.appDirectory);
 
   if (options.deploy) {
     // Run the constraint solver and build local packages.

tools/cli/main.js

@@ -287,7 +287,7 @@ main.captureAndExit = async function (header, title, f) {
 
 // NB: files required up to this point may not define commands
 
-require('./commands.js');
+const { isModernWatcherEnabled } = require('./commands.js');
 require('./commands-packages.js');
 require('./commands-packages-query.js');
 require('./commands-cordova.js');
@@ -866,6 +866,7 @@ makeGlobalAsyncLocalStorage().run({}, async function () {
   if (appDir) {
     appDir = files.pathResolve(appDir);
   }
+  global.modernWatcher = isModernWatcherEnabled(appDir);
 
   await require('../tool-env/isopackets.js').ensureIsopacketsLoadable();
 

tools/fs/safe-watcher-legacy.ts

@@ -0,0 +1,476 @@
+import { FSWatcher, Stats, BigIntStats } from "fs";
+import { Profile } from "../tool-env/profile";
+import {
+    statOrNull,
+    convertToOSPath,
+    watchFile,
+    unwatchFile,
+    toPosixPath,
+    pathRelative
+} from "./files";
+import {
+    join as nativeJoin
+} from 'path';
+import nsfw from 'vscode-nsfw';
+
+const pathwatcher = require('pathwatcher');
+
+// Default to prioritizing changed files, but disable that behavior (and
+// thus prioritize all files equally) if METEOR_WATCH_PRIORITIZE_CHANGED
+// is explicitly set to a string that parses to a falsy value.
+var PRIORITIZE_CHANGED = true;
+if (process.env.METEOR_WATCH_PRIORITIZE_CHANGED &&
+    ! JSON.parse(process.env.METEOR_WATCH_PRIORITIZE_CHANGED)) {
+    PRIORITIZE_CHANGED = false;
+}
+
+var DEFAULT_POLLING_INTERVAL =
+    +(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 5000);
+
+var NO_WATCHER_POLLING_INTERVAL =
+    +(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 500);
+
+// This may seems like a long time to wait before actually closing the
+// file watchers, but it's to our advantage if they survive restarts.
+const WATCHER_CLEANUP_DELAY_MS = 30000;
+
+// Since linux doesn't have recursive file watching, nsfw has to walk the
+// watched folder and create a separate watcher for each subfolder. Until it has a
+// way for us to filter which folders it walks we will continue to use
+// pathwatcher to avoid having too many watchers.
+let watcherLibrary = process.env.METEOR_WATCHER_LIBRARY ||
+    (process.platform === 'linux' ? 'pathwatcher' : 'nsfw');
+
+// Pathwatcher complains (using console.error, ugh) if you try to watch
+// two files with the same stat.ino number but different paths on linux, so we have
+// to deduplicate files by ino.
+const DEDUPLICATE_BY_INO = watcherLibrary === 'pathwatcher';
+// Set METEOR_WATCH_FORCE_POLLING environment variable to a truthy value to
+// force the use of files.watchFile instead of watchLibrary.watch.
+let watcherEnabled = ! JSON.parse(
+    process.env.METEOR_WATCH_FORCE_POLLING || "false"
+);
+
+const entriesByIno = new Map;
+
+export type SafeWatcher = {
+    close: () => void;
+}
+
+type EntryCallback = (event: string) => void;
+
+interface Entry extends SafeWatcher {
+    callbacks: Set<EntryCallback>;
+    rewatch: () => void;
+    release: (callback: EntryCallback) => void;
+    _fire: (event: string) => void;
+}
+
+const entries: Record<string, Entry | null> = Object.create(null);
+
+// Folders that are watched recursively
+let watchRoots = new Set<string>();
+
+// Set of paths for which a change event has been fired, watched with
+// watchLibrary.watch if available. This could be an LRU cache, but in
+// practice it should never grow large enough for that to matter.
+const changedPaths = new Set;
+
+function hasPriority(absPath: string) {
+    // If we're not prioritizing changed files, then all files have
+    // priority, which means they should be watched with native file
+    // watchers if the platform supports them. If we are prioritizing
+    // changed files, then only changed files get priority.
+    return PRIORITIZE_CHANGED
+        ? changedPaths.has(absPath)
+        : true;
+}
+
+function acquireWatcher(absPath: string, callback: EntryCallback) {
+    const entry = entries[absPath] || (
+        entries[absPath] = startNewWatcher(absPath));
+
+    // Watches successfully established in the past may have become invalid
+    // because the watched file was deleted or renamed, so we need to make
+    // sure we're still watching every time we call safeWatcher.watch.
+    entry.rewatch();
+
+    // The size of the entry.callbacks Set also serves as a reference count
+    // for this watcher.
+    entry.callbacks.add(callback);
+
+    return entry;
+}
+
+function startNewWatcher(absPath: string): Entry {
+    let stat: Stats | BigIntStats | null | undefined = null;
+
+    if (DEDUPLICATE_BY_INO) {
+        stat = statOrNull(absPath);
+        if (stat && stat.ino > 0 && entriesByIno.has(stat.ino)) {
+            const entry = entriesByIno.get(stat.ino);
+            if (entries[absPath] === entry) {
+                return entry;
+            }
+        }
+    } else {
+        let entry = entries[absPath];
+        if (entry) {
+            return entry;
+        }
+    }
+
+    function safeUnwatch() {
+        if (watcher) {
+            watcher.close();
+            watcher = null;
+            if (stat && stat.ino > 0) {
+                entriesByIno.delete(stat.ino);
+            }
+        }
+    }
+
+    let lastWatcherEventTime = Date.now();
+    const callbacks = new Set<EntryCallback>();
+    let watcherCleanupTimer: ReturnType<typeof setTimeout> | null = null;
+    let watcher: FSWatcher | null = null;
+
+    // Determines the polling interval to be used for the fs.watchFile-based
+    // safety net that works on all platforms and file systems.
+    function getPollingInterval() {
+        if (hasPriority(absPath)) {
+            // Regardless of whether we have a native file watcher and it works
+            // correctly on this file system, poll prioritized files (that is,
+            // files that have been changed at least once) at a higher frequency
+            // (every 500ms by default).
+            return NO_WATCHER_POLLING_INTERVAL;
+        }
+
+        if (watcherEnabled || PRIORITIZE_CHANGED) {
+            // As long as native file watching is enabled (even if it doesn't
+            // work correctly) and the developer hasn't explicitly opted out of
+            // the file watching priority system, poll unchanged files at a
+            // lower frequency (every 5000ms by default).
+            return DEFAULT_POLLING_INTERVAL;
+        }
+
+        // If native file watching is disabled and the developer has
+        // explicitly opted out of the priority system, poll everything at the
+        // higher frequency (every 500ms by default). Note that this leads to
+        // higher idle CPU usage, so the developer may want to adjust the
+        // METEOR_WATCH_POLLING_INTERVAL_MS environment variable.
+        return NO_WATCHER_POLLING_INTERVAL;
+    }
+
+    function fire(event: string) {
+        if (event !== "change") {
+            // When we receive a "delete" or "rename" event, the watcher is
+            // probably not going to generate any more notifications for this
+            // file, so we close and nullify the watcher to ensure that
+            // entry.rewatch() will attempt to reestablish the watcher the next
+            // time we call safeWatcher.watch.
+            safeUnwatch();
+
+            // Make sure we don't throttle the watchFile callback after a
+            // "delete" or "rename" event, since it is now our only reliable
+            // source of file change notifications.
+            lastWatcherEventTime = 0;
+
+        } else {
+            changedPaths.add(absPath);
+            rewatch();
+        }
+
+        callbacks.forEach(cb => cb(event));
+    }
+
+    function watchWrapper(event: string) {
+        lastWatcherEventTime = Date.now();
+        fire(event);
+
+        // It's tempting to call unwatchFile(absPath, watchFileWrapper) here,
+        // but previous watcher success is no guarantee of future watcher
+        // reliability. For example, watchLibrary.watch works just fine when file
+        // changes originate from within a Vagrant VM, but changes to shared
+        // files made outside the VM are invisible to watcher, so our only
+        // hope of catching them is to continue polling.
+    }
+
+    function rewatch() {
+        if (hasPriority(absPath)) {
+            if (watcher) {
+                // Already watching; nothing to do.
+                return;
+            }
+            watcher = watchLibraryWatch(absPath, watchWrapper);
+        } else if (watcher) {
+            safeUnwatch();
+        }
+
+        // Since we're about to restart the stat-based file watcher, we don't
+        // want to miss any of its events because of the lastWatcherEventTime
+        // throttling that it attempts to do.
+        lastWatcherEventTime = 0;
+
+        // We use files.watchFile in addition to watcher.watch as a fail-safe
+        // to detect file changes even on network file systems.  However
+        // (unless the user disabled watcher or this watcher call failed), we
+        // use a relatively long default polling interval of 5000ms to save
+        // CPU cycles.
+        statWatch(absPath, getPollingInterval(), watchFileWrapper);
+    }
+
+    function watchFileWrapper(newStat: Stats, oldStat: Stats) {
+        if (newStat.ino === 0 &&
+            oldStat.ino === 0 &&
+            +newStat.mtime === +oldStat.mtime) {
+            // Node calls the watchFile listener once with bogus identical stat
+            // objects, which should not trigger a file change event.
+            return;
+        }
+
+        // If a watcher event fired in the last polling interval, ignore
+        // this event.
+        if (Date.now() - lastWatcherEventTime > getPollingInterval()) {
+            fire("change");
+        }
+    }
+
+    const entry = {
+        callbacks,
+        rewatch,
+
+        release(callback: EntryCallback) {
+            if (! entries[absPath]) {
+                return;
+            }
+
+            callbacks.delete(callback);
+            if (callbacks.size > 0) {
+                return;
+            }
+
+            // Once there are no more callbacks in the Set, close both watchers
+            // and nullify the shared data.
+            if (watcherCleanupTimer) {
+                clearTimeout(watcherCleanupTimer);
+            }
+
+            watcherCleanupTimer = setTimeout(() => {
+                if (callbacks.size > 0) {
+                    // If another callback was added while the timer was pending, we
+                    // can avoid tearing anything down.
+                    return;
+                }
+                entry.close();
+            }, WATCHER_CLEANUP_DELAY_MS);
+        },
+
+        close() {
+            if (entries[absPath] !== entry) return;
+            entries[absPath] = null;
+
+            if (watcherCleanupTimer) {
+                clearTimeout(watcherCleanupTimer);
+                watcherCleanupTimer = null;
+            }
+
+            safeUnwatch();
+
+            unwatchFile(absPath, watchFileWrapper);
+        },
+        _fire: fire
+    };
+
+    if (stat && stat.ino > 0) {
+        entriesByIno.set(stat.ino, entry);
+    }
+
+    return entry;
+}
+
+export function closeAllWatchers() {
+    Object.keys(entries).forEach(absPath => {
+        const entry = entries[absPath];
+        if (entry) {
+            entry.close();
+        }
+    });
+}
+
+const statWatchers = Object.create(null);
+
+function statWatch(
+    absPath: string,
+    interval: number,
+    callback: (current: Stats, previous: Stats) => void,
+) {
+    let statWatcher = statWatchers[absPath];
+
+    if (!statWatcher) {
+        statWatcher = {
+            interval,
+            changeListeners: [],
+            stat: null
+        };
+        statWatchers[absPath] = statWatcher;
+    }
+
+    // If the interval needs to be changed, replace the watcher.
+    // Node will only recreate the watcher with the new interval if all old
+    // watchers are stopped (which unwatchFile does when not passed a
+    // specific listener)
+    if (statWatcher.interval !== interval && statWatcher.stat) {
+        // This stops all stat watchers for the file, not just those created by
+        // statWatch
+        unwatchFile(absPath);
+        statWatcher.stat = null;
+        statWatcher.interval = interval;
+    }
+
+    if (!statWatcher.changeListeners.includes(callback)) {
+        statWatcher.changeListeners.push(callback);
+    }
+
+    if (!statWatcher.stat) {
+        const newStat = watchFile(absPath, {
+            persistent: false, // never persistent
+            interval,
+        }, (newStat, oldStat) => {
+            statWatcher.changeListeners.forEach((
+                listener: (newStat: Stats, oldStat: Stats) => void
+            ) => {
+                listener(newStat, oldStat);
+            });
+        });
+
+        newStat.on("stop", () => {
+            if (statWatchers[absPath] === statWatch) {
+                delete statWatchers[absPath];
+            }
+        });
+
+        statWatcher.stat = newStat;
+    }
+
+    return statWatcher;
+}
+
+function watchLibraryWatch(absPath: string, callback: EntryCallback) {
+    if (watcherEnabled && watcherLibrary === 'pathwatcher') {
+        try {
+            return pathwatcher.watch(convertToOSPath(absPath), callback);
+        } catch (e: any) {
+            maybeSuggestRaisingWatchLimit(e);
+            // ... ignore the error.  We'll still have watchFile, which is good
+            // enough.
+        }
+    }
+
+    return null;
+}
+
+let suggestedRaisingWatchLimit = false;
+
+function maybeSuggestRaisingWatchLimit(error: Error & { errno: number }) {
+    var constants = require('constants');
+    var archinfo = require('../utils/archinfo');
+    if (! suggestedRaisingWatchLimit &&
+        // Note: the not-super-documented require('constants') maps from
+        // strings to SYSTEM errno values. System errno values aren't the same
+        // as the numbers used internally by libuv! Once we're upgraded
+        // to Node 0.12, we'll have the system errno as a string (on 'code'),
+        // but the support for that wasn't in Node 0.10's uv.
+        // See our PR https://github.com/atom/node-pathwatcher/pull/53
+        // (and make sure to read the final commit message, not the original
+        // proposed PR, which had a slightly different interface).
+        error.errno === constants.ENOSPC &&
+        // The only suggestion we currently have is for Linux.
+        archinfo.matches(archinfo.host(), 'os.linux')) {
+
+        // Check suggestedRaisingWatchLimit again because archinfo.host() may
+        // have yielded.
+        if (suggestedRaisingWatchLimit) return;
+        suggestedRaisingWatchLimit = true;
+
+        var Console = require('../console/console.js').Console;
+        if (! Console.isHeadless()) {
+            Console.arrowWarn(
+                "It looks like a simple tweak to your system's configuration will " +
+                "make many tools (including this Meteor command) more efficient. " +
+                "To learn more, see " +
+                Console.url("https://github.com/meteor/docs/blob/master/long-form/file-change-watcher-efficiency.md"));
+        }
+    }
+}
+
+export const watch = Profile(
+    "safeWatcher.watch",
+    (absPath: string, callback: EntryCallback) => {
+        const entry = acquireWatcher(absPath, callback);
+        return {
+            close() {
+                entry.release(callback);
+            }
+        } as SafeWatcher;
+    }
+);
+
+const fireNames = {
+    [nsfw.actions.CREATED]: 'change',
+    [nsfw.actions.MODIFIED]: 'change',
+    [nsfw.actions.DELETED]: 'delete'
+}
+
+export function addWatchRoot(absPath: string) {
+    if (watchRoots.has(absPath) || watcherLibrary !== 'nsfw' || !watcherEnabled) {
+        return;
+    }
+
+    watchRoots.add(absPath);
+
+    // If there already is a watcher for a parent directory, there is no need
+    // to create this watcher.
+    for (const path of watchRoots) {
+        let relativePath = pathRelative(path, absPath);
+        if (
+            path !== absPath &&
+            !relativePath.startsWith('..') &&
+            !relativePath.startsWith('/')
+        ) {
+            return;
+        }
+    }
+
+    // TODO: check if there are any existing watchers that are children of this
+    // watcher and stop them
+
+    nsfw(
+        convertToOSPath(absPath),
+        (events) => {
+            events.forEach(event => {
+                if(event.action === nsfw.actions.RENAMED) {
+                    let oldPath = nativeJoin(event.directory, event.oldFile);
+                    let oldEntry = entries[toPosixPath(oldPath)];
+                    if (oldEntry) {
+                        oldEntry._fire('rename');
+                    }
+
+                    let path = nativeJoin(event.newDirectory, event.newFile);
+                    let newEntry = entries[toPosixPath(path)];
+                    if (newEntry) {
+                        newEntry._fire('change');
+                    }
+                } else {
+                    let path = nativeJoin(event.directory, event.file);
+                    let entry = entries[toPosixPath(path)];
+                    if (entry) {
+                        entry._fire(fireNames[event.action]);
+                    }
+                }
+            })
+        }
+    ).then(watcher => {
+        watcher.start()
+    });
+}

tools/fs/safe-watcher.ts

@@ -1,5 +1,6 @@
 import { Stats } from 'fs';
 import ParcelWatcher from "@parcel/watcher";
+import { watch as watchLegacy, addWatchRoot as addWatchRootLegacy, closeAllWatchers as closeAllWatchersLegacy } from './safe-watcher-legacy';
 
 import { Profile } from "../tool-env/profile";
 import { statOrNull, lstat, toPosixPath, convertToOSPath, pathRelative, watchFile, unwatchFile, pathResolve, pathDirname } from "./files";
@@ -380,6 +381,11 @@ export const watch = Profile(
     absPath: string,
     callback: ChangeCallback
     ): SafeWatcher => {
+      // @ts-ignore
+      if (!global.modernWatcher) {
+        // @ts-ignore
+        return watchLegacy(absPath, callback);
+      }
       absPath = toPosixPath(absPath);
 
       // If the path should be ignored, immediately return a noop SafeWatcher.
@@ -428,6 +434,12 @@ export const watch = Profile(
  * If the provided path is a file, its parent directory is used.
  */
 export function addWatchRoot(absPath: string) {
+  // @ts-ignore
+  if (!global.modernWatcher) {
+    // @ts-ignore
+    return addWatchRootLegacy(absPath);
+  }
+
   absPath = toPosixPath(absPath);
   let watchTarget = absPath;
   try {
@@ -455,6 +467,11 @@ async function safeUnsubscribeSub(root: string) {
 }
 
 export async function closeAllWatchers() {
+  // @ts-ignore
+  if (!global.modernWatcher) {
+    // @ts-ignore
+    return closeAllWatchersLegacy();
+  }
   for (const root of Array.from(watchRoots)) {
     await safeUnsubscribeSub(root);
   }