meteor/tools/fs/safe-watcher.ts

import { FSWatcher, Stats, BigIntStats } from "fs";
import { Profile } from "../tool-env/profile";
import {
  statOrNull,
  convertToOSPath,
  watchFile,
  unwatchFile,
  toPosixPath,
  pathRelative
} from "./files";
import {
  join as nativeJoin
} from 'path';
import nsfw from 'vscode-nsfw';

const pathwatcher = require('pathwatcher');

// Default to prioritizing changed files, but disable that behavior (and
// thus prioritize all files equally) if METEOR_WATCH_PRIORITIZE_CHANGED
// is explicitly set to a string that parses to a falsy value.
var PRIORITIZE_CHANGED = true;
if (process.env.METEOR_WATCH_PRIORITIZE_CHANGED &&
    ! JSON.parse(process.env.METEOR_WATCH_PRIORITIZE_CHANGED)) {
  PRIORITIZE_CHANGED = false;
}

var DEFAULT_POLLING_INTERVAL =
  +(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 5000);

var NO_WATCHER_POLLING_INTERVAL =
  +(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 500);

// This may seems like a long time to wait before actually closing the
// file watchers, but it's to our advantage if they survive restarts.
const WATCHER_CLEANUP_DELAY_MS = 30000;

// Since linux doesn't have recursive file watching, nsfw has to walk the
// watched folder and create a separate watcher for each subfolder. Until it has a
// way for us to filter which folders it walks we will continue to use
// pathwatcher to avoid having too many watchers.
let watcherLibrary = process.env.METEOR_WATCHER_LIBRARY ||
  (process.platform === 'linux' ? 'pathwatcher' : 'nsfw');

// Pathwatcher complains (using console.error, ugh) if you try to watch
// two files with the same stat.ino number but different paths on linux, so we have
// to deduplicate files by ino.
const DEDUPLICATE_BY_INO = watcherLibrary === 'pathwatcher';
// Set METEOR_WATCH_FORCE_POLLING environment variable to a truthy value to
// force the use of files.watchFile instead of watchLibrary.watch.
let watcherEnabled = ! JSON.parse(
  process.env.METEOR_WATCH_FORCE_POLLING || "false"
);

const entriesByIno = new Map;

export type SafeWatcher = {
  close: () => void;
}

type EntryCallback = (event: string) => void;

interface Entry extends SafeWatcher {
  callbacks: Set<EntryCallback>;
  rewatch: () => void;
  release: (callback: EntryCallback) => void;
  _fire: (event: string) => void;
}

const entries: Record<string, Entry | null> = Object.create(null);

// Folders that are watched recursively
let watchRoots = new Set<string>();

// Set of paths for which a change event has been fired, watched with
// watchLibrary.watch if available. This could be an LRU cache, but in
// practice it should never grow large enough for that to matter.
const changedPaths = new Set;

function hasPriority(absPath: string) {
  // If we're not prioritizing changed files, then all files have
  // priority, which means they should be watched with native file
  // watchers if the platform supports them. If we are prioritizing
  // changed files, then only changed files get priority.
  return PRIORITIZE_CHANGED
    ? changedPaths.has(absPath)
    : true;
}

function acquireWatcher(absPath: string, callback: EntryCallback) {
  const entry = entries[absPath] || (
    entries[absPath] = startNewWatcher(absPath));

  // Watches successfully established in the past may have become invalid
  // because the watched file was deleted or renamed, so we need to make
  // sure we're still watching every time we call safeWatcher.watch.
  entry.rewatch();

  // The size of the entry.callbacks Set also serves as a reference count
  // for this watcher.
  entry.callbacks.add(callback);

  return entry;
}

function startNewWatcher(absPath: string): Entry {
  let stat: Stats | BigIntStats | null | undefined = null;

  if (DEDUPLICATE_BY_INO) {
    stat = statOrNull(absPath);
    if (stat && stat.ino > 0 && entriesByIno.has(stat.ino)) {
      const entry = entriesByIno.get(stat.ino);
      if (entries[absPath] === entry) {
        return entry;
      }
    }
  } else {
    let entry = entries[absPath];
    if (entry) {
      return entry;
    }
  }

  function safeUnwatch() {
    if (watcher) {
      watcher.close();
      watcher = null;
      if (stat && stat.ino > 0) {
        entriesByIno.delete(stat.ino);
      }
    }
  }

  let lastWatcherEventTime = Date.now();
  const callbacks = new Set<EntryCallback>();
  let watcherCleanupTimer: ReturnType<typeof setTimeout> | null = null;
  let watcher: FSWatcher | null = null;

  // Determines the polling interval to be used for the fs.watchFile-based
  // safety net that works on all platforms and file systems.
  function getPollingInterval() {
    if (hasPriority(absPath)) {
      // Regardless of whether we have a native file watcher and it works
      // correctly on this file system, poll prioritized files (that is,
      // files that have been changed at least once) at a higher frequency
      // (every 500ms by default).
      return NO_WATCHER_POLLING_INTERVAL;
    }

    if (watcherEnabled || PRIORITIZE_CHANGED) {
      // As long as native file watching is enabled (even if it doesn't
      // work correctly) and the developer hasn't explicitly opted out of
      // the file watching priority system, poll unchanged files at a
      // lower frequency (every 5000ms by default).
      return DEFAULT_POLLING_INTERVAL;
    }

    // If native file watching is disabled and the developer has
    // explicitly opted out of the priority system, poll everything at the
    // higher frequency (every 500ms by default). Note that this leads to
    // higher idle CPU usage, so the developer may want to adjust the
    // METEOR_WATCH_POLLING_INTERVAL_MS environment variable.
    return NO_WATCHER_POLLING_INTERVAL;
  }

  function fire(event: string) {
    if (event !== "change") {
      // When we receive a "delete" or "rename" event, the watcher is
      // probably not going to generate any more notifications for this
      // file, so we close and nullify the watcher to ensure that
      // entry.rewatch() will attempt to reestablish the watcher the next
      // time we call safeWatcher.watch.
      safeUnwatch();

      // Make sure we don't throttle the watchFile callback after a
      // "delete" or "rename" event, since it is now our only reliable
      // source of file change notifications.
      lastWatcherEventTime = 0;

    } else {
      changedPaths.add(absPath);
      rewatch();
    }

    callbacks.forEach(cb => cb(event));
  }

  function watchWrapper(event: string) {
    lastWatcherEventTime = Date.now();
    fire(event);

    // It's tempting to call unwatchFile(absPath, watchFileWrapper) here,
    // but previous watcher success is no guarantee of future watcher
    // reliability. For example, watchLibrary.watch works just fine when file
    // changes originate from within a Vagrant VM, but changes to shared
    // files made outside the VM are invisible to watcher, so our only
    // hope of catching them is to continue polling.
  }

  function rewatch() {
    if (hasPriority(absPath)) {
      if (watcher) {
        // Already watching; nothing to do.
        return;
      }
      watcher = watchLibraryWatch(absPath, watchWrapper);
    } else if (watcher) {
      safeUnwatch();
    }

    // Since we're about to restart the stat-based file watcher, we don't
    // want to miss any of its events because of the lastWatcherEventTime
    // throttling that it attempts to do.
    lastWatcherEventTime = 0;

    // We use files.watchFile in addition to watcher.watch as a fail-safe
    // to detect file changes even on network file systems.  However
    // (unless the user disabled watcher or this watcher call failed), we
    // use a relatively long default polling interval of 5000ms to save
    // CPU cycles.
    statWatch(absPath, getPollingInterval(), watchFileWrapper);
  }

  function watchFileWrapper(newStat: Stats, oldStat: Stats) {
    if (newStat.ino === 0 &&
        oldStat.ino === 0 &&
        +newStat.mtime === +oldStat.mtime) {
      // Node calls the watchFile listener once with bogus identical stat
      // objects, which should not trigger a file change event.
      return;
    }

    // If a watcher event fired in the last polling interval, ignore
    // this event.
    if (Date.now() - lastWatcherEventTime > getPollingInterval()) {
      fire("change");
    }
  }

  const entry = {
    callbacks,
    rewatch,

    release(callback: EntryCallback) {
      if (! entries[absPath]) {
        return;
      }

      callbacks.delete(callback);
      if (callbacks.size > 0) {
        return;
      }

      // Once there are no more callbacks in the Set, close both watchers
      // and nullify the shared data.
      if (watcherCleanupTimer) {
        clearTimeout(watcherCleanupTimer);
      }

      watcherCleanupTimer = setTimeout(() => {
        if (callbacks.size > 0) {
          // If another callback was added while the timer was pending, we
          // can avoid tearing anything down.
          return;
        }
        entry.close();
      }, WATCHER_CLEANUP_DELAY_MS);
    },

    close() {
      if (entries[absPath] !== entry) return;
      entries[absPath] = null;

      if (watcherCleanupTimer) {
        clearTimeout(watcherCleanupTimer);
        watcherCleanupTimer = null;
      }

      safeUnwatch();

      unwatchFile(absPath, watchFileWrapper);
    },
    _fire: fire
  };

  if (stat && stat.ino > 0) {
    entriesByIno.set(stat.ino, entry);
  }

  return entry;
}

export function closeAllWatchers() {
  Object.keys(entries).forEach(absPath => {
    const entry = entries[absPath];
    if (entry) {
      entry.close();
    }
  });
}

const statWatchers = Object.create(null);

function statWatch(
  absPath: string,
  interval: number,
  callback: (current: Stats, previous: Stats) => void,
) {
  let statWatcher = statWatchers[absPath];

  if (!statWatcher) {
    statWatcher = {
      interval,
      changeListeners: [],
      stat: null
    };
    statWatchers[absPath] = statWatcher;
  }

  // If the interval needs to be changed, replace the watcher.
  // Node will only recreate the watcher with the new interval if all old
  // watchers are stopped (which unwatchFile does when not passed a
  // specific listener)
  if (statWatcher.interval !== interval && statWatcher.stat) {
    // This stops all stat watchers for the file, not just those created by
    // statWatch
    unwatchFile(absPath);
    statWatcher.stat = null;
    statWatcher.interval = interval;
  }

  if (!statWatcher.changeListeners.includes(callback)) {
    statWatcher.changeListeners.push(callback);
  }

  if (!statWatcher.stat) {
    const newStat = watchFile(absPath, {
      persistent: false, // never persistent
      interval,
    }, (newStat, oldStat) => {
      statWatcher.changeListeners.forEach((
        listener: (newStat: Stats, oldStat: Stats) => void
      ) => {
          listener(newStat, oldStat);
      });
    });

    newStat.on("stop", () => {
      if (statWatchers[absPath] === statWatch) {
        delete statWatchers[absPath];
      }
    });

    statWatcher.stat = newStat;
  }

  return statWatcher;
}

function watchLibraryWatch(absPath: string, callback: EntryCallback) {
  if (watcherEnabled && watcherLibrary === 'pathwatcher') {
    try {
      return pathwatcher.watch(convertToOSPath(absPath), callback);
    } catch (e: any) {
      maybeSuggestRaisingWatchLimit(e);
      // ... ignore the error.  We'll still have watchFile, which is good
      // enough.
    }
  }

  return null;
}

let suggestedRaisingWatchLimit = false;

function maybeSuggestRaisingWatchLimit(error: Error & { errno: number }) {
  var constants = require('constants');
  var archinfo = require('../utils/archinfo');
  if (! suggestedRaisingWatchLimit &&
      // Note: the not-super-documented require('constants') maps from
      // strings to SYSTEM errno values. System errno values aren't the same
      // as the numbers used internally by libuv! Once we're upgraded
      // to Node 0.12, we'll have the system errno as a string (on 'code'),
      // but the support for that wasn't in Node 0.10's uv.
      // See our PR https://github.com/atom/node-pathwatcher/pull/53
      // (and make sure to read the final commit message, not the original
      // proposed PR, which had a slightly different interface).
      error.errno === constants.ENOSPC &&
      // The only suggestion we currently have is for Linux.
      archinfo.matches(archinfo.host(), 'os.linux')) {

    // Check suggestedRaisingWatchLimit again because archinfo.host() may
    // have yielded.
    if (suggestedRaisingWatchLimit) return;
    suggestedRaisingWatchLimit = true;

    var Console = require('../console/console.js').Console;
    if (! Console.isHeadless()) {
      Console.arrowWarn(
        "It looks like a simple tweak to your system's configuration will " +
          "make many tools (including this Meteor command) more efficient. " +
          "To learn more, see " +
          Console.url("https://github.com/meteor/docs/blob/master/long-form/file-change-watcher-efficiency.md"));
    }
  }
}

export const watch = Profile(
  "safeWatcher.watch",
  (absPath: string, callback: EntryCallback) => {
    const entry = acquireWatcher(absPath, callback);
    return {
      close() {
        entry.release(callback);
      }
    } as SafeWatcher;
  }
);

const fireNames = {
  [nsfw.actions.CREATED]: 'change',
  [nsfw.actions.MODIFIED]: 'change',
  [nsfw.actions.DELETED]: 'delete'
}

export function addWatchRoot(absPath: string) {
  if (watchRoots.has(absPath) || watcherLibrary !== 'nsfw' || !watcherEnabled) {
    return;
  }

  watchRoots.add(absPath);

  // If there already is a watcher for a parent directory, there is no need
  // to create this watcher.
  for (const path of watchRoots) {
    let relativePath = pathRelative(path, absPath);
    if (
      path !== absPath &&
      !relativePath.startsWith('..') &&
      !relativePath.startsWith('/')
    ) {
      return;
    }
  }

  // TODO: check if there are any existing watchers that are children of this
  // watcher and stop them

  nsfw(
    convertToOSPath(absPath),
    (events) => {
      events.forEach(event => {
        if(event.action === nsfw.actions.RENAMED) {
          let oldPath = nativeJoin(event.directory, event.oldFile);
          let oldEntry = entries[toPosixPath(oldPath)];
          if (oldEntry) {
            oldEntry._fire('rename');
          }

          let path = nativeJoin(event.newDirectory, event.newFile);
          let newEntry = entries[toPosixPath(path)];
          if (newEntry) {
            newEntry._fire('change');
          }
        } else {
            let path = nativeJoin(event.directory, event.file);
            let entry = entries[toPosixPath(path)];
            if (entry) {
              entry._fire(fireNames[event.action]);
            }
        }
      })
    }
  ).then(watcher => {
    watcher.start()
  });
}