meteor/tools/fs/watch.ts

import { Stats, BigIntStats, FSWatcher, Dirent } from "fs";
import * as files from "./files";
import * as safeWatcher from "./safe-watcher";
import { createHash } from "crypto";
import { coalesce } from "../utils/func-utils";
import { Profile } from "../tool-env/profile";
import {
  optimisticHashOrNull,
  optimisticStatOrNull,
} from "./optimistic";

const _ = require("underscore");

const WATCH_COALESCE_MS =
  +(process.env.METEOR_FILE_WATCH_COALESCE_MS || 100);

// Watch for changes to a set of files, and the first time that any of
// the files change, call a user-provided callback. (If you want a
// second callback, you'll need to create a second Watcher.)
//
// You describe the structure you want to watch in a WatchSet; you then create a
// Watcher to watch it. Watcher does not mutate WatchSet, so you can create
// several Watchers from the same WatchSet. WatchSet can be easily converted to
// and from JSON for serialization.
//
// You can set up two kinds of watches, file and directory watches.
//
// In a file watch, you provide an absolute path to a file and a SHA1 (encoded
// as hex) of the contents of that file. If the file ever changes so that its
// contents no longer match that SHA1, the callback triggers. You can also
// provide `null` for the SHA1, which means the file should not exist.
//
// In a directory watch, you provide an absolute path to a directory,
// two lists of regular expressions specifying the entries to
// include and exclude, a list of specific names to include (which ignores
// the exclude regexp list) and an array of which entries to expect.
//
// For directory watches, the regular expressions work as follows. You provide
// two arrays of regular expressions, an include list and an exclude list. An
// entry in the directory matches if it matches at least one regular expression
// in the include list, and doesn't match any regular expressions in the exclude
// list. The string that is matched against the regular expression ends with a
// '/' if the entry is directory. There is NO IMPLICIT RECURSION here: a
// directory watch ONLY watches the immediate children of the directory! If you
// want a recursive watch, you need to do the recursive walk while building the
// WatchSet and add a bunch of separate directory watches.  In addition, you
// can provide a list of specific names to expect; these are not filtered
// by the exclude list.  (For example, you might want to see all "*.js" files
// but ignore files starting with dots (which are often temporary files), but
// explicitly ask for ".jshintrc".)
//
// There can be multiple directory watches on the same directory. There is no
// relationship between the files found in directory watches and the files
// watched by file watches; they are parallel mechanisms.
//
// Regular expressions are checked only against individual path components (the
// actual name of the file or the subdirectory) plus the trailing '/' for
// directories, not against the entire path.
//
// You can call stop() to stop watching and tear down the
// watcher. Calling stop() guarantees that you will not receive a
// callback (if you have not already). Calling stop() is unnecessary
// if you've received a callback.
//
// To do a "one-shot" (to see if any files have been modified, compared to the
// dependencies, at a particular point in time), use the isUpToDate function.
//
// XXX Symlinks are currently treated transparently: we treat them as the thing
// they point to (ie, as a directory if they point to a directory, as
// nonexistent if they point to something nonexist, etc). Not sure if this is
// correct.

type DirectoryEntry = {
  absPath: string;
  include: RegExp[];
  exclude: RegExp[];
  names: string[];
  contents: string[] | null;
}

export class WatchSet {
  // Set this to true if any Watcher built on this WatchSet must immediately
  // fire (eg, if this WatchSet was given two different sha1 for the same file).
  public alwaysFire = false;

  // Map from the absolute path to a file, to a sha1 hash, or null if the file
  // should not exist. A Watcher created from this set fires when the file
  // changes from that sha, or is deleted (if non-null) or created (if null).
  //
  // Note that Isopack.getSourceFilesUnderSourceRoot() depends on this field
  // existing (it's not just an internal implementation detail of watch.ts).
  public readonly files: Record<string, string | null> = Object.create(null);

  // This represents the assertion that 'absPath' is a directory and that
  // 'contents' is its immediate contents, as filtered by the regular
  // expressions.  Entries in 'contents' are file and subdirectory names;
  // directory names end with '/'. 'contents' is sorted. An entry is in
  // 'contents' if its value (including the slash, for directories) matches at
  // least one regular expression in 'include' and no regular expressions in
  // 'exclude'... or if it is in 'names'.
  //
  // There is no recursion here: files contained in subdirectories never appear.
  //
  // A directory may have multiple entries (presumably with different
  // include/exclude filters).
  public readonly directories: DirectoryEntry[] = [];

  public addFile(filePath: string, hash: string | null) {
    // Calling addFile directly instead of addPotentiallyUnusedFile implies
    // that the file will be used, so we can mark it as such here. Note that
    // addPotentiallyUnusedFile overrides this value based on the alreadyUsed
    // variable after calling this.addFile.
    this.potentiallyUnusedFiles.delete(filePath);

    // No need to update if this is in always-fire mode already.
    if (this.alwaysFire) {
      return;
    }

    if (_.has(this.files, filePath)) {
      // Redundant?
      if (this.files[filePath] === hash) {
        return;
      }
      // Nope, inconsistent.
      this.alwaysFire = true;
      return;
    }

    this.files[filePath] = hash;
  }

  public hasFile(filePath: string): boolean {
    return _.has(this.files, filePath);
  }

  // Files added via addPotentiallyUnusedFile will be included in this Set
  // until addFile is called at a later time, which removes them from the
  // potentiallyUnusedFiles Set and prevents them from being considered as
  // potentially unused in the future. Either way, this.files will have
  // the same contents as if addFile was called instead, which is
  // important for checks such as IsopackCache._checkUpToDate, which need
  // to take everything (even potentially unused files) into account.
  private potentiallyUnusedFiles = new Set<string>();

  public isDefinitelyUsed(filePath: string): boolean {
    return this.hasFile(filePath) && ! this.isPotentiallyUnused(filePath);
  }

  public isPotentiallyUnused(filePath: string): boolean {
    return this.potentiallyUnusedFiles.has(filePath);
  }

  public addPotentiallyUnusedFile(filePath: string, hash: string | null) {
    const alreadyUsed = this.isDefinitelyUsed(filePath);
    this.addFile(filePath, hash);
    if (! alreadyUsed) {
      this.potentiallyUnusedFiles.add(filePath);
    }
  }

  public addDirectory({
    absPath,
    include,
    exclude,
    names,
    contents,
  }: DirectoryEntry) {
    if (this.alwaysFire) return;
    if (_.isEmpty(include) && _.isEmpty(names)) {
      return;
    }
    this.directories.push({
      absPath,
      include,
      exclude,
      names,
      contents: contents && contents.slice(0).sort(),
    });
  }

  // Merges another WatchSet into this one. This one will now fire if either
  // WatchSet would have fired.
  public merge(that: WatchSet) {
    if (this.alwaysFire) return;
    if (that.alwaysFire) {
      this.alwaysFire = true;
      return;
    }

    Object.keys(that.files).forEach(name => {
      if (that.isPotentiallyUnused(name)) {
        this.addPotentiallyUnusedFile(name, that.files[name]);
      } else {
        this.addFile(name, that.files[name]);
      }
    });

    that.directories.forEach(dir => {
      // XXX this doesn't deep-clone the directory, but I think these objects
      // are never mutated #WatchSetShallowClone #TypeScriptOpportunity
      this.directories.push(dir);
    });
  }

  public clone() {
    const ret = new WatchSet();

    ret.alwaysFire = this.alwaysFire;

    Object.keys(this.files).forEach(name => {
      ret.files[name] = this.files[name];
    });

    this.potentiallyUnusedFiles.forEach(name => {
      ret.potentiallyUnusedFiles.add(name);
    });

    // XXX doesn't bother to deep-clone the directory info
    // #WatchSetShallowClone
    this.directories.forEach(entry => {
      ret.directories.push(entry);
    });

    return ret;
  }

  toJSON() {
    if (this.alwaysFire) {
      return { alwaysFire: true };
    }

    function reToJSON(r: RegExp) {
      let options = '';
      if (r.ignoreCase) {
        options += 'i';
      }
      if (r.multiline) {
        options += 'm';
      }
      if (r.global) {
        options += 'g';
      }
      if (options) {
        return {
          $regex: r.source,
          $options: options,
        };
      }
      return r.source;
    }

    const potentiallyUnusedFiles: string[] = [];
    this.potentiallyUnusedFiles.forEach(name => {
      potentiallyUnusedFiles.push(name);
    });

    return {
      files: this.files,
      potentiallyUnusedFiles,
      directories: this.directories.map(d => ({
        absPath: d.absPath,
        include: d.include.map(reToJSON),
        exclude: d.exclude.map(reToJSON),
        names: d.names,
        contents: d.contents
      })),
    };
  }

  static fromJSON(json: any) {
    const watchSet = new WatchSet();

    if (! json) {
      return watchSet;
    }

    if (json.alwaysFire) {
      watchSet.alwaysFire = true;
      return watchSet;
    }

    Object.keys(json.files).forEach(name => {
      watchSet.files[name] = json.files[name];
    });

    if (Array.isArray(json.potentiallyUnusedFiles)) {
      json.potentiallyUnusedFiles.forEach((name: string) => {
        watchSet.potentiallyUnusedFiles.add(name);
      });
    }

    function reFromJSON(j: any) {
      if (j.$regex) {
        return new RegExp(j.$regex, j.$options);
      }
      return new RegExp(j);
    }

    json.directories.forEach((d: any) => {
      watchSet.directories.push({
        absPath: d.absPath,
        include: d.include.map(reFromJSON),
        exclude: d.exclude.map(reFromJSON),
        names: d.names,
        contents: d.contents,
      })
    });

    return watchSet;
  }
}

export function readFile(absPath: string) {
  try {
    return files.readFile(absPath);
  } catch (e: any) {
    // Rethrow most errors.
    if (! e || (e.code !== 'ENOENT' && e.code !== 'EISDIR')) {
      throw e;
    }
    // File does not exist (or is a directory).
    return null;
  }
};

export const sha1 = Profile("sha1", function (...args: (string | Buffer)[]) {
  const hash = createHash('sha1');
  args.forEach(arg => hash.update(arg));
  return hash.digest('hex');
});

export const sha512 = Profile("sha512", function (...args: (string | Buffer)[]) {
  const hash = createHash('sha512');
  args.forEach(arg => hash.update(arg));
  return hash.digest('base64');
});

function readAndStatDirectory(absPath: string) {
  // Read the directory.
  try {
    var contents = files.readdirWithTypes(absPath);
  } catch (e: any) {
    // If the path is not a directory, return null; let other errors through.
    if (e && (e.code === 'ENOENT' || e.code === 'ENOTDIR')) {
      return null;
    }
    throw e;
  }

  // Add slashes to the end of directories.
  const contentsWithSlashes: string[] = [];
  contents.forEach(entry => {
    let stat: Dirent | Stats | BigIntStats | null | undefined = entry;
    let name = entry.name;

    if (entry.isSymbolicLink()) {
      // We do stat instead of lstat here, so that we treat symlinks to
      // directories just like directories themselves.
      stat = optimisticStatOrNull(files.pathJoin(absPath, entry.name));
    }
    if (! stat) {
      // Disappeared after the readdir (or a dangling symlink)?
      // Eh, pretend it was never there in the first place.
      return;
    }

    if (stat.isDirectory()) {
      name += '/';
    }

    contentsWithSlashes.push(name);
  });

  return contentsWithSlashes;
}

function filterDirectoryContents(contents: string[], {
  include,
  exclude,
  names,
}: {
  include?: RegExp[],
  exclude?: RegExp[],
  names?: string[],
}) {
  // Filter based on regexps.
  return contents.filter(entry => {
    // Is it one of the names we explicitly requested?
    if (names && names.indexOf(entry) !== -1) {
      return true;
    }
    // Is it ruled out by an exclude rule?
    if (exclude && exclude.some(re => re.test(entry))) {
      return false;
    }
    // Is it ruled in by an include rule?
    if (include && include.some(re => re.test(entry))) {
      return true;
    }
    return false;
  }).sort();
}

export function readDirectory({ absPath, include, exclude, names }: {
  absPath: string;
  include?: RegExp[],
  exclude?: RegExp[],
  names?: string[],
}) {
  const contents = readAndStatDirectory(absPath);
  return contents ? filterDirectoryContents(contents, {
    include, exclude, names
  }) : [];
}

// All fields are private.
export class Watcher {
  private watchSet: WatchSet;
  private onChange: () => any;
  private stopped = false;
  private justCheckOnce = false;
  private async = false;
  private includePotentiallyUnusedFiles = true;

  private watches: Record<string, {
    // Null until safeWatcher.watch succeeds in watching the file.
    watcher: safeWatcher.SafeWatcher | null;
    // Undefined until we stat the file for the first time, then null
    // if the file is observed to be missing.
    lastStat?: Stats | BigIntStats | null
  }> = Object.create(null);

  constructor(options: {
    watchSet: WatchSet;
    onChange: () => any;
    async?: boolean;
    justCheckOnce?: boolean;
    includePotentiallyUnusedFiles?: boolean;
  }) {
    this.async = !! options.async;
    this.watchSet = options.watchSet;
    this.onChange = options.onChange;
    this.justCheckOnce = !! options.justCheckOnce;
    if (options.includePotentiallyUnusedFiles === false) {
      this.includePotentiallyUnusedFiles = false;
    }

    // Were we given an inconsistent WatchSet? Fire now and be done with it.
    if (this.watchSet.alwaysFire) {
      this.fire();
      return;
    }

    this.startFileWatches();
    this.checkDirectories();
  }

  private fireIfFileChanged(absPath: string) {
    if (this.stopped) {
      return true;
    }

    if (
      ! this.includePotentiallyUnusedFiles &&
      this.watchSet.isPotentiallyUnused(absPath)
    ) {
      return false;
    }

    const oldHash = this.watchSet.files[absPath];
    if (typeof oldHash === "undefined") {
      throw new Error("Checking unknown file " + absPath);
    }

    const newHash = optimisticHashOrNull(absPath);
    if (newHash === null) {
      // File does not exist (or is a directory).
      // Is this what we expected?
      if (oldHash === null) {
        return false;
      }
      // Nope, not what we expected.
      this.fire();
      return true;
    }

    // File exists! Is that what we expected?
    if (oldHash === null) {
      this.fire();
      return true;
    }

    // Unchanged?
    if (newHash === oldHash) {
      return false;
    }

    this.fire();
    return true;
  }

  // infos must all be for the same directory
  private fireIfDirectoryChanged(infos: DirectoryEntry[]) {
    if (this.stopped) {
      return true;
    }

    const contents = readAndStatDirectory(infos[0].absPath);

    for (const info of infos) {
      const newContents = filterDirectoryContents(contents || [], info);

      // If the directory has changed (including being deleted or created).
      if (! _.isEqual(info.contents, newContents)) {
        this.fire();
        return true;
      }
    }

    return false;
  }

  private startFileWatches() {
    const keys = Object.keys(this.watchSet.files);

    // Set up a watch for each file
    this.processBatches(keys, absPath => {
      if (! this.justCheckOnce) {
        this.watchFileOrDirectory(absPath, true);
      }

      // Check for the case where by the time we created the watch,
      // the file had already changed from the sha we were provided.
      this.fireIfFileChanged(absPath);
    });
  }

  private watchFileOrDirectory(absPath: string, skipCheck = false) {
    if (! _.has(this.watches, absPath)) {
      this.watches[absPath] = {
        watcher: null,
        // Initially undefined (instead of null) to indicate we have never
        // called files.stat on this file before.
        lastStat: undefined
      };
    }

    const entry = this.watches[absPath];
    if (entry.watcher) {
      // Already watching this path.
      return;
    }

    if (files.statOrNull(absPath)) {
      if (this.mustNotExist(absPath)) {
        this.fire();
        return;
      }

      const onWatchEvent = this.makeWatchEventCallback(absPath);
      entry.watcher = safeWatcher.watch(absPath, onWatchEvent);

      if (!skipCheck) {
        // If we successfully created the watcher, invoke the callback
        // immediately, so that we examine this file at least once.
        onWatchEvent();
      } else {
        this.updateStatForWatch(absPath);
      }

    } else {
      if (this.mustBeAFile(absPath)) {
        this.fire();
        return;
      }

      const parentDir = files.pathDirname(absPath);
      if (parentDir === absPath) {
        throw new Error("Unable to watch parent directory of " + absPath);
      }

      this.watchFileOrDirectory(parentDir);
    }
  }

  private makeWatchEventCallback(absPath: string) {
    // Sometimes we receive a rapid succession of change events, perhaps
    // because several files were modified at once (e.g. by git reset
    // --hard), or a file was deleted and then recreated by an editor like
    // Vim. Because detecting changes can be costly, and because we care
    // most about the settled state of the file system, we use the
    // funcUtils.coalesce helper to delay calls to the callback by
    // METEOR_FILE_WATCH_COALESCE_MS or 100 milliseconds, canceling any
    // additional calls if they happen within that window of time, so that
    // a rapid succession of calls will tend to trigger only one inspection
    // of the file system.
    return coalesce<WatchSet>(WATCH_COALESCE_MS, () => {
      if (this.stopped) {
        return;
      }

      // This helper method will call this._fire() if the old and new stat
      // objects have different types (missing, file, or directory), so we
      // can assume they have the same type for the rest of this method.
      const stat = this.updateStatForWatch(absPath);
      if (this.stopped) {
        return;
      }

      if (stat === null || stat.isFile()) {
        if (_.has(this.watchSet.files, absPath)) {
          this.fireIfFileChanged(absPath);
          // XXX #3335 We probably should check again in a second, due to low
          // filesystem modtime resolution.
        }

      } else if (stat.isDirectory()) {
        try {
          var dirFiles = files.readdir(absPath);
        } catch (err: any) {
          if (err.code === "ENOENT" ||
              err.code === "ENOTDIR") {
            // The directory was removed or changed type since we called
            // this._updateStatForWatch, so we fire unconditionally.
            this.fire();
            return;
          }
          throw err;
        }

        dirFiles.forEach(file => {
          const fullPath = files.pathJoin(absPath, file);

          // Recursively watch new files, if we ever previously tried to
          // watch them. Recall that when we attempt to watch a
          // non-existent file, we actually watch the closest enclosing
          // directory that exists, so once the file (and/or any
          // intermediate directories) are created, we begin watching
          // those directories in response to change events fired for
          // directories we're already watching.
          if (_.has(this.watches, fullPath)) {
            this.watchFileOrDirectory(fullPath);
          }
        });

        // If this.watchSet.directories contains any entries for the
        // directory we are examining, call this._fireIfDirectoryChanged.
        const infos = this.watchSet.directories.filter(info => info.absPath === absPath );
        if (infos.length) {
          this.fireIfDirectoryChanged(infos);
        }
        // XXX #3335 We probably should check again in a second, due to low
        // filesystem modtime resolution.
      }
    });
  }

  private mustNotExist(absPath: string) {
    const wsFiles = this.watchSet.files;
    if (_.has(wsFiles, absPath)) {
      return wsFiles[absPath] === null;
    }
    return false;
  }

  private mustBeAFile(absPath: string) {
    const wsFiles = this.watchSet.files;
    if (_.has(wsFiles, absPath)) {
      return _.isString(wsFiles[absPath]);
    }
    return false;
  }

  private updateStatForWatch(absPath: string) {
    const entry = this.watches[absPath];
    const lastStat = entry.lastStat;
    let stat = files.statOrNull(absPath);
    const mustNotExist = this.mustNotExist(absPath);
    const mustBeAFile = this.mustBeAFile(absPath);

    if (stat && lastStat === undefined) {
      // We have not checked for this file before, so our expectations are
      // somewhat relaxed (namely, we don't care about lastStat), but
      // this._fire() might still need to be called if this.watchSet.files
      // has conflicting expectations.
      if (stat.isFile()) {
        if (mustNotExist) {
          this.fire();
        }
      } else if (stat.isDirectory()) {
        if (mustNotExist || mustBeAFile) {
          this.fire();
        }
      } else {
        // Neither a file nor a directory, so treat as non-existent.
        stat = null;
        if (mustBeAFile) {
          this.fire();
        }
      }

      // We have not checked for this file before, so just record the new
      // stat object.
      entry.lastStat = stat;

    } else if (stat && stat.isFile()) {
      entry.lastStat = stat;
      if (! lastStat || ! lastStat.isFile()) {
        this.fire();
      }

    } else if (stat && stat.isDirectory()) {
      entry.lastStat = stat;
      if (! lastStat || ! lastStat.isDirectory()) {
        this.fire();
      }

    } else {
      entry.lastStat = stat = null;
      if (lastStat) {
        this.fire();
      }
    }

    return stat;
  }

  // Iterates over the array, calling handleItem for each item.
  // When this._async is true, it pauses occasionally to avoid blocking for too long.
  // Stops iterating after watcher is stopped.
  private processBatches<T>(
    array: T[],
    handleItem: (item: T) => any,
  ) {
    let index = 0;

    const processBatch = () => {
      const stopTime = this.async ? Date.now() + 50 : Infinity;
      while (Date.now() < stopTime && index < array.length) {
        if (this.stopped) {
          return;
        }

        handleItem(array[index]);

        index += 1;
      }

      if (index < array.length) {
        if (this.async) {
          setImmediate(processBatch);
        } else {
          processBatch();
        }
      }
    }

    processBatch();
  }

  private checkDirectories() {
    const dirs = Object.values(this.watchSet.directories.reduce((result, dir) => {
      const dirs = result[dir.absPath];
      if (dirs) {
        dirs.push(dir);
      } else {
        result[dir.absPath] = [dir];
      }
      return result;
    }, {} as Record<string, DirectoryEntry[]>));

    if (this.stopped) {
      return;
    }

    this.processBatches(dirs, entries => {
      if (! this.justCheckOnce) {
        this.watchFileOrDirectory(entries[0].absPath, true);
      }

      // Check for the case where by the time we created the watch, the
      // directory has already changed.
      this.fireIfDirectoryChanged(entries);
    });
  }

  private fire() {
    if (this.stopped) return;
    this.stop();
    this.onChange();
  }

  stop() {
    this.stopped = true;
    // Clean up file watches
    _.each(this.watches, function (entry: { watcher: FSWatcher | null }) {
      if (entry.watcher) {
        entry.watcher.close();
        entry.watcher = null;
      }
    });
    this.watches = Object.create(null);
  }
}

// Given a WatchSet, returns true if it currently describes the state of the
// disk.
export function isUpToDate(
  watchSet: WatchSet,
  includePotentiallyUnusedFiles = true
) {
  return Profile.time('watch.isUpToDate', () => {
    let upToDate = true;
    const watcher = new Watcher({
      watchSet: watchSet,
      onChange() {
        upToDate = false;
      },
      // internal flag which prevents us from starting watches and timers that
      // we're about to cancel anyway
      justCheckOnce: true,
      includePotentiallyUnusedFiles,
    });
    watcher.stop();
    return upToDate;
  });
}

// Options should have absPath/include/exclude/names.
export function readAndWatchDirectory(
  watchSet: WatchSet,
  options: DirectoryEntry,
) {
  const contents = readDirectory(options);
  watchSet.addDirectory({
    ...options,
    contents,
  });
  return contents;
}

// Calculating the sha hash can be expensive for large files.  By
// returning the calculated hash along with the file contents, the
// hash doesn't need to be calculated again for static files.
export function readAndWatchFileWithHash(watchSet: WatchSet, absPath: string) {
  const result: {
    contents: string | Buffer | null;
    hash: string | null;
  } = {
    contents: null,
    hash: null,
  };

  try {
    result.contents = files.readFile(absPath);
  } catch (e: any) {
    if (e && e.code === "EISDIR") {
      // Avoid adding directories to the watchSet as files.
      return result;
    }

    if (e && e.code === "ENOENT") {
      // Continue, leaving result.{contents,hash} both null.
    } else {
      // Throw all other errors.
      throw e;
    }
  }

  if (result.contents !== null) {
    result.hash = sha1(result.contents);
  }

  // Allow null watchSet, if we want to use readFile-style error handling in a
  // context where we might not always have a WatchSet (eg, reading
  // settings.json where we watch for "meteor run" but not for "meteor deploy").
  if (watchSet) {
    watchSet.addFile(absPath, result.hash);
  }

  return result;
}

export function readAndWatchFile(watchSet: WatchSet, absPath: string) {
  return readAndWatchFileWithHash(watchSet, absPath).contents;
}