Revert "support @parcel/watcher as the watcher alternative cross-os perserving polling fallback"

This reverts commit a38bcaee9a.
This commit is contained in:
Nacho Codoñer
2025-04-16 10:55:58 +02:00
parent a38bcaee9a
commit 6de4d9dd14
2 changed files with 443 additions and 330 deletions

View File

@@ -57,7 +57,8 @@ var packageJson = {
escope: "3.6.0",
split2: "3.2.2",
multipipe: "2.0.1",
"@parcel/watcher": "2.5.1",
pathwatcher: "8.1.2",
"vscode-nsfw": "2.1.8",
// The @wry/context package version must be compatible with the
// version constraint imposed by optimism/package.json.
optimism: "0.16.1",

View File

@@ -1,364 +1,476 @@
import { Stats } from 'fs';
import { dirname } from "path";
import ParcelWatcher from "@parcel/watcher";
import { FSWatcher, Stats, BigIntStats } from "fs";
import { Profile } from "../tool-env/profile";
import { statOrNull, lstat, toPosixPath, convertToOSPath, pathRelative, watchFile, unwatchFile } from "./files";
import {
statOrNull,
convertToOSPath,
watchFile,
unwatchFile,
toPosixPath,
pathRelative
} from "./files";
import {
join as nativeJoin
} from 'path';
import nsfw from 'vscode-nsfw';
export type SafeWatcher = { close: () => void; };
type ChangeCallback = (event: string) => void;
interface Entry extends SafeWatcher {
callbacks: Set<ChangeCallback>;
_fire(event: string): void;
}
// Registry mapping normalized absolute paths to their watcher entry.
const entries: Record<string, Entry | null> = Object.create(null);
// Watch roots are directories for which we have an active ParcelWatcher subscription.
const watchRoots = new Set<string>();
// For each watch root, store its active subscription.
const dirSubscriptions = new Map<string, ParcelWatcher.AsyncSubscription>();
// A set of roots that are known to be unwatchable.
const ignoredWatchRoots = new Set<string>();
// Set METEOR_WATCH_FORCE_POLLING environment variable to a truthy value to
// force the use of files.watchFile instead of ParcelWatcher.
let watcherEnabled = !JSON.parse(process.env.METEOR_WATCH_FORCE_POLLING || "false");
/**
* Polling fallback globals.
* The legacy Meteor strategy used polling for cases where native watchers failed.
* We keep track of files that changed, so that we can poll them faster.
*/
var DEFAULT_POLLING_INTERVAL =
+(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 5000);
var NO_WATCHER_POLLING_INTERVAL =
+(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 500);
const pathwatcher = require('pathwatcher');
// Default to prioritizing changed files, but disable that behavior (and
// thus prioritize all files equally) if METEOR_WATCH_PRIORITIZE_CHANGED
// is explicitly set to a string that parses to a falsy value.
var PRIORITIZE_CHANGED = true;
if (process.env.METEOR_WATCH_PRIORITIZE_CHANGED &&
! JSON.parse(process.env.METEOR_WATCH_PRIORITIZE_CHANGED)) {
PRIORITIZE_CHANGED = false;
}
var DEFAULT_POLLING_INTERVAL =
+(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 5000);
var NO_WATCHER_POLLING_INTERVAL =
+(process.env.METEOR_WATCH_POLLING_INTERVAL_MS || 500);
// This may seems like a long time to wait before actually closing the
// file watchers, but it's to our advantage if they survive restarts.
const WATCHER_CLEANUP_DELAY_MS = 30000;
// Since linux doesn't have recursive file watching, nsfw has to walk the
// watched folder and create a separate watcher for each subfolder. Until it has a
// way for us to filter which folders it walks we will continue to use
// pathwatcher to avoid having too many watchers.
let watcherLibrary = process.env.METEOR_WATCHER_LIBRARY ||
(process.platform === 'linux' ? 'pathwatcher' : 'nsfw');
// Pathwatcher complains (using console.error, ugh) if you try to watch
// two files with the same stat.ino number but different paths on linux, so we have
// to deduplicate files by ino.
const DEDUPLICATE_BY_INO = watcherLibrary === 'pathwatcher';
// Set METEOR_WATCH_FORCE_POLLING environment variable to a truthy value to
// force the use of files.watchFile instead of watchLibrary.watch.
let watcherEnabled = ! JSON.parse(
process.env.METEOR_WATCH_FORCE_POLLING || "false"
);
const entriesByIno = new Map;
export type SafeWatcher = {
close: () => void;
}
type EntryCallback = (event: string) => void;
interface Entry extends SafeWatcher {
callbacks: Set<EntryCallback>;
rewatch: () => void;
release: (callback: EntryCallback) => void;
_fire: (event: string) => void;
}
const entries: Record<string, Entry | null> = Object.create(null);
// Folders that are watched recursively
let watchRoots = new Set<string>();
// Set of paths for which a change event has been fired, watched with
// watchLibrary.watch if available. This could be an LRU cache, but in
// practice it should never grow large enough for that to matter.
const changedPaths = new Set;
function shouldIgnorePath(absPath: string): boolean {
const posixPath = toPosixPath(absPath);
const parts = posixPath.split('/');
// Check for .meteor: allow the .meteor directory itself,
// but ignore its "local" subdirectory (or any immediate child folder that indicates cache).
const meteorIndex = parts.indexOf(".meteor");
if (meteorIndex !== -1) {
const nextPart = parts[meteorIndex + 1];
if (nextPart && nextPart === "local") {
// Ignore anything inside .meteor/local
return true;
}
// Otherwise, do not automatically ignore .meteor (which includes .meteor/packages, etc).
}
// For node_modules: ignore contents unless the package directory is a symlink (which means its under active development).
const nmIndex = parts.indexOf("node_modules");
if (nmIndex !== -1) {
try {
const stat = lstat(absPath);
if (!stat?.isSymbolicLink()) {
return true;
}
// If the package folder is symlinked, we want to watch it.
} catch (e) {
return true;
}
}
return false;
}
/**
* Ensure that the given directory is being watched by @parcel/watcher.
* If it is not a directory or is unwatchable, it is immediately added to an ignore set.
*/
async function ensureWatchRoot(dirPath: string): Promise<void> {
if (!watcherEnabled || watchRoots.has(dirPath) || ignoredWatchRoots.has(dirPath)) {
return;
}
// If an ancestor is already watched, skip this one.
for (const root of watchRoots) {
const rel = pathRelative(root, dirPath);
if (root !== dirPath && !rel.startsWith("..") && !rel.startsWith("/")) {
return;
}
}
// Remove any existing roots that are now encompassed by the new one.
for (const root of Array.from(watchRoots)) {
const rel = pathRelative(dirPath, root);
if (root !== dirPath && !rel.startsWith("..") && !rel.startsWith("/")) {
const sub = dirSubscriptions.get(root);
if (sub) {
try {
await sub.unsubscribe();
} catch (_) {
/* ignore errors */
}
}
dirSubscriptions.delete(root);
watchRoots.delete(root);
}
}
const osDirPath = convertToOSPath(dirPath);
// Check that osDirPath is indeed a directory.
try {
const stats = statOrNull(osDirPath);
if (!stats?.isDirectory()) {
console.warn(`Skipping watcher for ${osDirPath}: not a directory`);
ignoredWatchRoots.add(dirPath);
return;
}
} catch (e) {
console.error(`Failed to stat ${osDirPath}:`, e);
ignoredWatchRoots.add(dirPath);
return;
}
// Set up ignore patterns to skip deep node_modules and .meteor/local cache
const ignorePatterns = ["**/node_modules/**", "**/.meteor/local/**"];
try {
const subscription = await ParcelWatcher.subscribe(
osDirPath,
(err, events) => {
if (err) {
console.error(`Parcel watcher error on ${osDirPath}:`, err);
// Only disable native watching for critical errors (like ENOSPC).
// @ts-ignore
if (err.code === "ENOSPC" || err.errno === require("constants").ENOSPC) {
fallbackToPolling();
}
return;
}
// Dispatch each event to any registered entries.
for (const event of events) {
const changedPath = toPosixPath(event.path);
const entry = entries[changedPath];
if (!entry) continue;
// In Meteor's safe-watcher API, both create/update trigger "change" events.
const evtType = event.type === "delete" ? "delete" : "change";
entry._fire(evtType);
}
},
{ ignore: ignorePatterns }
);
watchRoots.add(dirPath);
dirSubscriptions.set(dirPath, subscription);
} catch (e: any) {
if (
e &&
(e.code === "ENOTDIR" ||
/Not a directory/.test(e.message) ||
e.code === "EBADF" ||
/Bad file descriptor/.test(e.message))
) {
console.warn(`Skipping watcher for ${osDirPath}: not a directory`);
ignoredWatchRoots.add(dirPath);
} else {
console.error(`Failed to start watcher for ${osDirPath}:`, e);
if (e.code === "ENOSPC" || e.errno === require("constants").ENOSPC) {
fallbackToPolling();
}
}
}
}
/**
* Creates a new watch entry for a specific file (or directory) and
* holds its registered callbacks.
*/
function startNewEntry(absPath: string): Entry {
const callbacks = new Set<ChangeCallback>();
let closed = false;
const entry: Entry = {
callbacks,
close() {
if (closed) return;
closed = true;
delete entries[absPath];
},
_fire(event: string) {
callbacks.forEach(cb => {
try {
cb(event);
} catch (e) {
// Ignore callback errors.
}
});
}
};
return entry;
}
/**
* The primary API function to watch a file or directory.
* This registers the callback on the internally managed entry and
* ensures that a Parcel watcher is subscribed to a covering directory.
*/
export const watch = Profile(
"safeWatcher.watch",
(
absPath: string,
callback: ChangeCallback
): SafeWatcher => {
absPath = toPosixPath(absPath);
// If the path should be ignored, immediately return a noop SafeWatcher.
if (shouldIgnorePath(absPath)) {
return { close() {} };
}
// If native watching is disabled, use the polling strategy.
if (!watcherEnabled) {
return startPolling(absPath, callback);
}
// Try to reuse an existing entry if one was created before.
let entry = entries[absPath];
if (!entry) {
entry = startNewEntry(absPath);
entries[absPath] = entry;
// Determine the directory that should be watched.
let watchTarget: string;
try {
const st = statOrNull(convertToOSPath(absPath));
watchTarget = st?.isDirectory() ? absPath : toPosixPath(dirname(convertToOSPath(absPath)));
} catch (e) {
watchTarget = toPosixPath(dirname(convertToOSPath(absPath)));
}
// Set up a watcher on the parent directory (or the directory itself) if not already active.
ensureWatchRoot(watchTarget);
}
// Register the callback for this file.
entry.callbacks.add(callback);
return {
close() {
if (entries[absPath]) {
entries[absPath]!.callbacks.delete(callback);
if (entries[absPath]!.callbacks.size === 0) {
entries[absPath]!.close();
}
}
}
};
});
/**
* Externally force a directory to be watched.
* If the provided path is a file, its parent directory is used.
*/
export function addWatchRoot(absPath: string) {
absPath = toPosixPath(absPath);
let watchTarget = absPath;
try {
const st = statOrNull(convertToOSPath(absPath));
if (!st?.isDirectory()) {
watchTarget = toPosixPath(dirname(convertToOSPath(absPath)));
}
} catch (e) {
watchTarget = toPosixPath(dirname(convertToOSPath(absPath)));
}
ensureWatchRoot(watchTarget);
}
async function safeUnsubscribeSub(root: string) {
const sub = dirSubscriptions.get(root);
if (!sub) return; // Already unsubscribed.
// Remove from our maps immediately to prevent further unsubscribe calls.
dirSubscriptions.delete(root);
watchRoots.delete(root);
try {
await sub.unsubscribe();
} catch (e) {
console.error(`Error during unsubscribe for ${root}:`, e);
}
}
export async function closeAllWatchers() {
for (const root of Array.from(watchRoots)) {
await safeUnsubscribeSub(root);
}
}
function hasPriority(absPath: string) {
// If we're not prioritizing changed files, then all files have
// priority, which means they should be watched with native file
// watchers if the platform supports them. If we are prioritizing
// changed files, then only changed files get priority.
return PRIORITIZE_CHANGED
? changedPaths.has(absPath)
: true;
? changedPaths.has(absPath)
: true;
}
// Determines the polling interval to be used for the fs.watchFile-based
// safety net that works on all platforms and file systems.
function getPollingInterval(absPath: string): number {
if (hasPriority(absPath)) {
// Regardless of whether we have a native file watcher and it works
// correctly on this file system, poll prioritized files (that is,
// files that have been changed at least once) at a higher frequency
// (every 500ms by default).
function acquireWatcher(absPath: string, callback: EntryCallback) {
const entry = entries[absPath] || (
entries[absPath] = startNewWatcher(absPath));
// Watches successfully established in the past may have become invalid
// because the watched file was deleted or renamed, so we need to make
// sure we're still watching every time we call safeWatcher.watch.
entry.rewatch();
// The size of the entry.callbacks Set also serves as a reference count
// for this watcher.
entry.callbacks.add(callback);
return entry;
}
function startNewWatcher(absPath: string): Entry {
let stat: Stats | BigIntStats | null | undefined = null;
if (DEDUPLICATE_BY_INO) {
stat = statOrNull(absPath);
if (stat && stat.ino > 0 && entriesByIno.has(stat.ino)) {
const entry = entriesByIno.get(stat.ino);
if (entries[absPath] === entry) {
return entry;
}
}
} else {
let entry = entries[absPath];
if (entry) {
return entry;
}
}
function safeUnwatch() {
if (watcher) {
watcher.close();
watcher = null;
if (stat && stat.ino > 0) {
entriesByIno.delete(stat.ino);
}
}
}
let lastWatcherEventTime = Date.now();
const callbacks = new Set<EntryCallback>();
let watcherCleanupTimer: ReturnType<typeof setTimeout> | null = null;
let watcher: FSWatcher | null = null;
// Determines the polling interval to be used for the fs.watchFile-based
// safety net that works on all platforms and file systems.
function getPollingInterval() {
if (hasPriority(absPath)) {
// Regardless of whether we have a native file watcher and it works
// correctly on this file system, poll prioritized files (that is,
// files that have been changed at least once) at a higher frequency
// (every 500ms by default).
return NO_WATCHER_POLLING_INTERVAL;
}
if (watcherEnabled || PRIORITIZE_CHANGED) {
// As long as native file watching is enabled (even if it doesn't
// work correctly) and the developer hasn't explicitly opted out of
// the file watching priority system, poll unchanged files at a
// lower frequency (every 5000ms by default).
return DEFAULT_POLLING_INTERVAL;
}
// If native file watching is disabled and the developer has
// explicitly opted out of the priority system, poll everything at the
// higher frequency (every 500ms by default). Note that this leads to
// higher idle CPU usage, so the developer may want to adjust the
// METEOR_WATCH_POLLING_INTERVAL_MS environment variable.
return NO_WATCHER_POLLING_INTERVAL;
}
if (watcherEnabled || PRIORITIZE_CHANGED) {
// As long as native file watching is enabled (even if it doesn't
// work correctly) and the developer hasn't explicitly opted out of
// the file watching priority system, poll unchanged files at a
// lower frequency (every 5000ms by default).
return DEFAULT_POLLING_INTERVAL;
}
function fire(event: string) {
if (event !== "change") {
// When we receive a "delete" or "rename" event, the watcher is
// probably not going to generate any more notifications for this
// file, so we close and nullify the watcher to ensure that
// entry.rewatch() will attempt to reestablish the watcher the next
// time we call safeWatcher.watch.
safeUnwatch();
// If native file watching is disabled and the developer has
// explicitly opted out of the priority system, poll everything at the
// higher frequency (every 500ms by default). Note that this leads to
// higher idle CPU usage, so the developer may want to adjust the
// METEOR_WATCH_POLLING_INTERVAL_MS environment variable.
return NO_WATCHER_POLLING_INTERVAL;
}
// Make sure we don't throttle the watchFile callback after a
// "delete" or "rename" event, since it is now our only reliable
// source of file change notifications.
lastWatcherEventTime = 0;
function startPolling(absPath: string, callback: ChangeCallback): SafeWatcher {
const osPath = convertToOSPath(absPath);
// Initial polling interval.
let interval = getPollingInterval(absPath);
const pollCallback = (curr: Stats, prev: Stats) => {
// Compare modification times to detect a change.
if (+curr.mtime !== +prev.mtime) {
} else {
changedPaths.add(absPath);
callback("change");
rewatch();
}
};
watchFile(osPath, { interval }, pollCallback);
return {
callbacks.forEach(cb => cb(event));
}
function watchWrapper(event: string) {
lastWatcherEventTime = Date.now();
fire(event);
// It's tempting to call unwatchFile(absPath, watchFileWrapper) here,
// but previous watcher success is no guarantee of future watcher
// reliability. For example, watchLibrary.watch works just fine when file
// changes originate from within a Vagrant VM, but changes to shared
// files made outside the VM are invisible to watcher, so our only
// hope of catching them is to continue polling.
}
function rewatch() {
if (hasPriority(absPath)) {
if (watcher) {
// Already watching; nothing to do.
return;
}
watcher = watchLibraryWatch(absPath, watchWrapper);
} else if (watcher) {
safeUnwatch();
}
// Since we're about to restart the stat-based file watcher, we don't
// want to miss any of its events because of the lastWatcherEventTime
// throttling that it attempts to do.
lastWatcherEventTime = 0;
// We use files.watchFile in addition to watcher.watch as a fail-safe
// to detect file changes even on network file systems. However
// (unless the user disabled watcher or this watcher call failed), we
// use a relatively long default polling interval of 5000ms to save
// CPU cycles.
statWatch(absPath, getPollingInterval(), watchFileWrapper);
}
function watchFileWrapper(newStat: Stats, oldStat: Stats) {
if (newStat.ino === 0 &&
oldStat.ino === 0 &&
+newStat.mtime === +oldStat.mtime) {
// Node calls the watchFile listener once with bogus identical stat
// objects, which should not trigger a file change event.
return;
}
// If a watcher event fired in the last polling interval, ignore
// this event.
if (Date.now() - lastWatcherEventTime > getPollingInterval()) {
fire("change");
}
}
const entry = {
callbacks,
rewatch,
release(callback: EntryCallback) {
if (! entries[absPath]) {
return;
}
callbacks.delete(callback);
if (callbacks.size > 0) {
return;
}
// Once there are no more callbacks in the Set, close both watchers
// and nullify the shared data.
if (watcherCleanupTimer) {
clearTimeout(watcherCleanupTimer);
}
watcherCleanupTimer = setTimeout(() => {
if (callbacks.size > 0) {
// If another callback was added while the timer was pending, we
// can avoid tearing anything down.
return;
}
entry.close();
}, WATCHER_CLEANUP_DELAY_MS);
},
close() {
unwatchFile(osPath, pollCallback);
changedPaths.delete(absPath);
}
if (entries[absPath] !== entry) return;
entries[absPath] = null;
if (watcherCleanupTimer) {
clearTimeout(watcherCleanupTimer);
watcherCleanupTimer = null;
}
safeUnwatch();
unwatchFile(absPath, watchFileWrapper);
},
_fire: fire
};
if (stat && stat.ino > 0) {
entriesByIno.set(stat.ino, entry);
}
return entry;
}
/**
* Fall back to polling. If a critical error occurs,
* we disable native watching and close all existing native watchers.
*/
function fallbackToPolling() {
if (watcherEnabled) {
console.error("Critical native watcher error encountered. Falling back to polling for all entries.");
watcherEnabled = false;
closeAllWatchers();
export function closeAllWatchers() {
Object.keys(entries).forEach(absPath => {
const entry = entries[absPath];
if (entry) {
entry.close();
}
});
}
const statWatchers = Object.create(null);
function statWatch(
absPath: string,
interval: number,
callback: (current: Stats, previous: Stats) => void,
) {
let statWatcher = statWatchers[absPath];
if (!statWatcher) {
statWatcher = {
interval,
changeListeners: [],
stat: null
};
statWatchers[absPath] = statWatcher;
}
// If the interval needs to be changed, replace the watcher.
// Node will only recreate the watcher with the new interval if all old
// watchers are stopped (which unwatchFile does when not passed a
// specific listener)
if (statWatcher.interval !== interval && statWatcher.stat) {
// This stops all stat watchers for the file, not just those created by
// statWatch
unwatchFile(absPath);
statWatcher.stat = null;
statWatcher.interval = interval;
}
if (!statWatcher.changeListeners.includes(callback)) {
statWatcher.changeListeners.push(callback);
}
if (!statWatcher.stat) {
const newStat = watchFile(absPath, {
persistent: false, // never persistent
interval,
}, (newStat, oldStat) => {
statWatcher.changeListeners.forEach((
listener: (newStat: Stats, oldStat: Stats) => void
) => {
listener(newStat, oldStat);
});
});
newStat.on("stop", () => {
if (statWatchers[absPath] === statWatch) {
delete statWatchers[absPath];
}
});
statWatcher.stat = newStat;
}
return statWatcher;
}
function watchLibraryWatch(absPath: string, callback: EntryCallback) {
if (watcherEnabled && watcherLibrary === 'pathwatcher') {
try {
return pathwatcher.watch(convertToOSPath(absPath), callback);
} catch (e: any) {
maybeSuggestRaisingWatchLimit(e);
// ... ignore the error. We'll still have watchFile, which is good
// enough.
}
}
return null;
}
let suggestedRaisingWatchLimit = false;
function maybeSuggestRaisingWatchLimit(error: Error & { errno: number }) {
var constants = require('constants');
var archinfo = require('../utils/archinfo');
if (! suggestedRaisingWatchLimit &&
// Note: the not-super-documented require('constants') maps from
// strings to SYSTEM errno values. System errno values aren't the same
// as the numbers used internally by libuv! Once we're upgraded
// to Node 0.12, we'll have the system errno as a string (on 'code'),
// but the support for that wasn't in Node 0.10's uv.
// See our PR https://github.com/atom/node-pathwatcher/pull/53
// (and make sure to read the final commit message, not the original
// proposed PR, which had a slightly different interface).
error.errno === constants.ENOSPC &&
// The only suggestion we currently have is for Linux.
archinfo.matches(archinfo.host(), 'os.linux')) {
// Check suggestedRaisingWatchLimit again because archinfo.host() may
// have yielded.
if (suggestedRaisingWatchLimit) return;
suggestedRaisingWatchLimit = true;
var Console = require('../console/console.js').Console;
if (! Console.isHeadless()) {
Console.arrowWarn(
"It looks like a simple tweak to your system's configuration will " +
"make many tools (including this Meteor command) more efficient. " +
"To learn more, see " +
Console.url("https://github.com/meteor/docs/blob/master/long-form/file-change-watcher-efficiency.md"));
}
}
}
export const watch = Profile(
"safeWatcher.watch",
(absPath: string, callback: EntryCallback) => {
const entry = acquireWatcher(absPath, callback);
return {
close() {
entry.release(callback);
}
} as SafeWatcher;
}
);
const fireNames = {
[nsfw.actions.CREATED]: 'change',
[nsfw.actions.MODIFIED]: 'change',
[nsfw.actions.DELETED]: 'delete'
}
export function addWatchRoot(absPath: string) {
if (watchRoots.has(absPath) || watcherLibrary !== 'nsfw' || !watcherEnabled) {
return;
}
watchRoots.add(absPath);
// If there already is a watcher for a parent directory, there is no need
// to create this watcher.
for (const path of watchRoots) {
let relativePath = pathRelative(path, absPath);
if (
path !== absPath &&
!relativePath.startsWith('..') &&
!relativePath.startsWith('/')
) {
return;
}
}
// TODO: check if there are any existing watchers that are children of this
// watcher and stop them
nsfw(
convertToOSPath(absPath),
(events) => {
events.forEach(event => {
if(event.action === nsfw.actions.RENAMED) {
let oldPath = nativeJoin(event.directory, event.oldFile);
let oldEntry = entries[toPosixPath(oldPath)];
if (oldEntry) {
oldEntry._fire('rename');
}
let path = nativeJoin(event.newDirectory, event.newFile);
let newEntry = entries[toPosixPath(path)];
if (newEntry) {
newEntry._fire('change');
}
} else {
let path = nativeJoin(event.directory, event.file);
let entry = entries[toPosixPath(path)];
if (entry) {
entry._fire(fireNames[event.action]);
}
}
})
}
).then(watcher => {
watcher.start()
});
}