repro: surface and retry MkdirAll errors during dir creation

Parallel relative-path MkdirAll on the bindflt emptyDir was silently
failing for some dirs (error discarded), so later file writes hit
'path not found'. Count errno-87 from MkdirAll and retry up to 5x so
the synthetic repro can proceed — and so we can see whether the
bindflt race also affects CreateDirectoryW.

.github/actions/build-electron/action.yml

@@ -93,8 +93,26 @@ runs:
           echo "Skipping build-stats.mjs upload because DD_API_KEY is not set"
         fi
         node electron/script/build-stats.mjs $BUILD_STATS_ARGS || true
+    - name: io-repro — set up Go
+      if: ${{ inputs.target-platform == 'win' && github.ref_name == 'sam/windows-io-repro' }}
+      uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+      with:
+        go-version: '1.26'
+        cache: false
+    - name: io-repro — gn gen then scan real out/Default subninjas
+      if: ${{ inputs.target-platform == 'win' && github.ref_name == 'sam/windows-io-repro' }}
+      shell: powershell
+      run: |
+        cd src\electron
+        git pack-refs
+        cd ..
+        cd electron\script\windows-io-repro
+        go build -o repro.exe .
+        cd ..\..\..
+        e build --only-gen
+        electron\script\windows-io-repro\repro.exe -skip-create -dir out\Default -rounds 12
     - name: Build Electron (Windows) ${{ inputs.step-suffix }}
-      if: ${{ inputs.target-platform == 'win' }}
+      if: ${{ inputs.target-platform == 'win' && github.ref_name != 'sam/windows-io-repro' }}
       shell: powershell
       run: |
         cd src\electron

.github/actions/install-build-tools/action.yml

@@ -30,3 +30,19 @@ runs:
         echo "$HOME/.electron_build_tools/third_party/depot_tools" >> $GITHUB_PATH
         echo "$HOME/.electron_build_tools/third_party/depot_tools/python-bin" >> $GITHUB_PATH
       fi
+  - name: Install patched siso (Windows)
+    # Overrides the CIPD siso with a build from electron/siso that carries a
+    # retry for transient ERROR_INVALID_PARAMETER during the subninja scan on
+    # container bind-mounted out/ directories. Remove once the fix has rolled
+    # into Chromium's siso_version.
+    if: ${{ runner.os == 'Windows' }}
+    shell: bash
+    env:
+      ELECTRON_SISO_URL: https://github.com/electron/siso/releases/download/v1.5.7-electron.2/siso-windows-amd64.exe
+      ELECTRON_SISO_SHA256: 0e6b754820be3324d5ea4ca3af3634b4cfcf806d89140d78fec4e2a8ef636c9d
+    run: |
+      set -eo pipefail
+      mkdir -p /c/electron-siso
+      curl --fail --retry 3 -sSL "$ELECTRON_SISO_URL" -o /c/electron-siso/siso.exe
+      echo "$ELECTRON_SISO_SHA256  /c/electron-siso/siso.exe" | sha256sum --check --strict -
+      echo "SISO_PATH=C:\\electron-siso\\siso.exe" >> "$GITHUB_ENV"

.github/workflows/windows-io-repro.yml

@@ -0,0 +1,42 @@
+name: windows-io-repro
+
+on:
+  push:
+    branches:
+      - sam/windows-io-repro
+
+permissions: {}
+
+jobs:
+  standalone:
+    permissions:
+      contents: read
+    runs-on: electron-arc-centralus-windows-amd64-32core
+    timeout-minutes: 60
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+      - name: Set up Go
+        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
+        with:
+          go-version: '1.26'
+          cache: false
+      - name: Build repro
+        shell: powershell
+        run: |
+          cd script/windows-io-repro
+          go build -o repro.exe .
+      - name: Writer process — create 1.1M files in deep tree
+        shell: powershell
+        env:
+          REPRO_DIR: ${{ github.workspace }}\repro-files
+        run: |
+          script\windows-io-repro\repro.exe -dir "$env:REPRO_DIR" -files 1100000 -write-only
+      - name: Scanner process — double-open scan 60k subset
+        shell: powershell
+        env:
+          REPRO_DIR: ${{ github.workspace }}\repro-files
+        run: |
+          script\windows-io-repro\repro.exe -dir "$env:REPRO_DIR" -files 1100000 -no-create -scan-first 60000 -rounds 12

script/windows-io-repro/go.mod

@@ -0,0 +1,3 @@
+module github.com/electron/electron/script/windows-io-repro
+
+go 1.26

script/windows-io-repro/main.go

@@ -0,0 +1,322 @@
+// Command windows-io-repro stress-tests concurrent file opens on Windows
+// to reproduce the intermittent ERROR_INVALID_PARAMETER (errno 87) seen
+// in siso's subninja scan when out/ is served through a container
+// bind-mount filter driver.
+//
+// It creates a large set of small files in -dir, then runs multiple
+// NumCPU-parallel scan rounds using two strategies:
+//   - stdlib:   os.Open (Go passes FILE_FLAG_BACKUP_SEMANTICS)
+//   - nobackup: direct CreateFileW without FILE_FLAG_BACKUP_SEMANTICS
+//
+// Each errno-87 hit is logged with whether a 5ms retry recovers, so the
+// two strategies can be compared head-to-head.
+//
+//go:build windows
+
+package main
+
+import (
+	"crypto/rand"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"sync"
+	"sync/atomic"
+	"syscall"
+	"time"
+)
+
+const errnoInvalidParameter = syscall.Errno(87)
+
+// innerSema mirrors siso's fsema: the chunk-read goroutine acquires a
+// NumCPU-wide slot before doing its second open.
+var innerSema = make(chan struct{}, runtime.NumCPU())
+
+// openStdlib replicates siso's fileParser.readFile: outer os.Open + Stat,
+// then a goroutine that does a second os.Open on the same path and ReadAt,
+// with the outer handle still held until the goroutine returns.
+func openStdlib(path string) error {
+	f, err := os.Open(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil {
+		return err
+	}
+	buf := make([]byte, st.Size())
+	errCh := make(chan error, 1)
+	go func() {
+		innerSema <- struct{}{}
+		defer func() { <-innerSema }()
+		f2, err := os.Open(path)
+		if err != nil {
+			errCh <- err
+			return
+		}
+		defer f2.Close()
+		_, err = f2.ReadAt(buf, 0)
+		errCh <- err
+	}()
+	return <-errCh
+}
+
+func createNoBackup(path string) (syscall.Handle, error) {
+	p, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return syscall.InvalidHandle, err
+	}
+	h, err := syscall.CreateFile(
+		p,
+		syscall.GENERIC_READ,
+		syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
+		nil,
+		syscall.OPEN_EXISTING,
+		syscall.FILE_ATTRIBUTE_NORMAL,
+		0,
+	)
+	if err != nil {
+		return syscall.InvalidHandle, &os.PathError{Op: "CreateFile", Path: path, Err: err}
+	}
+	return h, nil
+}
+
+// openNoBackup mirrors openStdlib's double-open shape but uses
+// CreateFileW without FILE_FLAG_BACKUP_SEMANTICS for both opens.
+func openNoBackup(path string) error {
+	h1, err := createNoBackup(path)
+	if err != nil {
+		return err
+	}
+	defer syscall.CloseHandle(h1)
+	errCh := make(chan error, 1)
+	go func() {
+		innerSema <- struct{}{}
+		defer func() { <-innerSema }()
+		h2, err := createNoBackup(path)
+		if err != nil {
+			errCh <- err
+			return
+		}
+		errCh <- syscall.CloseHandle(h2)
+	}()
+	return <-errCh
+}
+
+type result struct {
+	opens      atomic.Int64
+	err87      atomic.Int64
+	err87Retry atomic.Int64 // errno 87 that cleared on immediate retry
+	otherErr   atomic.Int64
+}
+
+func scan(name string, open func(string) error, files []string, workers int, r *result) time.Duration {
+	start := time.Now()
+	ch := make(chan string, workers)
+	var wg sync.WaitGroup
+	for range workers {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for path := range ch {
+				r.opens.Add(1)
+				err := open(path)
+				if err == nil {
+					continue
+				}
+				if errors.Is(err, errnoInvalidParameter) {
+					r.err87.Add(1)
+					time.Sleep(5 * time.Millisecond)
+					if open(path) == nil {
+						r.err87Retry.Add(1)
+						fmt.Printf("[%s] errno87 %s — recovered on retry\n", name, path)
+					} else {
+						fmt.Printf("[%s] errno87 %s — retry FAILED\n", name, path)
+					}
+				} else {
+					r.otherErr.Add(1)
+					fmt.Printf("[%s] other error %s: %v\n", name, path, err)
+				}
+			}
+		}()
+	}
+	for _, f := range files {
+		ch <- f
+	}
+	close(ch)
+	wg.Wait()
+	return time.Since(start)
+}
+
+func main() {
+	dir := flag.String("dir", "repro-files", "directory to create/scan test files in")
+	nfiles := flag.Int("files", 50000, "number of test files to create")
+	rounds := flag.Int("rounds", 10, "scan rounds per strategy")
+	workers := flag.Int("workers", runtime.NumCPU(), "concurrent openers per scan")
+	writeOnly := flag.Bool("write-only", false, "create files then exit (for cross-process write→scan mode)")
+	skipCreate := flag.Bool("skip-create", false, "scan existing files in -dir (walks the tree)")
+	ext := flag.String("ext", "", "with -skip-create, only scan files with this extension (e.g. .ninja); empty = all")
+	noCreate := flag.Bool("no-create", false, "regenerate the deterministic path list from -dir/-files but do not write files")
+	scanFirst := flag.Int("scan-first", 0, "if >0, scan only the first N files (create all, scan a subset)")
+	flag.Parse()
+
+	fmt.Printf("GOOS=%s GOARCH=%s NumCPU=%d GOMAXPROCS=%d workers=%d\n",
+		runtime.GOOS, runtime.GOARCH, runtime.NumCPU(), runtime.GOMAXPROCS(0), *workers)
+
+	if err := os.MkdirAll(*dir, 0o755); err != nil {
+		fmt.Fprintln(os.Stderr, "mkdir:", err)
+		os.Exit(1)
+	}
+	abs, _ := filepath.Abs(*dir)
+	fmt.Printf("test dir: %s\n", abs)
+
+	var files []string
+	if *skipCreate {
+		err := filepath.WalkDir(*dir, func(p string, d os.DirEntry, err error) error {
+			if err == nil && !d.IsDir() && (*ext == "" || filepath.Ext(p) == *ext) {
+				files = append(files, p)
+			}
+			return nil
+		})
+		if err != nil {
+			fmt.Fprintln(os.Stderr, "walk:", err)
+			os.Exit(1)
+		}
+		fmt.Printf("scanning %d existing files\n", len(files))
+	} else {
+		// Deep tree: 6 levels, fanout 7 → 7^6 = 117,649 leaf dirs (~137k dirs
+		// total), long-ish component names so full paths are ~120 chars — closer
+		// to the real src/out tree (~99k dirs, depth 5–10, long paths).
+		const depth = 6
+		const fanout = 7
+		leafDir := func(i int) string {
+			parts := make([]string, depth)
+			n := i
+			for d := depth - 1; d >= 0; d-- {
+				parts[d] = fmt.Sprintf("lvl%d_component_%02d", d, n%fanout)
+				n /= fanout
+			}
+			return filepath.Join(parts...)
+		}
+		leaves := 1
+		for range depth {
+			leaves *= fanout
+		}
+		payload := make([]byte, 512)
+		_, _ = rand.Read(payload)
+		files = make([]string, *nfiles)
+		for i := range *nfiles {
+			files[i] = filepath.Join(*dir, leafDir(i%leaves), fmt.Sprintf("some_build_target_name_%07d.ninja", i))
+		}
+		if *noCreate {
+			fmt.Printf("regenerated %d paths (no-create)\n", len(files))
+			goto scan
+		}
+		fmt.Printf("creating %d leaf dirs (depth %d, fanout %d)...\n", leaves, depth, fanout)
+		{
+			start := time.Now()
+			ch := make(chan int, *workers)
+			var wg sync.WaitGroup
+			var mkdirErr87 atomic.Int64
+			for range *workers {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					for i := range ch {
+						p := filepath.Join(*dir, leafDir(i))
+						for attempt := 0; ; attempt++ {
+							err := os.MkdirAll(p, 0o755)
+							if err == nil {
+								break
+							}
+							if errors.Is(err, errnoInvalidParameter) {
+								mkdirErr87.Add(1)
+							}
+							if attempt >= 4 {
+								fmt.Fprintf(os.Stderr, "mkdirall %s: %v (giving up)\n", p, err)
+								break
+							}
+							time.Sleep(5 * time.Millisecond)
+						}
+					}
+				}()
+			}
+			for i := range leaves {
+				ch <- i
+			}
+			close(ch)
+			wg.Wait()
+			fmt.Printf("dirs created in %s (mkdir err87=%d)\n", time.Since(start).Round(time.Second), mkdirErr87.Load())
+		}
+		fmt.Printf("creating %d files (%d-byte random payload, %d parallel writers)...\n", *nfiles, len(payload), *workers)
+		start := time.Now()
+		var created atomic.Int64
+		ch := make(chan string, *workers)
+		var wg sync.WaitGroup
+		for range *workers {
+			wg.Add(1)
+			go func() {
+				defer wg.Done()
+				for p := range ch {
+					if err := os.WriteFile(p, payload, 0o644); err != nil {
+						fmt.Fprintln(os.Stderr, "write:", err)
+						os.Exit(1)
+					}
+					if n := created.Add(1); n%100000 == 0 {
+						fmt.Printf("  ... %d/%d (%s)\n", n, *nfiles, time.Since(start).Round(time.Second))
+					}
+				}
+			}()
+		}
+		for _, p := range files {
+			ch <- p
+		}
+		close(ch)
+		wg.Wait()
+		fmt.Printf("created %d files across %d leaf dirs in %s\n", len(files), leaves, time.Since(start).Round(time.Second))
+	}
+	if *writeOnly {
+		fmt.Println("write-only mode; exiting")
+		return
+	}
+scan:
+	if *scanFirst > 0 && *scanFirst < len(files) {
+		files = files[:*scanFirst]
+		fmt.Printf("scanning first %d of %d files\n", len(files), *nfiles)
+	}
+
+	type strat struct {
+		name string
+		fn   func(string) error
+		r    result
+	}
+	strats := []*strat{
+		{name: "stdlib", fn: openStdlib},
+		{name: "nobackup", fn: openNoBackup},
+	}
+
+	for round := 1; round <= *rounds; round++ {
+		for _, s := range strats {
+			d := scan(s.name, s.fn, files, *workers, &s.r)
+			fmt.Printf("round %d/%d [%s] %d opens in %s (err87 so far: %d)\n",
+				round, *rounds, s.name, len(files), d.Round(time.Millisecond), s.r.err87.Load())
+		}
+	}
+
+	fmt.Println("\n=== summary ===")
+	for _, s := range strats {
+		fmt.Printf("[%s] opens=%d err87=%d err87_recovered=%d other_err=%d\n",
+			s.name, s.r.opens.Load(), s.r.err87.Load(), s.r.err87Retry.Load(), s.r.otherErr.Load())
+	}
+	var total int64
+	for _, s := range strats {
+		total += s.r.err87.Load()
+	}
+	if total == 0 {
+		fmt.Println("no ERROR_INVALID_PARAMETER observed")
+	}
+}

script/windows-io-repro/repro.c

@@ -0,0 +1,152 @@
+// Double-open errno-87 repro in plain C / Win32, to rule Go in/out.
+//
+// Build (cross, macOS/Linux):
+//   zig cc -target x86_64-windows-gnu -O2 -o repro_c.exe repro.c
+// or on Windows:
+//   cl /O2 repro.c /Fe:repro_c.exe
+//
+// Usage:
+//   cd C:\work
+//   repro_c.exe src\third_party 32 12
+//     arg1 = relative dir to walk for files (.h only)
+//     arg2 = worker threads (default 32)
+//     arg3 = rounds (default 12)
+
+#define WIN32_LEAN_AND_MEAN
+#include <process.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <windows.h>
+
+#define MAX_FILES 200000
+
+static wchar_t* g_files[MAX_FILES];
+static volatile LONG g_nfiles = 0;
+static volatile LONG g_idx;
+static volatile LONG g_err87 = 0;
+static volatile LONG g_err87_recovered = 0;
+static volatile LONG g_other = 0;
+static HANDLE g_inner_sema;
+
+static void walk(const wchar_t* dir) {
+  wchar_t pat[1024];
+  _snwprintf(pat, 1024, L"%ls\\*", dir);
+  WIN32_FIND_DATAW fd;
+  HANDLE h = FindFirstFileW(pat, &fd);
+  if (h == INVALID_HANDLE_VALUE)
+    return;
+  do {
+    if (!wcscmp(fd.cFileName, L".") || !wcscmp(fd.cFileName, L".."))
+      continue;
+    wchar_t full[1024];
+    _snwprintf(full, 1024, L"%ls\\%ls", dir, fd.cFileName);
+    if (fd.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) {
+      walk(full);
+    } else {
+      size_t n = wcslen(full);
+      if (n > 2 && !_wcsicmp(full + n - 2, L".h") && g_nfiles < MAX_FILES) {
+        g_files[g_nfiles] = _wcsdup(full);
+        InterlockedIncrement(&g_nfiles);
+      }
+    }
+  } while (FindNextFileW(h, &fd));
+  FindClose(h);
+}
+
+static HANDLE open_ro(const wchar_t* p) {
+  return CreateFileW(p, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL,
+                     OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+}
+
+struct inner_arg {
+  const wchar_t* path;
+  volatile LONG done;
+  DWORD err;
+};
+
+static unsigned __stdcall inner_thread(void* arg) {
+  struct inner_arg* a = (struct inner_arg*)arg;
+  WaitForSingleObject(g_inner_sema, INFINITE);
+  HANDLE h = open_ro(a->path);
+  if (h == INVALID_HANDLE_VALUE) {
+    a->err = GetLastError();
+  } else {
+    a->err = 0;
+    CloseHandle(h);
+  }
+  ReleaseSemaphore(g_inner_sema, 1, NULL);
+  InterlockedExchange(&a->done, 1);
+  return 0;
+}
+
+static unsigned __stdcall worker(void* unused) {
+  (void)unused;
+  for (;;) {
+    LONG i = InterlockedIncrement(&g_idx) - 1;
+    if (i >= g_nfiles)
+      return 0;
+    const wchar_t* p = g_files[i];
+
+    HANDLE h1 = open_ro(p);
+    if (h1 == INVALID_HANDLE_VALUE) {
+      DWORD e = GetLastError();
+      if (e == ERROR_INVALID_PARAMETER)
+        InterlockedIncrement(&g_err87);
+      else
+        InterlockedIncrement(&g_other);
+      continue;
+    }
+
+    struct inner_arg a = {p, 0, 0};
+    HANDLE th = (HANDLE)_beginthreadex(NULL, 0, inner_thread, &a, 0, NULL);
+    WaitForSingleObject(th, INFINITE);
+    CloseHandle(th);
+
+    if (a.err == ERROR_INVALID_PARAMETER) {
+      InterlockedIncrement(&g_err87);
+      Sleep(5);
+      HANDLE h2 = open_ro(p);
+      if (h2 != INVALID_HANDLE_VALUE) {
+        InterlockedIncrement(&g_err87_recovered);
+        CloseHandle(h2);
+      }
+      fwprintf(stderr, L"[c] errno87 %ls\n", p);
+    } else if (a.err != 0) {
+      InterlockedIncrement(&g_other);
+    }
+    CloseHandle(h1);
+  }
+}
+
+int wmain(int argc, wchar_t** argv) {
+  const wchar_t* dir = (argc > 1) ? argv[1] : L"src\\third_party";
+  int nworkers = (argc > 2) ? _wtoi(argv[2]) : 32;
+  int rounds = (argc > 3) ? _wtoi(argv[3]) : 12;
+
+  wprintf(L"dir=%ls workers=%d rounds=%d\n", dir, nworkers, rounds);
+  walk(dir);
+  wprintf(L"files=%ld\n", g_nfiles);
+  if (g_nfiles == 0)
+    return 1;
+
+  g_inner_sema = CreateSemaphoreW(NULL, nworkers, nworkers, NULL);
+  HANDLE* ths = (HANDLE*)calloc(nworkers, sizeof(HANDLE));
+
+  for (int r = 1; r <= rounds; r++) {
+    g_idx = 0;
+    DWORD t0 = GetTickCount();
+    for (int w = 0; w < nworkers; w++)
+      ths[w] = (HANDLE)_beginthreadex(NULL, 0, worker, NULL, 0, NULL);
+    for (int w = 0; w < nworkers; w++) {
+      WaitForSingleObject(ths[w], INFINITE);
+      CloseHandle(ths[w]);
+    }
+    wprintf(L"round %d/%d: %ld opens in %lums (err87 so far: %ld)\n", r, rounds,
+            g_nfiles, GetTickCount() - t0, g_err87);
+  }
+
+  wprintf(L"\n=== summary ===\nopens=%ld err87=%ld recovered=%ld other=%ld\n",
+          (long)g_nfiles * rounds, g_err87, g_err87_recovered, g_other);
+  return 0;
+}