ci: make src-cache upload atomic (#50747)

ci: make src-cache upload atomic and sweep orphaned temp files

The checkout action's cp of the ~6GB zstd archive directly to the final
path on the cache share is non-atomic; an interrupted copy or a
concurrent reader produces zstd "Read error (39): premature end" on
restore, and the truncated file then satisfies the existence check so
no later run repairs it.

Upload to a run-unique *.tar.upload-<run_id>-<attempt> temp name on the
share and mv to the final path, discarding our temp if a concurrent run
got there first. A new clean-orphaned-cache-uploads workflow removes
temp files older than 4h every 4 hours.

Co-authored-by: trop[bot] <37223003+trop[bot]@users.noreply.github.com>
Co-authored-by: Sam Attard <sattard@anthropic.com>

.github/actions/checkout/action.yml

@@ -191,19 +191,31 @@ runs:
       # only permits .tar/.tgz so we keep the extension and decode on restore.
       tar -cf - src | zstd -T0 --long=30 -f -o $CACHE_FILE
       echo "Compressed src to $(du -sh $CACHE_FILE | cut -f1 -d' ')"
-      cp ./$CACHE_FILE $CACHE_DRIVE/
   - name: Persist Src Cache
     if: ${{ steps.check-cache.outputs.cache_exists == 'false' && inputs.use-cache == 'true' }}
     shell: bash
     run: |
       final_cache_path=$CACHE_DRIVE/$CACHE_FILE
+      # Upload to a run-unique temp name first so concurrent readers never
+      # observe a partially-written file, and an interrupted copy can't leave
+      # a truncated file at the final path. Orphaned temp files get swept by
+      # the clean-orphaned-cache-uploads workflow.
+      tmp_cache_path=$final_cache_path.upload-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}
+      echo "Uploading to temp path: $tmp_cache_path"
+      cp ./$CACHE_FILE $tmp_cache_path
+
       echo "Using cache key: $DEPSHASH"
-      echo "Checking path: $final_cache_path"
+      if [ -f "$final_cache_path" ]; then
+        echo "Cache already persisted at $final_cache_path by a concurrent run; discarding ours"
+        rm -f $tmp_cache_path
+      else
+        mv -f $tmp_cache_path $final_cache_path
+        echo "Cache key persisted in $final_cache_path"
+      fi
+
       if [ ! -f "$final_cache_path" ]; then
         echo "Cache key not found"
         exit 1
-      else
-        echo "Cache key persisted in $final_cache_path"
       fi
   - name: Wait for active SSH sessions
     shell: bash

.github/workflows/clean-orphaned-cache-uploads.yml

@@ -0,0 +1,32 @@
+name: Clean Orphaned Cache Uploads
+
+# Description:
+# Sweeps orphaned in-flight upload temp files left on the src-cache volumes
+# by checkout/action.yml when its cp-to-share step dies before the rename.
+# A successful upload finishes in minutes, so anything older than 4h is dead.
+
+on:
+  schedule:
+    - cron: "0 */4 * * *"
+  workflow_dispatch:
+
+permissions: {}
+
+jobs:
+  clean-orphaned-uploads:
+    if: github.repository == 'electron/electron'
+    runs-on: electron-arc-centralus-linux-amd64-32core
+    permissions:
+      contents: read
+    container:
+      image: ghcr.io/electron/build:bc2f48b2415a670de18d13605b1cf0eb5fdbaae1
+      options: --user root
+      volumes:
+        - /mnt/cross-instance-cache:/mnt/cross-instance-cache
+        - /mnt/win-cache:/mnt/win-cache
+    steps:
+    - name: Remove Orphaned Upload Temp Files
+      shell: bash
+      run: |
+        find /mnt/cross-instance-cache -maxdepth 1 -type f -name '*.tar.upload-*' -mmin +240 -print -delete
+        find /mnt/win-cache -maxdepth 1 -type f -name '*.tar.upload-*' -mmin +240 -print -delete