build: cache gitcache dir (#47407)

build: cache gitcache dir (#47328)

* revert build: migrate to new chromium git auth

* build: cache gitcache dir

Co-authored-by: trop[bot] <37223003+trop[bot]@users.noreply.github.com>
Co-authored-by: John Kleinschmidt <jkleinsc@electronjs.org>
This commit is contained in:
trop[bot]
2025-06-09 19:13:27 -04:00
committed by GitHub
parent ded238107f
commit eb6c475b5f
14 changed files with 254 additions and 75 deletions

View File

@@ -0,0 +1,83 @@
name: 'Build Git Cache'
description: 'Runs a gclient sync to build the git cache for Electron'
inputs:
target-platform:
description: 'Target platform, should be linux, win, macos'
runs:
using: "composite"
steps:
- name: Set GIT_CACHE_PATH to make gclient to use the cache
shell: bash
run: |
echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Set up cache drive
shell: bash
run: |
if [ "${{ inputs.target-platform }}" = "win" ]; then
echo "CACHE_DRIVE=/mnt/win-cache" >> $GITHUB_ENV
else
echo "CACHE_DRIVE=/mnt/cross-instance-cache" >> $GITHUB_ENV
fi
- name: Check cross instance cache disk space
shell: bash
run: |
# if there is less than 35 GB free space then creating the cache might fail so exit early
freespace=`df -m $CACHE_DRIVE | grep -w $CACHE_DRIVE | awk '{print $4}'`
freespace_human=`df -h $CACHE_DRIVE | grep -w $CACHE_DRIVE | awk '{print $4}'`
if [ $freespace -le 35000 ]; then
echo "The cross mount cache has $freespace_human free space which is not enough - exiting"
exit 1
else
echo "The cross mount cache has $freespace_human free space - continuing"
fi
- name: Restore gitcache
shell: bash
run: |
GIT_CACHE_TAR="$CACHE_DRIVE/gitcache.tar"
if [ ! -f "$GIT_CACHE_TAR" ]; then
echo "Git cache tar file does not exist, skipping restore"
exit 0
fi
echo "Restoring git cache from $GIT_CACHE_TAR to $GIT_CACHE_PATH"
mkdir -p $GIT_CACHE_PATH
tar -xf $GIT_CACHE_TAR -C $GIT_CACHE_PATH
- name: Gclient Sync
shell: bash
run: |
e d gclient config \
--name "src/electron" \
--unmanaged \
${GCLIENT_EXTRA_ARGS} \
"$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
if [ "$TARGET_OS" != "" ]; then
echo "target_os=['$TARGET_OS']" >> ./.gclient
fi
ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 e d gclient sync --with_branch_heads --with_tags --nohooks -vv
- name: Compress Git Cache Directory
shell: bash
run: |
echo "Uncompressed gitcache size: $(du -sh $GIT_CACHE_PATH | cut -f1 -d' ')"
cd $GIT_CACHE_PATH
tar -cf ../gitcache.tar .
cd ..
echo "Compressed gitcache to $(du -sh gitcache.tar | cut -f1 -d' ')"
# remove the old cache file if it exists
if [ -f $CACHE_DRIVE/gitcache.tar ]; then
echo "Removing old gitcache.tar from $CACHE_DRIVE"
rm $CACHE_DRIVE/gitcache.tar
fi
cp ./gitcache.tar $CACHE_DRIVE/
- name: Wait for active SSH sessions
shell: bash
if: always() && !cancelled()
run: |
while [ -f /var/.ssh-lock ]
do
sleep 60
done

View File

@@ -20,8 +20,8 @@ runs:
echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV
- name: Install Dependencies
uses: ./src/electron/.github/actions/install-dependencies
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Generate DEPS Hash
@@ -83,6 +83,18 @@ runs:
- name: Add patch conflict problem matcher
shell: bash
run: echo "::add-matcher::src/electron/.github/problem-matchers/patch-conflict.json"
- name: Restore gitcache
if: steps.check-cache.outputs.cache_exists == 'false'
shell: bash
run: |
GIT_CACHE_TAR="$CACHE_DRIVE/gitcache.tar"
if [ ! -f "$GIT_CACHE_TAR" ]; then
echo "Git cache tar file does not exist, skipping restore"
exit 0
fi
echo "Restoring git cache from $GIT_CACHE_TAR to $GIT_CACHE_PATH"
mkdir -p $GIT_CACHE_PATH
tar -xf $GIT_CACHE_TAR -C $GIT_CACHE_PATH
- name: Gclient Sync
if: steps.check-cache.outputs.cache_exists == 'false'
shell: bash

View File

@@ -0,0 +1,58 @@
name: 'Set Chromium Git Cookie'
description: 'Sets an authenticated cookie from Chromium to allow for a higher request limit'
runs:
using: "composite"
steps:
- name: Set the git cookie from chromium.googlesource.com (Unix)
if: ${{ runner.os != 'Windows' }}
shell: bash
run: |
if [[ -z "${{ env.CHROMIUM_GIT_COOKIE }}" ]]; then
echo "CHROMIUM_GIT_COOKIE is not set - cannot authenticate."
exit 0
fi
eval 'set +o history' 2>/dev/null || setopt HIST_IGNORE_SPACE 2>/dev/null
touch ~/.gitcookies
chmod 0600 ~/.gitcookies
git config --global http.cookiefile ~/.gitcookies
tr , \\t <<\__END__ >>~/.gitcookies
${{ env.CHROMIUM_GIT_COOKIE }}
__END__
eval 'set -o history' 2>/dev/null || unsetopt HIST_IGNORE_SPACE 2>/dev/null
RESPONSE=$(curl -s -b ~/.gitcookies https://chromium-review.googlesource.com/a/accounts/self)
if [[ $RESPONSE == ")]}'"* ]]; then
# Extract account email for verification
EMAIL=$(echo "$RESPONSE" | tail -c +5 | jq -r '.email // "No email found"')
echo "Cookie authentication successful - authenticated as: $EMAIL"
else
echo "Cookie authentication failed - ensure CHROMIUM_GIT_COOKIE is set correctly"
echo $RESPONSE
fi
- name: Set the git cookie from chromium.googlesource.com (Windows)
if: ${{ runner.os == 'Windows' }}
shell: cmd
run: |
if "%CHROMIUM_GIT_COOKIE_WINDOWS_STRING%"=="" (
echo CHROMIUM_GIT_COOKIE_WINDOWS_STRING is not set - cannot authenticate.
exit /b 0
)
git config --global http.cookiefile "%USERPROFILE%\.gitcookies"
powershell -noprofile -nologo -command Write-Output "${{ env.CHROMIUM_GIT_COOKIE_WINDOWS_STRING }}" >>"%USERPROFILE%\.gitcookies"
curl -s -b "%USERPROFILE%\.gitcookies" https://chromium-review.googlesource.com/a/accounts/self > response.txt
findstr /B /C:")]}'" response.txt > nul
if %ERRORLEVEL% EQU 0 (
echo Cookie authentication successful
powershell -NoProfile -Command "& {$content = Get-Content -Raw response.txt; $content = $content.Substring(4); try { $json = ConvertFrom-Json $content; if($json.email) { Write-Host 'Authenticated as:' $json.email } else { Write-Host 'No email found in response' } } catch { Write-Host 'Error parsing JSON:' $_ }}"
) else (
echo Cookie authentication failed - ensure CHROMIUM_GIT_COOKIE_WINDOWS_STRING is set correctly
type response.txt
)
del response.txt

View File

@@ -1,41 +0,0 @@
name: 'Set Chromium Git Helper'
description: 'Sets Chromium Git Helper to allow for a higher request limit'
runs:
using: "composite"
steps:
- name: Save the chromium git credentials to a file
shell: bash
run: |
if [[ -z "${{ env.CHROMIUM_GIT_AUTH }}" ]]; then
echo "CHROMIUM_GIT_AUTH is not set - cannot authenticate."
exit 0
fi
if [[ "${{ runner.os }}" != "Windows" ]]; then
cd $HOME
fi
echo "${{ env.CHROMIUM_GIT_AUTH }}" > .chromium_git_auth
- name: Set the chromium git helper to use auth from a file
shell: bash
run: |
if [[ "${{ runner.os }}" == "Windows" ]]; then
if [[ ! -f "/c/actions-runner/_work/electron/electron/.chromium_git_auth" ]]; then
echo "File /c/actions-runner/_work/electron/electron/.chromium_git_auth does not exist - cannot authenticate."
exit 0
fi
else
if [[ ! -f "$HOME/.chromium_git_auth" ]]; then
echo "File $HOME/.chromium_git_auth does not exist - cannot authenticate."
exit 0
fi
fi
if [[ -z "${{ env.CHROMIUM_GIT_USER }}" ]]; then
echo "CHROMIUM_GIT_USER is not set - cannot authenticate."
exit 0
fi
git config --global credential.https://chromium.googlesource.com.username "${{ env.CHROMIUM_GIT_USER }}"
if [[ "${{ runner.os }}" == "Windows" ]]; then
git config --global credential.https://chromium.googlesource.com.helper '!f() { test "$1" = get && echo "password=$(cat /c/actions-runner/_work/electron/electron/.chromium_git_auth)"; }; f'
else
git config --global credential.https://chromium.googlesource.com.helper '!f() { test "$1" = get && echo "password=$(cat $HOME/.chromium_git_auth)"; }; f'
fi

74
.github/workflows/build-git-cache.yml vendored Normal file
View File

@@ -0,0 +1,74 @@
name: Build Git Cache
# This workflow updates git cache on the cross-instance cache volumes
# It runs daily at midnight.
on:
schedule:
- cron: "0 0 * * *"
jobs:
build-git-cache-linux:
runs-on: electron-arc-linux-amd64-32core
container:
image: ghcr.io/electron/build:bc2f48b2415a670de18d13605b1cf0eb5fdbaae1
options: --user root
volumes:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
env:
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- name: Checkout Electron
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: src/electron
fetch-depth: 0
- name: Build Git Cache
uses: ./src/electron/.github/actions/build-git-cache
with:
target-platform: linux
build-git-cache-windows:
runs-on: electron-arc-linux-amd64-32core
container:
image: ghcr.io/electron/build:bc2f48b2415a670de18d13605b1cf0eb5fdbaae1
options: --user root --device /dev/fuse --cap-add SYS_ADMIN
volumes:
- /mnt/win-cache:/mnt/win-cache
env:
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True'
TARGET_OS: 'win'
steps:
- name: Checkout Electron
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: src/electron
fetch-depth: 0
- name: Build Git Cache
uses: ./src/electron/.github/actions/build-git-cache
with:
target-platform: win
build-git-cache-macos:
runs-on: electron-arc-linux-amd64-32core
# This job updates the same git cache as linux, so it needs to run after the linux one.
needs: build-git-cache-linux
container:
image: ghcr.io/electron/build:bc2f48b2415a670de18d13605b1cf0eb5fdbaae1
options: --user root
volumes:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
env:
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- name: Checkout Electron
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
with:
path: src/electron
fetch-depth: 0
- name: Build Git Cache
uses: ./src/electron/.github/actions/build-git-cache
with:
target-platform: macos

View File

@@ -100,8 +100,7 @@ jobs:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
outputs:
build-image-sha: ${{ needs.setup.outputs.build-image-sha }}
@@ -129,8 +128,7 @@ jobs:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
PATCH_UP_APP_CREDS: ${{ secrets.PATCH_UP_APP_CREDS }}
outputs:
@@ -156,8 +154,8 @@ jobs:
- /mnt/win-cache:/mnt/win-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
CHROMIUM_GIT_COOKIE_WINDOWS_STRING: ${{ secrets.CHROMIUM_GIT_COOKIE_WINDOWS_STRING }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True'
TARGET_OS: 'win'
ELECTRON_DEPOT_TOOLS_WIN_TOOLCHAIN: '1'

View File

@@ -27,8 +27,7 @@ jobs:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True'
steps:
- name: Checkout Electron

View File

@@ -28,8 +28,7 @@ jobs:
- /mnt/cross-instance-cache:/mnt/cross-instance-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac'
steps:
- name: Checkout Electron

View File

@@ -13,8 +13,7 @@ concurrency:
cancel-in-progress: ${{ github.ref_protected != true }}
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
jobs:
lint:
@@ -31,8 +30,8 @@ jobs:
ref: ${{ github.event.pull_request.head.sha }}
- name: Install Dependencies
uses: ./src/electron/.github/actions/install-dependencies
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Setup third_party Depot Tools
shell: bash
run: |

View File

@@ -65,8 +65,8 @@ concurrency:
cancel-in-progress: ${{ github.ref_protected != true }}
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
CHROMIUM_GIT_COOKIE_WINDOWS_STRING: ${{ secrets.CHROMIUM_GIT_COOKIE_WINDOWS_STRING }}
ELECTRON_ARTIFACTS_BLOB_STORAGE: ${{ secrets.ELECTRON_ARTIFACTS_BLOB_STORAGE }}
ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
SUDOWOODO_EXCHANGE_URL: ${{ secrets.SUDOWOODO_EXCHANGE_URL }}
@@ -127,8 +127,8 @@ jobs:
GN_EXTRA_ARGS='is_asan=true'
fi
echo "GN_EXTRA_ARGS=$GN_EXTRA_ARGS" >> $GITHUB_ENV
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Generate DEPS Hash

View File

@@ -66,8 +66,8 @@ jobs:
- name: Check disk space after freeing up space
if: ${{ inputs.target-platform == 'macos' }}
run: df -h
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Enable windows toolchain

View File

@@ -36,8 +36,8 @@ permissions:
pull-requests: read
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
CHROMIUM_GIT_COOKIE_WINDOWS_STRING: ${{ secrets.CHROMIUM_GIT_COOKIE_WINDOWS_STRING }}
ELECTRON_OUT_DIR: Default
ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
@@ -126,8 +126,8 @@ jobs:
ref: ${{ github.event.pull_request.head.sha }}
- name: Install Dependencies
uses: ./src/electron/.github/actions/install-dependencies
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Get Depot Tools
timeout-minutes: 5
run: |

View File

@@ -31,8 +31,7 @@ concurrency:
cancel-in-progress: ${{ github.ref_protected != true }}
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE: ${{ secrets.CHROMIUM_GIT_COOKIE }}
ELECTRON_OUT_DIR: Default
ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }}
@@ -52,8 +51,8 @@ jobs:
path: src/electron
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Init Build Tools
@@ -106,8 +105,8 @@ jobs:
path: src/electron
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- name: Set Chromium Git Helper
uses: ./src/electron/.github/actions/set-chromium-git-helper
- name: Set Chromium Git Cookie
uses: ./src/electron/.github/actions/set-chromium-cookie
- name: Install Build Tools
uses: ./src/electron/.github/actions/install-build-tools
- name: Init Build Tools

View File

@@ -28,8 +28,7 @@ jobs:
- /mnt/win-cache:/mnt/win-cache
- /var/run/sas:/var/run/sas
env:
CHROMIUM_GIT_AUTH: ${{ secrets.CHROMIUM_GIT_AUTH }}
CHROMIUM_GIT_USER: ${{ secrets.CHROMIUM_GIT_USER }}
CHROMIUM_GIT_COOKIE_WINDOWS_STRING: ${{ secrets.CHROMIUM_GIT_COOKIE_WINDOWS_STRING }}
GCLIENT_EXTRA_ARGS: '--custom-var=checkout_win=True'
TARGET_OS: 'win'
ELECTRON_DEPOT_TOOLS_WIN_TOOLCHAIN: '1'