diff --git a/.github/workflows/Repology.yml b/.github/workflows/Repology.yml index 9177cb61c..1083f985b 100644 --- a/.github/workflows/Repology.yml +++ b/.github/workflows/Repology.yml @@ -2,7 +2,7 @@ name: Generate Repology JSON on: schedule: - - cron: '0 0 * * 0' # Weekly + - cron: '0 12 * * 0' # Weekly jobs: generate: runs-on: ubuntu-24.04 @@ -26,7 +26,7 @@ jobs: run: | ruby -Ctools json.rb - name: Create Pull Request - uses: peter-evans/create-pull-request@v6 + uses: peter-evans/create-pull-request@v7 with: add-paths: "tools/repology.json" title: "Update Repology JSON" diff --git a/.github/workflows/Updater.yml b/.github/workflows/Updater.yml new file mode 100644 index 000000000..a1ae9d755 --- /dev/null +++ b/.github/workflows/Updater.yml @@ -0,0 +1,146 @@ +--- +name: Generate Updates PR +on: + schedule: + - cron: '0 0 * * *' # Daily +jobs: + update-check: + permissions: + actions: 'write' + runs-on: ubuntu-24.04 + outputs: + output1: ${{ steps.update-check.outputs.BRANCH_NAME }} # https://stackoverflow.com/a/75142892 + output2: ${{ steps.update-check.outputs.TIMESTAMP }} + steps: + - uses: actions/checkout@v4 + - uses: ruby/setup-ruby@v1 + with: + ruby-version: '3.3.6' + - name: Install Python pip + run: sudo apt install -y python3-pip + - name: Install activesupport + run: sudo apt install -y ruby-activesupport + - name: Configure git + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' + - name: Check for updates in pip packages. + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ruby -Ctools update_python_pip_packages.rb + # Create a new branch with the updated package files only + # if there are updated packages. Otherwise exit early. + if [ -n "$(git status --porcelain)" ]; then + echo "BRANCH_NAME=updates-$(date --date='UTC' +%F-%H)" >> $GITHUB_OUTPUT + echo "TIMESTAMP=$(date --date='UTC' +%F-%H)" >> $GITHUB_OUTPUT + git checkout -b ${BRANCH_NAME} + git add -A + git commit -m "Add unbuilt updated pip packages to ${BRANCH_NAME}" + git push + else + # https://stackoverflow.com/a/75809743 + gh run cancel ${{ github.run_id }} + gh run watch ${{ github.run_id }} + fi + generate: + strategy: + matrix: + arch: [x86_64, armv7l, i686] + runner: [self-hosted, ubuntu-24.04] + max-parallel: 1 + exclude: + - arch: x86_64 + runner: self-hosted + - arch: i686 + runner: self-hosted + - arch: armv7l + runner: ubuntu-24.04 + runs-on: ${{ matrix.runner }} + needs: update-check + if: ${{ !cancelled() }} + concurrency: + group: ${{ matrix.arch }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: true + - name: Export target docker container to github context + env: + TARGET_ARCH: ${{ matrix.arch }} + run: | + case $TARGET_ARCH in + x86_64) + echo "CONTAINER=nocturne-x86_64.m90" >> $GITHUB_ENV + echo "PLATFORM=linux/amd64" >> $GITHUB_ENV + echo "LIB_SUFFIX=64" >> $GITHUB_ENV + ;; + armv7l) + echo "CONTAINER=fievel-armv7l.m91" >> $GITHUB_ENV + echo "PLATFORM=linux/arm/v7" >> $GITHUB_ENV + echo "LIB_SUFFIX=" >> $GITHUB_ENV + ;; + i686) + echo "CONTAINER=alex-i686.m58" >> $GITHUB_ENV + echo "PLATFORM=linux/386" >> $GITHUB_ENV + echo "LIB_SUFFIX=" >> $GITHUB_ENV + ;; + esac + - name: Run Updater in container + env: + CREW_REPO: "https://github.com/chromebrew/chromebrew.git" + CREW_BRANCH: ${{ needs.update-check.outputs.output1 }} + run: | + git config --global user.name 'github-actions[bot]' + git config --global user.email '41898282+github-actions[bot]@users.noreply.github.com' + git pull && git checkout ${CREW_BRANCH} + docker pull --platform ${PLATFORM} satmandu/crewbuild:${CONTAINER} + docker run \ + --rm \ + --platform ${PLATFORM} \ + --privileged \ + -u chronos \ + -e LD_LIBRARY_PATH="/usr/local/lib${LIB_SUFFIX}" \ + -e GCONV_PATH="/usr/local/lib${LIB_SUFFIX}/gconv" \ + -e CREW_REPO="${CREW_REPO}" \ + -e CREW_BRANCH="${CREW_BRANCH}" \ + -e GITLAB_TOKEN="${{ secrets.GITLAB_TOKEN }}" \ + -e GITLAB_TOKEN_USERNAME="${{ secrets.GITLAB_TOKEN_USERNAME }}" \ + -v $(pwd):/output \ + "satmandu/crewbuild:${CONTAINER}" \ + /bin/chromebrewstart /output/tools/github_actions_update_builder.sh + - name: Add updated packages to PR. + env: + BRANCH_NAME: ${{ needs.update-check.outputs.output1 }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -n "$(git status --porcelain)" ]; then + git add -A + git commit -m "Add updated pip packages for ${PLATFORM} to ${BRANCH_NAME}" + git push + fi + linter-tests: + runs-on: ubuntu-24.04 + needs: + - update-check + - generate + if: ${{ !cancelled() }} + steps: + - name: fail if update jobs failed, otherwise create a PR + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + - run: echo "Update jobs succeeded. Creating a PR." + - name: Create Pull Request + uses: peter-evans/create-pull-request@v7 + with: + add-paths: | + manifest/** + packages/** + branch: ${{ needs.update-check.outputs.output1 }} + title: "Updated Packages for ${{ needs.update-check.outputs.output2 }}" + body: "Automatic PR to update packages" + author: "github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>" + commit-message: "Automated Package Updates for ${{ needs.update-check.outputs.output2 }}" + sign-commits: true + delete-branch: true diff --git a/tools/build_updated_packages.rb b/tools/build_updated_packages.rb index 631f007bb..6c0d21083 100755 --- a/tools/build_updated_packages.rb +++ b/tools/build_updated_packages.rb @@ -1,5 +1,5 @@ #!/usr/bin/env ruby -# build_updated_packages version 2.0 (for Chromebrew) +# build_updated_packages version 2.1 (for Chromebrew) # This updates the versions in python pip packages by calling # tools/update_python_pip_packages.rb, checks for updated ruby packages # by calling tools/update_ruby_gem_packages.rb, and then checks if any @@ -81,7 +81,7 @@ else load 'tools/update_ruby_gem_packages.rb' end changed_files = `git diff HEAD --name-only`.chomp.split -changed_files_previous_commit = `git diff --name-only HEAD HEAD~1`.chomp.split +changed_files_previous_commit = `git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/master)..$(git rev-parse --verify HEAD)`.chomp.split updated_packages = changed_files.select { |c| c.include?('packages/') } updated_packages.push(*changed_files_previous_commit.select { |c| c.include?('packages/') }) diff --git a/tools/github_actions_update_builder.sh b/tools/github_actions_update_builder.sh new file mode 100755 index 000000000..ce1e50068 --- /dev/null +++ b/tools/github_actions_update_builder.sh @@ -0,0 +1,11 @@ +#!/bin/bash +if ![[ -n ${GITLAB_TOKEN_USERNAME-} ]]; then + echo "GITLAB_TOKEN_USERNAME not set." + exit 1 +fi +if ![[ -n ${GITLAB_TOKEN-} ]]; then + echo "GITLAB_TOKEN not set." + exit 1 +fi +export CREW_AGREE_TIMEOUT_SECONDS=1 +tools/build_updated_packages.rb --skip $@ diff --git a/tools/update_python_pip_packages.rb b/tools/update_python_pip_packages.rb index e1d555c5e..ede8e7e11 100755 --- a/tools/update_python_pip_packages.rb +++ b/tools/update_python_pip_packages.rb @@ -1,12 +1,18 @@ #!/usr/bin/env ruby -# update_python_pip_packages version 1.2 (for Chromebrew) +# update_python_pip_packages version 1.3 (for Chromebrew) # This updates the versions in python pip packages. # # Author: Satadru Pramanik (satmandu) satadru at gmail dot com # Usage in root of cloned chromebrew repo: # tools/update_python_pip_packages.rb + +# Add >LOCAL< lib to LOAD_PATH +$LOAD_PATH.unshift '../lib' require_relative '../lib/color' -require_relative '../lib/const' + +CREW_GITLAB_PKG_REPO = 'https://gitlab.com/api/v4/projects/26210301/packages' +CREW_NPROC = `nproc`.chomp + def require_gem(gem_name_and_require = nil, require_override = nil) # Allow only loading gems when needed. return if gem_name_and_require.nil? @@ -51,7 +57,7 @@ relevant_pip_packages.each_with_index do |package, index| pip_name = package.gsub('.rb', '').sub('py3_', '').gsub('_', '-').gsub('packages/', '') prerelease = system("grep -q '^\ \ prerelease' #{package}") ? '--pre' : nil puts "\e[1A\e[K[#{(index + 1).to_s.rjust(numlength)}/#{total_files_to_check}] Checking pypi for #{prerelease.blank? ? '' : 'prerelease '}updates to #{pip_name}...\r".orange - pip_version = `python -m pip index versions #{prerelease} #{pip_name} 2>/dev/null | head -n 1 | awk '{print $2}'`.chomp.delete('()') + pip_version = `python3 -m pip index versions #{prerelease} #{pip_name} 2>/dev/null | head -n 1 | awk '{print $2}'`.chomp.delete('()') next package if pip_version.blank? relevant_pip_packages.delete(package)