From d22aef3b8e64a5fc4a3caa2fd90b36d1379ac00a Mon Sep 17 00:00:00 2001 From: Ed Reel Date: Mon, 17 Feb 2025 13:38:15 -0600 Subject: [PATCH] Add No-Compile-Needed.yml workflow (#11352) --- .github/workflows/No-Compile-Needed.yml | 371 +++++++++++++++++++++ tools/github_actions_no_compile_updater.sh | 9 + tools/update_no_compile_packages.rb | 58 ++++ 3 files changed, 438 insertions(+) create mode 100644 .github/workflows/No-Compile-Needed.yml create mode 100755 tools/github_actions_no_compile_updater.sh create mode 100755 tools/update_no_compile_packages.rb diff --git a/.github/workflows/No-Compile-Needed.yml b/.github/workflows/No-Compile-Needed.yml new file mode 100644 index 000000000..db6394a3f --- /dev/null +++ b/.github/workflows/No-Compile-Needed.yml @@ -0,0 +1,371 @@ +--- +name: No-Compile-Needed +on: + workflow_dispatch: + inputs: + branch: + description: "Branch of chromebrew/chromebrew to run on, if different from this branch." + required: false +env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # setting GH_TOKEN for the entire workflow + BRANCH: ${{ inputs.branch || github.ref_name }} +permissions: + actions: write + contents: write + packages: write + pull-requests: write + repository-projects: read +jobs: + debug: + if: ${{ ( github.repository_owner == 'chromebrew' ) }} + runs-on: ubuntu-24.04 + steps: + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJson(job) }} + run: echo "$JOB_CONTEXT" + - name: Dump steps context + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + setup: + if: ${{ ( github.repository_owner == 'chromebrew' ) && ( inputs.branch != 'master' ) }} + runs-on: ubuntu-24.04 + outputs: + timestamp: ${{ steps.set-timestamp.outputs.TIMESTAMP }} # https://stackoverflow.com/a/75142892 + changed_packages: ${{ steps.changed-packages.outputs.CHANGED_PACKAGES }} + no_compile_packages: ${{ steps.get-compatibility.outputs.NO_COMPILE_PACKAGES }} + glibc_232_compat: ${{ steps.get-compatibility.outputs.GLIBC_232_COMPATIBLE_PACKAGES }} + glibc_237_compat: ${{ steps.get-compatibility.outputs.GLIBC_237_COMPATIBLE_PACKAGES }} + i686_packages: ${{ steps.get-compatibility.outputs.i686_PACKAGES }} + x86_64_packages: ${{ steps.get-compatibility.outputs.x86_64_PACKAGES }} + armv7l_packages: ${{ steps.get-compatibility.outputs.armv7l_PACKAGES }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: true + ref: ${{ inputs.branch || github.ref_name }} + - name: Rebase to master + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git pull --rebase origin master + git push -f + - name: Push rebase changes + uses: ad-m/github-push-action@master + with: + branch: ${{ env.BRANCH }} + force: true + - name: Set Timestamp + id: set-timestamp + run: | + TIMESTAMP="$(date -u +%F-%H%Z)" + export TIMESTAMP + echo "TIMESTAMP=${TIMESTAMP}" >> "$GITHUB_OUTPUT" + - name: Get all changed package files + id: changed-ruby-files + uses: tj-actions/changed-files@v45 + with: + base_sha: master + files: packages/*.rb + since_last_remote_commit: true + - name: Export variables to github context + id: changed-packages + run: | + if [[ -z "${{ steps.changed-ruby-files.outputs.all_changed_files }}" ]]; then + echo "Branch ${{ env.BRANCH }} has no changed files." + exit 1 + fi + # Convert "packages/foo.rb packages/bar.rb" (from steps.changed-ruby-files.outputs.all_changed_files) into "foo bar" + echo "CHANGED_PACKAGES=$(echo "${{ steps.changed-ruby-files.outputs.all_changed_files }}" | xargs basename -s .rb | xargs)" >> "$GITHUB_ENV" + echo "CHANGED_PACKAGES=$(echo "${{ steps.changed-ruby-files.outputs.all_changed_files }}" | xargs basename -s .rb | xargs)" >> "$GITHUB_OUTPUT" + - name: Determine glibc and architecture package compatibility + id: get-compatibility + run: | + # If a package has 'no_compile_needed', add it to NO_COMPILE_PACKAGES. + NO_COMPILE_PACKAGES="$(for i in ${CHANGED_PACKAGES}; do grep -q "[[:space:]]no_compile_needed" packages/${i}.rb && echo ${i}; done | xargs)" + export NO_COMPILE_PACKAGES + if [[ -n ${NO_COMPILE_PACKAGES} ]]; then + echo "NO_COMPILE_PACKAGES=${NO_COMPILE_PACKAGES}" >> $GITHUB_ENV + echo "NO_COMPILE_PACKAGES=${NO_COMPILE_PACKAGES}" >> $GITHUB_OUTPUT + echo "Branch ${{ env.BRANCH }} has these no_compile_needed packages: ${NO_COMPILE_PACKAGES}" + else + echo "Branch ${{ env.BRANCH }} is missing no_compile_needed packages that require updates." + exit 1 + fi + + # If a package doesnt have a min_glibc value, or if it is below 2.32, add it to GLIBC_232_COMPATIBLE_PACKAGES. + GLIBC_232_COMPATIBLE_PACKAGES="$(for i in ${NO_COMPILE_PACKAGES} ; do grep min_glibc packages/"${i}".rb | tr -d \' | awk '{exit $2 <= 2.32}' || echo "${i}" ; done | xargs)" + export GLIBC_232_COMPATIBLE_PACKAGES + if [[ -n ${GLIBC_232_COMPATIBLE_PACKAGES} ]]; then + echo "GLIBC_232_COMPATIBLE_PACKAGES=${GLIBC_232_COMPATIBLE_PACKAGES}" >> "$GITHUB_ENV" + echo "GLIBC_232_COMPATIBLE_PACKAGES=${GLIBC_232_COMPATIBLE_PACKAGES}" >> "$GITHUB_OUTPUT" + echo "Branch ${{ env.BRANCH }} has these possibly Glibc 2.32 compatible packages: ${GLIBC_232_COMPATIBLE_PACKAGES}" + fi + + # If a package doesnt have a min_glibc value, or if it is below 2.37, add it to GLIBC_237_COMPATIBLE_PACKAGES. + GLIBC_237_COMPATIBLE_PACKAGES="$(for i in ${NO_COMPILE_PACKAGES} ; do grep min_glibc packages/"${i}".rb | tr -d \' | awk '{exit $2 <= 2.37}' || echo "${i}" ; done | xargs)" + export GLIBC_237_COMPATIBLE_PACKAGES + if [[ -n ${GLIBC_237_COMPATIBLE_PACKAGES} ]]; then + echo "GLIBC_237_COMPATIBLE_PACKAGES=${GLIBC_237_COMPATIBLE_PACKAGES}" >> "$GITHUB_ENV" + echo "GLIBC_237_COMPATIBLE_PACKAGES=${GLIBC_237_COMPATIBLE_PACKAGES}" >> "$GITHUB_OUTPUT" + echo "Branch ${{ env.BRANCH }} has these possibly Glibc 2.37 compatible packages: ${GLIBC_237_COMPATIBLE_PACKAGES}" + fi + + # If a package has a compatibility of 'all' or one that includes 'x86_64', add it to x86_64_PACKAGES. + x86_64_PACKAGES="$(for i in ${NO_COMPILE_PACKAGES}; do grep -q "[[:space:]]compatibility.*all\|[[:space:]]compatibility.*x86_64" packages/"${i}".rb && echo "${i}"; done | xargs)" + export x86_64_PACKAGES + if [[ -n ${x86_64_PACKAGES} ]]; then + echo "x86_64_PACKAGES=${x86_64_PACKAGES}" >> "$GITHUB_ENV" + echo "x86_64_PACKAGES=${x86_64_PACKAGES}" >> "$GITHUB_OUTPUT" + echo "Branch ${{ env.BRANCH }} has these x86_64 compatible packages: ${x86_64_PACKAGES}" + fi + + ## If a package has a compatibility of 'all' or one that includes 'armv7l', add it to armv7l_PACKAGES. + armv7l_PACKAGES="$(for i in ${NO_COMPILE_PACKAGES}; do grep -q "[[:space:]]compatibility.*all\|[[:space:]]compatibility.*armv7l" packages/"${i}".rb && echo "${i}"; done | xargs)" + export armv7l_PACKAGES + if [[ -n ${armv7l_PACKAGES} ]]; then + echo "armv7l_PACKAGES=${armv7l_PACKAGES}" >> "$GITHUB_ENV" + echo "armv7l_PACKAGES=${armv7l_PACKAGES}" >> "$GITHUB_OUTPUT" + echo "Branch ${{ env.BRANCH }} has these armv7l compatible packages: ${armv7l_PACKAGES}" + fi + + ## If a package has a compatibility of 'all' or one that includes 'i686', add it to i686_PACKAGES. + i686_PACKAGES="$(for i in ${NO_COMPILE_PACKAGES}; do grep -q "[[:space:]]compatibility.*all\|[[:space:]]compatibility.*i686" packages/"${i}".rb && echo "${i}"; done | xargs)" + export i686_PACKAGES + if [[ -n ${i686_PACKAGES} ]]; then + echo "i686_PACKAGES=${i686_PACKAGES}" >> "$GITHUB_ENV" + echo "i686_PACKAGES=${i686_PACKAGES}" >> "$GITHUB_OUTPUT" + echo "Branch ${{ env.BRANCH }} has these i686 compatible packages: ${i686_PACKAGES}" + fi + generate: + strategy: + max-parallel: 1 + matrix: + arch: [i686, x86_64, armv7l] + runner: + - [self-hosted, X64] + - [self-hosted, ARM] + exclude: + - arch: x86_64 + runner: [self-hosted, ARM] + - arch: i686 + runner: [self-hosted, ARM] + - arch: armv7l + runner: [self-hosted, X64] + runs-on: ${{ matrix.runner }} + needs: setup + env: + CREW_REPO: ${{ github.event.repository.clone_url }} + CREW_BRANCH: ${{ inputs.branch || github.ref_name }} + TARGET_ARCH: ${{ matrix.arch }} + TIMESTAMP: ${{ needs.setup.outputs.timestamp }} + NO_COMPILE_PACKAGES: ${{ needs.setup.outputs.no_compile_packages }} + GLIBC_232_COMPATIBLE_PACKAGES: ${{ needs.setup.outputs.glibc_232_compat }} + GLIBC_237_COMPATIBLE_PACKAGES: ${{ needs.setup.outputs.glibc_237_compat }} + i686_PACKAGES: ${{ needs.setup.outputs.i686_packages }} + x86_64_PACKAGES: ${{ needs.setup.outputs.x86_64_packages }} + armv7l_PACKAGES: ${{ needs.setup.outputs.armv7l_packages }} + if: ${{ !cancelled() }} + concurrency: + group: ${{ matrix.arch }}-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Container cleanup + run: | + sudo rm -rf release + - name: Dump GitHub context + env: + GITHUB_CONTEXT: ${{ toJson(github) }} + run: echo "$GITHUB_CONTEXT" + - name: Dump job context + env: + JOB_CONTEXT: ${{ toJson(job) }} + run: echo "$JOB_CONTEXT" + - name: Dump steps context + env: + STEPS_CONTEXT: ${{ toJson(steps) }} + run: echo "$STEPS_CONTEXT" + - name: Dump runner context + env: + RUNNER_CONTEXT: ${{ toJson(runner) }} + run: echo "$RUNNER_CONTEXT" + - name: Dump strategy context + env: + STRATEGY_CONTEXT: ${{ toJson(strategy) }} + run: echo "$STRATEGY_CONTEXT" + - name: Dump matrix context + env: + MATRIX_CONTEXT: ${{ toJson(matrix) }} + run: echo "$MATRIX_CONTEXT" + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: true + - name: Export target docker container to github context + run: | + case $TARGET_ARCH in + x86_64) + # Export the x86_64 container depending on whether this branch updates packages with appropriate minimum glibc. + if [[ $GLIBC_232_COMPATIBLE_PACKAGES ]]; then + echo "CONTAINER=nocturne-x86_64.m97" >> "$GITHUB_ENV" + elif [[ $GLIBC_237_COMPATIBLE_PACKAGES ]]; then + echo "CONTAINER=hatch-x86_64.m132" >> "$GITHUB_ENV" + else + echo "CONTAINER=nocturne-x86_64.m90" >> "$GITHUB_ENV" + fi + echo "PLATFORM=linux/amd64" >> "$GITHUB_ENV" + echo "LIB_SUFFIX=64" >> "$GITHUB_ENV" + ;; + armv7l) + # Export the armv7l container depending on whether this branch updates packages with appropriate minimum glibc. + if [[ $GLIBC_232_COMPATIBLE_PACKAGES ]]; then + echo "CONTAINER=fievel-armv7l.m97" >> "$GITHUB_ENV" + elif [[ $GLIBC_237_COMPATIBLE_PACKAGES ]]; then + echo "CONTAINER=strongbad-armv7l.m132" >> "$GITHUB_ENV" + else + echo "CONTAINER=fievel-armv7l.m91" >> "$GITHUB_ENV" + fi + echo "PLATFORM=linux/arm/v7" >> "$GITHUB_ENV" + echo "LIB_SUFFIX=" >> "$GITHUB_ENV" + ;; + i686) + # There is only one i686 container based upon M58 with glibc 2.23. + echo "CONTAINER=alex-i686.m58" >> "$GITHUB_ENV" + echo "PLATFORM=linux/386" >> "$GITHUB_ENV" + echo "LIB_SUFFIX=" >> "$GITHUB_ENV" + ;; + esac + - name: Run Updater in container + id: run-updater + if: ${{ !cancelled() }} + run: | + if [ "$PLATFORM" == 'linux/arm/v7' ] && [ -z "${armv7l_PACKAGES}" ]; then + # Exit the arm container if there are not armv7l compatible packages. + echo "Skipping armv7l container." + exit 0 + elif [ "$PLATFORM" == 'linux/amd64' ] && [ -z "${x86_64_PACKAGES}" ]; then + # Exit the x86_64 container if there are not x86_64 compatible packages. + echo "Skipping x86_64 container." + exit 0 + elif [ "$PLATFORM" == 'linux/386' ] && [ -z "${i686_PACKAGES}" ]; then + # Exit the i686 container if there are not i686 compatible packages. + echo "Skipping i686 container." + exit 0 + fi + + git fetch origin + git checkout "${{ env.BRANCH }}" + git reset --hard "origin/${{ env.BRANCH }}" + git log --oneline -10 + docker pull --platform "${PLATFORM}" "satmandu/crewbuild:${CONTAINER}" + sudo apt install -y acl + sudo setfacl -R -m u:1000:rwx . + + # Use docker-in-docker shim to mount volume inside docker. + # docker run --rm -v /var/run/docker.sock:/var/run/docker.sock \ + # ghcr.io/felipecrs/dond-shim:latest \ + (cd /tmp || exit ; curl -OLf https://github.com/felipecrs/docker-on-docker-shim/raw/refs/tags/v0.7.1/dond ; chmod +x /tmp/dond ) + /tmp/dond run \ + --rm \ + --platform "${PLATFORM}" \ + --privileged \ + -u chronos \ + -e LD_LIBRARY_PATH="/usr/local/lib${LIB_SUFFIX}" \ + -e GCONV_PATH="/usr/local/lib${LIB_SUFFIX}/gconv" \ + -e CREW_REPO="${CREW_REPO}" \ + -e CREW_BRANCH="${CREW_BRANCH}" \ + -e GITLAB_TOKEN="${{ secrets.GITLAB_TOKEN }}" \ + -e GITLAB_TOKEN_USERNAME="${{ secrets.GITLAB_TOKEN_USERNAME }}" \ + -v "$(pwd)":/output \ + "satmandu/crewbuild:${CONTAINER}" \ + /bin/chromebrewstart /output/tools/github_actions_no_compile_updater.sh > >(tee -a /tmp/update.log) 2> >(tee -a /tmp/update.log >&2) + grep "Updated:" /tmp/update.log || true + sudo rm -rf release + - name: Add updated packages to branch. + id: push-check + run: | + if [ -n "$(git status --porcelain)" ]; then + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git add -A + git commit -m "Add updated packages for ${PLATFORM} to ${{ env.BRANCH }}" && git push -f + git log --oneline -10 + fi + update-check: + runs-on: ubuntu-24.04 + needs: + - setup + - generate + if: ${{ !cancelled() }} + steps: + - name: exit if update failed, otherwise create a PR + if: ${{ contains(needs.*.result, 'failure') }} + run: exit 1 + - name: Report update success + run: echo "Update jobs succeeded. Creating a PR." + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: true + ref: ${{ inputs.branch || github.ref_name }} + - name: Get all changed manifest files + id: changed-manifest-files + uses: tj-actions/changed-files@v45 + with: + base_sha: master + files: manifest/** + since_last_remote_commit: true + - name: Rebase to master + run: | + git config user.name "${{ github.actor }}" + git config user.email "${{ github.actor }}@users.noreply.github.com" + git fetch origin + git checkout "${{ env.BRANCH }}" + git reset --hard "origin/${{ env.BRANCH }}" + git pull --rebase origin master && git push -f + git log --oneline -10 + - name: Create Pull Request + env: + CHANGED_MANIFEST_FILES: ${{ steps.changed-manifest-files.outputs.all_changed_files }} + CHANGED_PACKAGES: ${{ needs.setup.outputs.changed_packages }} + TIMESTAMP: ${{ needs.setup.outputs.timestamp }} + i686_PACKAGES: ${{ needs.setup.outputs.i686_packages }} + x86_64_PACKAGES: ${{ needs.setup.outputs.x86_64_packages }} + armv7l_PACKAGES: ${{ needs.setup.outputs.armv7l_packages }} + run: | + rm -rf /tmp/pr.txt + echo -e "## Description" >> /tmp/pr.txt + echo -e "- This PR has package updates using the No-Compile-Needed.yml workflow, which was pointed to the ${{ env.BRANCH }} branch.\n" >> /tmp/pr.txt + echo -e "Updated packages:" >> /tmp/pr.txt + for file in ${CHANGED_PACKAGES} + do + echo "- ${file}" >> /tmp/pr.txt + done + echo -e "##\nUpdates attempted for:" >> /tmp/pr.txt + [ -n "${x86_64_PACKAGES}" ] && echo -e "- [x] \`x86_64\`" >> /tmp/pr.txt + [ -n "${i686_PACKAGES}" ] && echo -e "- [x] \`i686\`" >> /tmp/pr.txt + [ -n "${armv7l_PACKAGES}" ] && echo -e "- [x] \`armv7l\`" >> /tmp/pr.txt + if [[ "${CHANGED_MANIFEST_FILES}" == "" ]]; then + echo -e "##\n- [x] This PR has no manifest .filelist changes. _(Package changes have neither added nor removed files.)_" >> /tmp/pr.txt + fi + echo -e "##\n### Run the following to get this pull request's changes locally for testing.\n\`\`\`bash" >> /tmp/pr.txt + echo -e "CREW_REPO=https://github.com/chromebrew/chromebrew.git CREW_BRANCH=${{ env.BRANCH }} crew update \\" >> /tmp/pr.txt + echo -e "&& yes | crew upgrade\n\`\`\`" >> /tmp/pr.txt + cat /tmp/pr.txt + PR_NUMBER=$(gh pr create --title "AutoUpdate: ${{ env.BRANCH }} started at ${TIMESTAMP}" -F /tmp/pr.txt | rev | cut -d"/" -f1 | rev) + echo "PR_NUMBER is ${PR_NUMBER}" + echo "PR_NUMBER=${PR_NUMBER}" >> "$GITHUB_ENV" + - name: Trigger Unit Test Workflow & Add Reviewer Team + env: + GH_TOKEN: ${{ secrets.CREW_PR_TOKEN }} + run: | + echo "Flipping pr ${PR_NUMBER} state." + gh pr close "${PR_NUMBER}" && gh pr reopen "${PR_NUMBER}" + gh pr edit "${PR_NUMBER}" --add-reviewer chromebrew/active + gh pr merge "${PR_NUMBER}" --auto diff --git a/tools/github_actions_no_compile_updater.sh b/tools/github_actions_no_compile_updater.sh new file mode 100755 index 000000000..78cee48b2 --- /dev/null +++ b/tools/github_actions_no_compile_updater.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Mapping of /output should be set in the docker run options for the +# run-updater step of the generate job in .github/workflows/Updater.yml +cd /output/ || exit 1 +crew update +git config --global --add safe.directory /output +export CREW_AGREE_TIMEOUT_SECONDS=1 +set -x +tools/update_no_compile_packages.rb || exit 1 diff --git a/tools/update_no_compile_packages.rb b/tools/update_no_compile_packages.rb new file mode 100755 index 000000000..efc56753e --- /dev/null +++ b/tools/update_no_compile_packages.rb @@ -0,0 +1,58 @@ +#!/usr/bin/env ruby +# update_no_compile_packages version 1.0 (for Chromebrew) +# This checks if packages with 'no_compile_needed' require updates. +# +# Author: Ed Reel (uberhacker) edreel at gmail dot com +# Usage in root of cloned chromebrew repo with a new branch checked out: +# tools/update_no_compile_packages.rb + +require 'English' +require_relative '../lib/color' +require_relative '../lib/const' +require_relative '../lib/package' +$LOAD_PATH.unshift File.expand_path('../lib', __dir__) + +unless Dir.exist?('packages') + abort 'Unable to locate packages. Please run from the repository root directory.'.lightred +end + +puts "Setting the CREW_AGREE_TIMEOUT_SECONDS environment variable to less than the default of #{CREW_AGREE_TIMEOUT_SECONDS} may speed this up...".orange if ENV['CREW_AGREE_TIMEOUT_SECONDS'].nil? + +def self.agree_default_yes(message = nil) + Timeout.timeout(CREW_AGREE_TIMEOUT_SECONDS) do + return agree_with_default("#{message} (YES/no)?", true, default: 'y') + end +rescue Timeout::Error + return true +end + +changed_files = `git diff HEAD --name-only`.chomp.split +updated_packages = changed_files.select { |c| c.include?('packages/') } +changed_files_previous_commit = `git diff-tree --no-commit-id --name-only -r $(git rev-parse origin/master)..$(git rev-parse --verify HEAD)`.chomp.split +unless changed_files_previous_commit.empty? + updated_previous_packages = changed_files_previous_commit.select { |c| c.include?('packages/') } + updated_packages.push(*updated_previous_packages) +end + +abort 'No packages need to be updated.'.orange if updated_packages.empty? + +puts 'Changed packages will be checked to see if they need updated.'.orange +updated_packages.each do |pkg| + @pkg_obj = Package.load_package(pkg) + updated_packages.delete(pkg) unless @pkg_obj.no_compile_needed? +end + +abort 'No packages need to be updated.'.orange if updated_packages.empty? + +updated_packages.uniq! +updated_packages.each { |p| puts p.sub('packages/', '').sub('.rb', '').to_s.lightblue } + +updated_packages.each do |pkg| + name = pkg.sub('packages/', '').sub('.rb', '') + + puts "Evaluating #{name} package...".orange + @pkg_obj = Package.load_package(pkg) + + system("crew install -f #{name}") + abort "#{name} install failed!".lightred unless $CHILD_STATUS.success? +end