chromebrew/.github/workflows/Build.yml

---
# Version 1.4
name: Build
run-name: Build ${{ inputs.packages_to_build }} for ${{ inputs.build-on }} on branch ${{ inputs.branch || github.head_ref || github.ref_name }} requested by @${{ github.actor }}
on:
  workflow_dispatch:
    inputs:
      with_pr:
        description: "Create a PR after building."
        required: false
        type: choice
        default: 'Pull Request'
        options:
          - 'Pull Request'
          - 'Draft Pull Request'
          - 'No'
      allow_fail:
        description: "Allow packages to fail building."
        required: false
        type: boolean
        default: 'false'
      packages_to_build:
        description: "Specify packages to build."
        required: false
      rebuild_packages:
        description: "Rebuild packages even if they already have binaries."
        required: false
        type: boolean
        default: 'false'
      pr_title:
        description: "Title of PR"
        required: false
      branch:
        description: "Branch of chromebrew/chromebrew to run on, if different from this branch."
        required: false
      update_rebase_to_master:
        description: "Update Branch."
        required: false
        type: boolean
        default: 'true'
      pr_label:
        description: "PR Label (Not Title)"
        required: false
      max_build_time:
        description: "Maximum Build Time (hours)"
        required: false
        type: number
        default: 5.5
      build-on:
        description: "Architectures to build on"
        default: 'i686 x86_64 armv7l'
        type: string
        required: false
env:
  BRANCH: ${{ inputs.branch || github.head_ref || github.ref_name }}
permissions:
  actions: write
  contents: write
  packages: write
  pull-requests: write
  repository-projects: read
jobs:
  debug:
    if: ${{ ( github.repository_owner == 'chromebrew' ) }}
    runs-on: ubuntu-24.04
    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJson(job) }}
        run: echo "$JOB_CONTEXT"
      - name: Dump steps context
        env:
          STEPS_CONTEXT: ${{ toJson(steps) }}
        run: echo "$STEPS_CONTEXT"
  setup:
    if: ${{ ( github.repository_owner == 'chromebrew' ) && ( inputs.branch != 'master' ) }}
    runs-on: ubuntu-24.04
    outputs:
      timestamp: ${{ steps.set-timestamp.outputs.TIMESTAMP }}  # https://stackoverflow.com/a/75142892
      current_head: ${{ steps.get-current-head.outputs.CURRENT_HEAD }}
      changed_packages: ${{ steps.changed-packages.outputs.CHANGED_PACKAGES }}
      matrix: ${{ steps.set-generate-matrix.outputs.matrix }}
    steps:
      - name: Set Timestamp
        id: set-timestamp
        run: |
          TIMESTAMP="$(date -u +%F-%H%Z)"
          export TIMESTAMP
          echo "TIMESTAMP=${TIMESTAMP}" >> "$GITHUB_OUTPUT"
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          persist-credentials: true
          ref: ${{ inputs.branch || github.head_ref || github.ref_name }}
      - name: Set PR to Draft
        id: set-to-draft
        env:
          GH_TOKEN: ${{ secrets.CREW_PR_TOKEN }}
        run: |
          PR_NUMBER=$(gh pr list -L 1 -s open -H ${{ inputs.branch || github.head_ref || github.ref_name }} | cut -f1)
          if [[ -n ${PR_NUMBER} ]]; then
            echo "Setting ${PR_NUMBER} to Draft while workflow runs."
            gh pr ready --undo || true
            # Try to update branch before building if a PR already
            # exists.
            [[ "${{ ( inputs.update_rebase_to_master ) }}" == 'true' ]] && gh pr update-branch --rebase || true
          fi
      - name: Get Current HEAD hash
        id: get-current-head
        run: |
          echo "CURRENT_HEAD=$(git ls-remote https://github.com/${{ github.repository }}.git | head -1 | sed 's/HEAD//')" >> "$GITHUB_OUTPUT"
          echo "CURRENT_HEAD is $(git ls-remote https://github.com/${{ github.repository }}.git | head -1 | sed 's/HEAD//')"
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v47
        with:
          base_sha: ${{ steps.get-current-head.outputs.CURRENT_HEAD }}
          files_yaml: |
            packages:
              - packages/*.rb
          since_last_remote_commit: true
      - name: Export variables to github context
        id: changed-packages
        run: |
            # Add inputs.packages_to_build to the packages we will
            # check for architecture and glibc compatibility.
            if [[ -z "${{ steps.changed-files.outputs.packages_all_changed_files }} ${{ inputs.packages_to_build }}" ]]; then
              echo "Branch ${{ inputs.branch || github.head_ref || github.ref_name }} has no changed package files."
              exit 1
            fi
            # Convert "packages/foo.rb packages/bar.rb" (from steps.changed-files.outputs.packages_all_changed_files) into "foo bar"
            echo "CHANGED_PACKAGES=$(echo "${{ steps.changed-files.outputs.packages_all_changed_files }} ${{ inputs.packages_to_build }}" | xargs basename -s .rb | sort -u | xargs)" >> "$GITHUB_ENV"
            echo "CHANGED_PACKAGES=$(echo "${{ steps.changed-files.outputs.packages_all_changed_files }} ${{ inputs.packages_to_build }}" | xargs basename -s .rb | sort -u | xargs)" >> "$GITHUB_OUTPUT"

  get-compatibility:
    needs: setup
    uses: ./.github/workflows/Determine-Compatibility.yml
    with:
      changed_packages: ${{ needs.setup.outputs.changed_packages }}
      branch: ${{ inputs.branch || github.head_ref || github.ref_name }}

  gen-matrix:
    needs: get-compatibility
    runs-on: ubuntu-24.04
    outputs:
      matrix: ${{ steps.set-generate-matrix.outputs.matrix }}
    steps:
      - name: Generate Creation Matrix
        id: set-generate-matrix
        env:
          i686_PACKAGES: ${{ needs.get-compatibility.outputs.i686_PACKAGES }}
          x86_64_PACKAGES: ${{ needs.get-compatibility.outputs.x86_64_PACKAGES }}
          armv7l_PACKAGES: ${{ needs.get-compatibility.outputs.armv7l_PACKAGES }}
        run: |
          function join_by { local IFS="$1"; shift; echo "$*"; }
          [[ -n "${i686_PACKAGES}" ]] && export CONTAINER_ARCH+=( "\"i686\"" )
          # Always run on x86_64
          export CONTAINER_ARCH+=( "\"x86_64\"" )
          [[ -n "${armv7l_PACKAGES}" ]] && export CONTAINER_ARCH+=( "\"armv7l\"" )
          export ARCHES="$(join_by , "${CONTAINER_ARCH[@]}")"
          echo "matrix=[${ARCHES}]" >> $GITHUB_OUTPUT
          echo "matrix=[${ARCHES}]"

  generate:
    needs:
      - gen-matrix
      - get-compatibility
      - setup
    strategy:
      fail-fast: false
      matrix:
        arch: ${{ fromJSON(needs.gen-matrix.outputs.matrix) }}
        runner:
          - ubuntu-24.04
          - ubuntu-24.04-arm
        exclude:
          - arch: x86_64
            runner: ubuntu-24.04-arm
          - arch: i686
            runner: ubuntu-24.04-arm
          - arch: armv7l
            runner: ubuntu-24.04
    runs-on: ${{ matrix.runner }}
    env:
      CREW_BUILD_NO_PACKAGE_FILE_HASH_UPDATES: 1
      CREW_REPO: ${{ github.event.repository.clone_url }}
      CREW_BRANCH: ${{ inputs.branch || github.head_ref || github.ref_name }}
      TARGET_ARCH: ${{ matrix.arch }}
      TIMESTAMP: ${{ needs.setup.outputs.timestamp }}
      GLIBC_232_COMPATIBLE_PACKAGES: ${{ needs.get-compatibility.outputs.GLIBC_232_COMPATIBLE_PACKAGES }}
      GLIBC_237_COMPATIBLE_PACKAGES: ${{ needs.get-compatibility.outputs.GLIBC_237_COMPATIBLE_PACKAGES }}
      i686_PACKAGES: ${{ needs.get-compatibility.outputs.i686_PACKAGES }}
      x86_64_PACKAGES: ${{ needs.get-compatibility.outputs.x86_64_PACKAGES }}
      armv7l_PACKAGES: ${{ needs.get-compatibility.outputs.armv7l_PACKAGES }}
    if: ${{ !cancelled() }}
    concurrency:
      group: ${{ matrix.arch }}-${{ github.workflow }}-${{ inputs.branch || github.head_ref || github.ref_name }}
    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - name: Dump job context
        env:
          JOB_CONTEXT: ${{ toJson(job) }}
        run: echo "$JOB_CONTEXT"
      - name: Dump steps context
        env:
          STEPS_CONTEXT: ${{ toJson(steps) }}
        run: echo "$STEPS_CONTEXT"
      - name: Dump runner context
        env:
          RUNNER_CONTEXT: ${{ toJson(runner) }}
        run: echo "$RUNNER_CONTEXT"
      - name: Dump strategy context
        env:
          STRATEGY_CONTEXT: ${{ toJson(strategy) }}
        run: echo "$STRATEGY_CONTEXT"
      - name: Dump matrix context
        env:
          MATRIX_CONTEXT: ${{ toJson(matrix) }}
        run: echo "$MATRIX_CONTEXT"
      - name: Build Container cleanup
        run: |
            sudo rm -rf release
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          persist-credentials: true
      - name: Export target docker container to github context
        run: |
          case $TARGET_ARCH in
              x86_64)
                # Export the x86_64 container depending on whether this branch updates packages with appropriate minimum glibc.
                if [[ $GLIBC_232_COMPATIBLE_PACKAGES ]]; then
                    echo "CONTAINER=satmandu/crewbuild:nocturne-x86_64.m97" >> "$GITHUB_ENV"
                elif [[ $GLIBC_237_COMPATIBLE_PACKAGES ]]; then
                    echo "CONTAINER=satmandu/crewbuild:hatch-x86_64.m145" >> "$GITHUB_ENV"
                else
                    echo "CONTAINER=satmandu/crew-pre-glibc-standalone:nocturne-x86_64.m90" >> "$GITHUB_ENV"
                fi
                echo "PLATFORM=linux/amd64" >> "$GITHUB_ENV"
                echo "LIB_SUFFIX=64" >> "$GITHUB_ENV"
              ;;
              armv7l)
                # Export the armv7l container depending on whether this branch updates packages with appropriate minimum glibc.
                if [[ $GLIBC_232_COMPATIBLE_PACKAGES ]]; then
                    echo "CONTAINER=satmandu/crewbuild:fievel-armv7l.m97" >> "$GITHUB_ENV"
                elif [[ $GLIBC_237_COMPATIBLE_PACKAGES ]]; then
                    echo "CONTAINER=satmandu/crewbuild:strongbad-armv7l.m145" >> "$GITHUB_ENV"
                else
                    echo "CONTAINER=satmandu/crew-pre-glibc-standalone:fievel-armv7l.m91" >> "$GITHUB_ENV"
                fi
                echo "PLATFORM=linux/arm/v7" >> "$GITHUB_ENV"
                echo "LIB_SUFFIX=" >> "$GITHUB_ENV"
              ;;
              i686)
                # There is only one i686 container based upon M58 with glibc 2.23.
                echo "CONTAINER=satmandu/crew-pre-glibc-standalone:alex-i686.m58" >> "$GITHUB_ENV"
                echo "PLATFORM=linux/386" >> "$GITHUB_ENV"
                echo "LIB_SUFFIX=" >> "$GITHUB_ENV"
              ;;
            esac
      - name: Run Updater in container
        id: run-updater
        if: ${{ !cancelled() }}
        env:
          CREW_MAX_BUILD_TIME_INPUT: ${{ inputs.max_build_time }}
        run: |
            if [[ -n ${CREW_MAX_BUILD_TIME_INPUT} ]]; then
              # Convert CREW_MAX_BUILD_TIME_INPUT to seconds.
              CREW_MAX_BUILD_TIME="$(bc <<<"scale=0;$CREW_MAX_BUILD_TIME_INPUT*3600/1")"
              echo "Maximum build time is $(bc <<<"scale=0;$CREW_MAX_BUILD_TIME_INPUT*60/1") minutes."
            fi
            [[ -n ${CI} ]] && echo 'CI variable is set.'
            if [ "$PLATFORM" == 'linux/arm/v7'  ] && [ -z "${armv7l_PACKAGES}" ]; then
              # Exit the arm container if there are not armv7l compatible packages.
              echo "Skipping armv7l container builds &/or package file updates."
              exit 0
            elif [ "$PLATFORM" == 'linux/amd64' ] && [ -z "${x86_64_PACKAGES}" ]; then
              # Exit the x86_64 container if there are not x86_64 compatible packages.
              echo "Skipping x86_64 container builds &/or package file updates."
              exit 0
            elif [ "$PLATFORM" == 'linux/386' ] && [ -z "${i686_PACKAGES}" ]; then
              # Exit the i686 container if there are not i686 compatible packages.
              echo "Skipping i686 container builds &/or package file updates."
              exit 0
            fi
            case $PLATFORM in
              linux/386)    CREW_ARCH_PACKAGES_ENCODED="$(echo ${i686_PACKAGES} | xxd -ps -c 0 | tr -d '\n')";;
              linux/amd64)  CREW_ARCH_PACKAGES_ENCODED="$(echo ${x86_64_PACKAGES} | xxd -ps -c 0 | tr -d '\n')";;
              linux/arm/v7) CREW_ARCH_PACKAGES_ENCODED="$(echo ${armv7l_PACKAGES} | xxd -ps -c 0 | tr -d '\n')";;
              *)             echo "$PLATFORM not determined." && exit 1 ;;
            esac
            # Container Cleanup
            # Adapted from https://github.com/BRAINSia/free-disk-space/blob/main/action.yml
            echo "Running some background container cleanups from BRAINSia/free-disk-space."
            ( sudo apt-get purge -qq -o=Dpkg::Use-Pty=0 -y man-db manpages manpages-dev 'aspnetcore-*' 'dotnet-*' 'llvm-*' '*php*' 'mongodb-*' 'mysql-*' azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri 'google-cloud-*' 'gcloud-*' &>/dev/null ; \
            sudo apt-get autoremove -qq -o=Dpkg::Use-Pty=0 -y &>/dev/null) &

            git fetch origin
            git checkout "${{ inputs.branch || github.head_ref || github.ref_name }}"
            git reset --hard "origin/${{ inputs.branch || github.head_ref || github.ref_name }}"
            git log --oneline -10
            docker pull --platform "${PLATFORM}" "${CONTAINER}"
            # Detection of /output/pkg_cache dir triggers setting
            # CREW_CACHE_DIR=1 and CREW_CACHE_ENABLED=1 in the build
            # container. Without these, upload fails.
            mkdir /tmp/pkg_cache
            sudo setfacl -R -m u:1000:rwx .
            sudo setfacl -R -m u:1000:rwx /tmp/pkg_cache
            # See https://github.com/containerd/containerd/pull/7566#issuecomment-1461134737 for why we set ulimit.
            # Pass in packages we definitely might want to build.
            if [ -z ${CREW_ARCH_PACKAGES_ENCODED+x} ]; then
              echo "${CREW_ARCH_PACKAGES_ENCODED} is not set."
            else
              ARCH_PACKAGES_PASSTHROUGH=-e
              ARCH_PACKAGES_PASSTHROUGH+=" "
              ARCH_PACKAGES_PASSTHROUGH+="CREW_ARCH_PACKAGES_ENCODED=${CREW_ARCH_PACKAGES_ENCODED}"
            fi
            if [ -z ${CI+x} ]; then
              echo "CI is not set."
            else
              CI_PASSTHROUGH=-e
              CI_PASSTHROUGH+=" "
              CI_PASSTHROUGH+=NESTED_CI=${CI}
            fi
            if [ -z ${CREW_MAX_BUILD_TIME+x} ]; then
              echo "CREW_MAX_BUILD_TIME is not set."
            else
              CREW_MAX_BUILD_TIME_PASSTHROUGH=-e
              CREW_MAX_BUILD_TIME_PASSTHROUGH+=" "
              CREW_MAX_BUILD_TIME_PASSTHROUGH+=CREW_MAX_BUILD_TIME=${CREW_MAX_BUILD_TIME}
            fi
            docker run \
              --rm \
              --platform "${PLATFORM}" \
              -e PUID=1000 \
              -e PGID=1000 \
              --privileged \
              -e GCONV_PATH="/usr/local/lib${LIB_SUFFIX}/gconv" \
              -e CREW_BUILD_NO_PACKAGE_FILE_HASH_UPDATES="${CREW_BUILD_NO_PACKAGE_FILE_HASH_UPDATES}" \
              -e CREW_REPO="${CREW_REPO}" \
              -e CREW_BRANCH="${{ inputs.branch || github.head_ref || github.ref_name }}" \
              -e GITLAB_TOKEN="${{ secrets.GITLAB_TOKEN }}" \
              -e GITLAB_TOKEN_USERNAME="${{ secrets.GITLAB_TOKEN_USERNAME }}" \
              ${CI_PASSTHROUGH} \
              ${ARCH_PACKAGES_PASSTHROUGH} \
              ${CREW_MAX_BUILD_TIME_PASSTHROUGH} \
              -v /tmp/pkg_cache:/usr/local/tmp/packages:rshared \
              -v "$(pwd)":/output:rshared \
              --tmpfs /tmp \
              --ulimit "nofile=$(ulimit -Sn):$(ulimit -Hn)" \
              "${CONTAINER}" \
              /bin/chromebrewstart "/output/tools/github_actions_update_builder.sh ${{ github.event.inputs.allow_fail == 'true' && '--continue-after-failed-builds' || ''}} ${{ github.event.inputs.rebuild_packages == 'true' && '--rebuild' || ''}}"  > >(tee -a /tmp/build.log) 2> >(tee -a /tmp/build.log >&2)
              grep "Built and Uploaded:" /tmp/build.log || true
              echo "Deleting build output directories."
              sudo rm -rf release pkg_cache
  build-check:
    runs-on: ubuntu-24.04
    needs:
      - setup
      - generate
    if: ${{ !cancelled() }}
    env:
      CHANGED_PACKAGES: ${{ needs.setup.outputs.changed_packages }}
    steps:
      - name: Fail if update or build jobs failed, otherwise create a PR
        if: ${{ contains(needs.*.result, 'failure') }}
        run: exit 1
      - name: Report update & build success
        run: echo "Update & build jobs succeeded."
      - name: Get GH Token
        if: ${{ github.event.inputs.with_pr != 'No' }}
        id: get_workflow_token
        uses: peter-murray/workflow-application-token-action@v4
        with:
          application_id: ${{ secrets.APPLICATION_ID }}
          application_private_key: ${{ secrets.APPLICATION_PRIVATE_KEY }}
          organization: chromebrew
          revoke_token: true
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0
          persist-credentials: true
          ref: ${{ inputs.branch || github.head_ref || github.ref_name }}
      - uses: ruby/setup-ruby@v1
        with:
          ruby-version: '4.0.1'
      - name: Install bin/crew gem dependencies
        run: |
          # Install required gems.
          require_gem () {
            for g in "$@"
            do
              install_gem=0
              # Check to see if the gem is recorded as installed AND if there are
              # gem contents before assuming that the gem is installed.
              if gem list --no-update-sources -l -e "$g" 2>/dev/null | grep -q "$g"; then
                :
              else
                install_gem=1
              fi
              # shellcheck disable=SC2143
              if [[ $(gem contents "$g" 2>/dev/null | grep 'Unable to find gem') ]] || [[ "$(gem contents "$g" 2>/dev/null | wc -l)" == "0" ]]; then
                install_gem=1
              else
                :
              fi
             [[ $install_gem == '1' ]] && gem install -N "$g"
            done
          }
          require_gem regexp_parser dagwood ruby-libversion highline ptools cgi rubocop rubocop-chromebrew
          # Force creation of temporary device.json.
          ruby bin/crew version
      - name: Add package build completion files.
        id: add-build-success-marks
        run: |
          for file in $(echo "${CHANGED_PACKAGES}" | xargs basename -s .rb | xargs)
            do
            ruby bin/crew prop ignore_updater ${file}
            [[ "$(ruby bin/crew prop ignore_updater ${file})" == "false" ]] && touch tools/automatically_updatable_packages/${file} || true
          done
      - uses: EndBug/add-and-commit@v9
        if: ${{ contains(needs.generate.result, 'success') }}
        env:
          GH_TOKEN: ${{ steps.get_workflow_token.outputs.token }}
        with:
          add: 'tools/automatically_updatable_packages'
          fetch: false
          message: 'Mark packages from successful builds as automatically buildable.'
      - name: Trigger PR workflow
        if: ${{ github.event.inputs.with_pr != 'No' }}
        id: trigger-pr-workflow
        env:
          GH_TOKEN: ${{ steps.get_workflow_token.outputs.token }}
          PR_TITLE: ${{ inputs.pr_title || inputs.branch || github.ref_name }}
        run: |
            echo "PR title being passed to the Generate PR workflow is: ${{ inputs.pr_title || inputs.branch || github.ref_name }}"
            gh workflow run Generate-PR.yml -R ${{ github.repository }} -r ${{ inputs.branch || github.head_ref || github.ref_name }} -f branch="${{ inputs.branch || github.head_ref || github.ref_name }}" -f packages_to_check_for_build="${{ inputs.packages_to_build }}" -f pr_label="${{ inputs.pr_label }}" -f draft_pr="${{ github.event.inputs.with_pr == 'Draft Pull Request' }}" -f pr_title="$(echo "${{ inputs.pr_title || inputs.branch || github.ref_name }}" | ( read -rsd '' x; echo ${x@Q} ))"