concrete/.github/workflows/concrete_python_build_and_test.yml

name: concrete-python
on:
  workflow_call:
  workflow_dispatch:

concurrency:
  group: concrete-python-${{ github.ref }}
  cancel-in-progress: true

env:
  ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
  IS_PR: ${{ github.event_name == 'pull_request' }}
  IS_WEEKLY: ${{ github.event_name == 'schedule' }}
  IS_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
  IS_PUSH_TO_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
  AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
  RUNNER_TOOL_CACHE: /opt/hostedtoolcache

jobs:
  matrix-preparation:
    runs-on: ubuntu-22.04
    outputs:
      linux-matrix: ${{ steps.set-matrix.outputs.linux-matrix }}
      macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }}
      needs-37-linux-runner: ${{ steps.set-matrix.outputs.needs-37-linux-runner }}
      needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }}
      needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }}
      needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }}
    steps:
      - name: Checkout code
        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b

      - name: Set matrix
        id: set-matrix
        run: |
          cd frontends/concrete-python
          BUILD_TYPE=
          if [[ "${IS_PR}" == "true" ]]; then
            BUILD_TYPE="pr"
          elif [[ "${IS_WEEKLY}" == "true" ]]; then
            BUILD_TYPE="weekly"
          elif [[ "${IS_RELEASE}" == "true" ]]; then
            BUILD_TYPE="release"
          elif [[ "${IS_PUSH_TO_MAIN}" == "true" ]]; then
            BUILD_TYPE="push_to_main"
          else
            echo "Unknown BUILD_TYPE! Aborting"
            exit 1
          fi
          MATRIX_JSON=$(mktemp --suffix=.json)
          echo "Prepared build matrix:"
          python3 ./script/actions_utils/generate_test_matrix.py \
            --output-json "${MATRIX_JSON}" \
            --build-type "${BUILD_TYPE}"
          LINUX_MATRIX=$(jq -rc '. | map(select(.os_kind=="linux"))' "${MATRIX_JSON}")
          MACOS_MATRIX=$(jq -rc '. | map(select(.os_kind=="macos"))' "${MATRIX_JSON}")

          echo "Linux Matrix:"
          echo "${LINUX_MATRIX}" | jq '.'

          echo "macOS Matrix:"
          echo "${MACOS_MATRIX}" | jq '.'

          echo "::set-output name=linux-matrix::${LINUX_MATRIX}"
          echo "::set-output name=macos-matrix::${MACOS_MATRIX}"

          NEEDS_LINUX_37_RUNNER=$(echo "${LINUX_MATRIX}" | \
            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.7")) | length > 0')
          NEEDS_LINUX_38_RUNNER=$(echo "${LINUX_MATRIX}" | \
            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.8")) | length > 0')
          NEEDS_LINUX_39_RUNNER=$(echo "${LINUX_MATRIX}" | \
            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0')
          NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \
            jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0')

          echo "Needs Linux 3.7 runner:"
          echo "${NEEDS_LINUX_37_RUNNER}"

          echo "Needs Linux 3.8 runner:"
          echo "${NEEDS_LINUX_38_RUNNER}"

          echo "Needs Linux 3.9 runner:"
          echo "${NEEDS_LINUX_39_RUNNER}"

          echo "Needs Linux 3.10 runner:"
          echo "${NEEDS_LINUX_310_RUNNER}"

          echo "::set-output name=needs-37-linux-runner::${NEEDS_LINUX_37_RUNNER}"
          echo "::set-output name=needs-38-linux-runner::${NEEDS_LINUX_38_RUNNER}"
          echo "::set-output name=needs-39-linux-runner::${NEEDS_LINUX_39_RUNNER}"
          echo "::set-output name=needs-310-linux-runner::${NEEDS_LINUX_310_RUNNER}"

  start-runner-linux:
    needs: [matrix-preparation]
    name: Start EC2 runner
    runs-on: ubuntu-22.04
    outputs:
      label-37: ${{ steps.start-ec2-runner-37.outputs.label }}
      ec2-instance-id-37: ${{ steps.start-ec2-runner-37.outputs.ec2-instance-id || '' }}
      label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
      ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
      label-39: ${{ steps.start-ec2-runner-39.outputs.label }}
      ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }}
      label-310: ${{ steps.start-ec2-runner-310.outputs.label }}
      ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }}
      matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b

      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
        with:
          aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }}

      - name: Start EC2 runner python 37
        id: start-ec2-runner-37
        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-37-linux-runner) }}
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        with:
          mode: start
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
          ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
          subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}

      - name: Start EC2 runner python 38
        id: start-ec2-runner-38
        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-38-linux-runner) }}
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        with:
          mode: start
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
          ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
          subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}

      - name: Start EC2 runner python 39
        id: start-ec2-runner-39
        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-39-linux-runner) }}
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        with:
          mode: start
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
          ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
          subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}

      - name: Start EC2 runner python 310
        id: start-ec2-runner-310
        if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-310-linux-runner) }}
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        with:
          mode: start
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
          ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
          subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
          security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}

      - name: Update Linux runs_on Matrix
        id: update-linux-matrix
        env:
          MATRIX: ${{ needs.matrix-preparation.outputs.linux-matrix }}
        run: |
          MATRIX=$(echo "${MATRIX}" | jq -rc \
          '(. | map(select(.os_kind=="linux" and .python_version=="3.7") |= . + {"runs_on": "${{ steps.start-ec2-runner-37.outputs.label }}"}) )')
          MATRIX=$(echo "${MATRIX}" | jq -rc \
          '(. | map(select(.os_kind=="linux" and .python_version=="3.8") |= . + {"runs_on": "${{ steps.start-ec2-runner-38.outputs.label }}"}) )')
          MATRIX=$(echo "${MATRIX}" | jq -rc \
          '(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )')
          MATRIX=$(echo "${MATRIX}" | jq -rc \
          '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )')

          echo "Updated matrix:"
          echo "${MATRIX}"

          echo "::set-output name=linux-matrix::${MATRIX}"

  build-linux:
    needs: [start-runner-linux]

    runs-on: ${{ matrix.runs_on }}
    # Run in a clean container
    container:
      image: ubuntu:22.04
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.start-runner-linux.outputs.matrix)) }}
    env:
      IS_REF_BUILD: ${{ matrix.python_version == '3.8' }}

    steps:
      - name: Docker container related setup and git installation
        run: |
          TZ=Europe/Paris
          echo "TZ=${TZ}" >> "$GITHUB_ENV"
          ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
          sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
          apt update && apt install git git-lfs -y

      - name: Checkout Code
        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
        # Fetch-detph 0 to have all commits for changelog generation
        with:
          fetch-depth: 0

      - name: Set up Python ${{ matrix.python_version }}
        uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install dependencies
        id: install-deps
        run: |
          cd frontends/concrete-python
          ./script/make_utils/setup_os_deps.sh
          make setup_env

      - name: Source code Conformance
        id: cs
        if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
        # pcc launches an internal target with proper flags
        run: |
          cd frontends/concrete-python
          make pcc

      - name: Generate release changelog
        id: changelog
        if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }}
        run: |
          cd frontends/concrete-python
          GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
          CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md"
          echo "::set-output name=changelog-file::${CHANGELOG_FILE}"
          poetry run python ./script/make_utils/changelog_helper.py \
          --to-ref "${GIT_TAG}" \
          --to-ref-must-have-tag \
          --ancestor-must-have-tag > "${CHANGELOG_FILE}"

      - name: Conformance status
        id: conformance
        if: ${{ always() && !cancelled() }}
        env:
          CONFORMANCE_STATUS: ${{ steps.cs.outcome == 'success' }}
        run: |
          cd frontends/concrete-python
          if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then
            echo "Conformance failed, check logs"
            exit 1
          fi

      # Create packages before tests, to be able to get them if some unexpected test failure happens
      # Build the package only once, as we don't have binary dependency this can be used on Linux
      # and macOS as long as the dependencies are available
      - name: Build wheel
        id: build-wheel
        if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }}
        run: |
          cd frontends/concrete-python
          rm -rf dist
          poetry build -f wheel

      - name: Upload wheel artifact
        if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }}
        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
        with:
          name: py3-wheel
          path: frontends/concrete-python/dist/*.whl

      - name: PyTest Source Code
        id: pytest
        if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
        run: |
          cd frontends/concrete-python
          make pytest

      # Compute coverage only on reference build
      - name: Test coverage
        id: coverage
        if: ${{ always() && fromJSON(env.IS_REF_BUILD) && steps.pytest.outcome != 'skipped' && !cancelled() }}
        run: |
          cd frontends/concrete-python
          ./script/actions_utils/coverage.sh .global-coverage.json

      - name: Comment with coverage
        uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd
        if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
        continue-on-error: true
        with:
          path: frontends/concrete-python/diff-coverage.txt
          recreate: true

  stop-runner-linux:
    name: Stop EC2 runner
    needs: [build-linux, start-runner-linux]
    runs-on: ubuntu-22.04
    if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
    steps:
      - name: Configure AWS credentials
        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
        with:
          aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }}

      - name: Stop EC2 runner python 37
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-37 }}
        with:
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-37 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-37 }}
          mode: stop

      - name: Stop EC2 runner python 38
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
        with:
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-38 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
          mode: stop

      - name: Stop EC2 runner python 39
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-39 }}
        with:
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-39 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-39 }}
          mode: stop

      - name: Stop EC2 runner python 310
        uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
        if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-310 }}
        with:
          github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
          label: ${{ needs.start-runner-linux.outputs.label-310 }}
          ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }}
          mode: stop

  build-macos:
    needs: [matrix-preparation]
    if: ${{ needs.matrix-preparation.outputs.macos-matrix != '[]' }}

    runs-on: ${{ matrix.runs_on }}
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.matrix-preparation.outputs.macos-matrix)) }}
    steps:
      - name: Checkout Code
        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b

      - name: Set up Python ${{ matrix.python_version }}
        uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install dependencies
        id: install-deps
        run: |
          cd frontends/concrete-python
          ./script/make_utils/setup_os_deps.sh

          PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
          echo "PATH=${PATH}" >> "$GITHUB_ENV"

          make setup_env

      - name: PyTest Source Code
        run: |
          cd frontends/concrete-python
          make pytest