diff --git a/.github/workflows/concrete_python_build_and_test.yml b/.github/workflows/concrete_python_build_and_test.yml new file mode 100644 index 000000000..be84078c8 --- /dev/null +++ b/.github/workflows/concrete_python_build_and_test.yml @@ -0,0 +1,384 @@ +name: concrete-python +on: + workflow_call: + +concurrency: + group: concrete-python-${{ github.ref }} + cancel-in-progress: true + +env: + ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + IS_PR: ${{ github.event_name == 'pull_request' }} + IS_WEEKLY: ${{ github.event_name == 'schedule' }} + IS_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} + IS_PUSH_TO_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache + RUNNER_TOOL_CACHE: /opt/hostedtoolcache + +jobs: + matrix-preparation: + runs-on: ubuntu-22.04 + outputs: + linux-matrix: ${{ steps.set-matrix.outputs.linux-matrix }} + macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }} + needs-37-linux-runner: ${{ steps.set-matrix.outputs.needs-37-linux-runner }} + needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }} + needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }} + needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }} + steps: + - name: Checkout code + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b + + - name: Set matrix + id: set-matrix + run: | + cd frontends/concrete-python + BUILD_TYPE= + if [[ "${IS_PR}" == "true" ]]; then + BUILD_TYPE="pr" + elif [[ "${IS_WEEKLY}" == "true" ]]; then + BUILD_TYPE="weekly" + elif [[ "${IS_RELEASE}" == "true" ]]; then + BUILD_TYPE="release" + elif [[ "${IS_PUSH_TO_MAIN}" == "true" ]]; then + BUILD_TYPE="push_to_main" + else + echo "Unknown BUILD_TYPE! Aborting" + exit 1 + fi + MATRIX_JSON=$(mktemp --suffix=.json) + echo "Prepared build matrix:" + python3 ./script/actions_utils/generate_test_matrix.py \ + --output-json "${MATRIX_JSON}" \ + --build-type "${BUILD_TYPE}" + LINUX_MATRIX=$(jq -rc '. | map(select(.os_kind=="linux"))' "${MATRIX_JSON}") + MACOS_MATRIX=$(jq -rc '. | map(select(.os_kind=="macos"))' "${MATRIX_JSON}") + + echo "Linux Matrix:" + echo "${LINUX_MATRIX}" | jq '.' + + echo "macOS Matrix:" + echo "${MACOS_MATRIX}" | jq '.' + + echo "::set-output name=linux-matrix::${LINUX_MATRIX}" + echo "::set-output name=macos-matrix::${MACOS_MATRIX}" + + NEEDS_LINUX_37_RUNNER=$(echo "${LINUX_MATRIX}" | \ + jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.7")) | length > 0') + NEEDS_LINUX_38_RUNNER=$(echo "${LINUX_MATRIX}" | \ + jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.8")) | length > 0') + NEEDS_LINUX_39_RUNNER=$(echo "${LINUX_MATRIX}" | \ + jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0') + NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \ + jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0') + + echo "Needs Linux 3.7 runner:" + echo "${NEEDS_LINUX_37_RUNNER}" + + echo "Needs Linux 3.8 runner:" + echo "${NEEDS_LINUX_38_RUNNER}" + + echo "Needs Linux 3.9 runner:" + echo "${NEEDS_LINUX_39_RUNNER}" + + echo "Needs Linux 3.10 runner:" + echo "${NEEDS_LINUX_310_RUNNER}" + + echo "::set-output name=needs-37-linux-runner::${NEEDS_LINUX_37_RUNNER}" + echo "::set-output name=needs-38-linux-runner::${NEEDS_LINUX_38_RUNNER}" + echo "::set-output name=needs-39-linux-runner::${NEEDS_LINUX_39_RUNNER}" + echo "::set-output name=needs-310-linux-runner::${NEEDS_LINUX_310_RUNNER}" + + start-runner-linux: + needs: [matrix-preparation] + name: Start EC2 runner + runs-on: ubuntu-22.04 + outputs: + label-37: ${{ steps.start-ec2-runner-37.outputs.label }} + ec2-instance-id-37: ${{ steps.start-ec2-runner-37.outputs.ec2-instance-id || '' }} + label-38: ${{ steps.start-ec2-runner-38.outputs.label }} + ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }} + label-39: ${{ steps.start-ec2-runner-39.outputs.label }} + ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }} + label-310: ${{ steps.start-ec2-runner-310.outputs.label }} + ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }} + matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }} + steps: + - name: Checkout Code + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 + with: + aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }} + + - name: Start EC2 runner python 37 + id: start-ec2-runner-37 + if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-37-linux-runner) }} + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + with: + mode: start + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }} + ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }} + subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }} + security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }} + + - name: Start EC2 runner python 38 + id: start-ec2-runner-38 + if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-38-linux-runner) }} + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + with: + mode: start + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }} + ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }} + subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }} + security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }} + + - name: Start EC2 runner python 39 + id: start-ec2-runner-39 + if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-39-linux-runner) }} + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + with: + mode: start + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }} + ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }} + subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }} + security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }} + + - name: Start EC2 runner python 310 + id: start-ec2-runner-310 + if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-310-linux-runner) }} + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + with: + mode: start + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }} + ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }} + subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }} + security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }} + + - name: Update Linux runs_on Matrix + id: update-linux-matrix + env: + MATRIX: ${{ needs.matrix-preparation.outputs.linux-matrix }} + run: | + MATRIX=$(echo "${MATRIX}" | jq -rc \ + '(. | map(select(.os_kind=="linux" and .python_version=="3.7") |= . + {"runs_on": "${{ steps.start-ec2-runner-37.outputs.label }}"}) )') + MATRIX=$(echo "${MATRIX}" | jq -rc \ + '(. | map(select(.os_kind=="linux" and .python_version=="3.8") |= . + {"runs_on": "${{ steps.start-ec2-runner-38.outputs.label }}"}) )') + MATRIX=$(echo "${MATRIX}" | jq -rc \ + '(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )') + MATRIX=$(echo "${MATRIX}" | jq -rc \ + '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )') + + echo "Updated matrix:" + echo "${MATRIX}" + + echo "::set-output name=linux-matrix::${MATRIX}" + + build-linux: + needs: [start-runner-linux] + + runs-on: ${{ matrix.runs_on }} + # Run in a clean container + container: + image: ubuntu:22.04 + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.start-runner-linux.outputs.matrix)) }} + env: + IS_REF_BUILD: ${{ matrix.python_version == '3.8' }} + + steps: + - name: Docker container related setup and git installation + run: | + TZ=Europe/Paris + echo "TZ=${TZ}" >> "$GITHUB_ENV" + ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone + sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list + apt update && apt install git git-lfs -y + + - name: Checkout Code + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b + # Fetch-detph 0 to have all commits for changelog generation + with: + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912 + with: + python-version: ${{ matrix.python_version }} + + - name: Install dependencies + id: install-deps + run: | + cd frontends/concrete-python + ./script/make_utils/setup_os_deps.sh + make setup_env + + - name: Source code Conformance + id: cs + if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }} + # pcc launches an internal target with proper flags + run: | + cd frontends/concrete-python + make pcc + + - name: Generate release changelog + id: changelog + if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }} + run: | + cd frontends/concrete-python + GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g') + CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md" + echo "::set-output name=changelog-file::${CHANGELOG_FILE}" + poetry run python ./script/make_utils/changelog_helper.py \ + --to-ref "${GIT_TAG}" \ + --to-ref-must-have-tag \ + --ancestor-must-have-tag > "${CHANGELOG_FILE}" + + - name: Conformance status + id: conformance + if: ${{ always() && !cancelled() }} + env: + CONFORMANCE_STATUS: ${{ steps.cs.outcome == 'success' }} + run: | + cd frontends/concrete-python + if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then + echo "Conformance failed, check logs" + exit 1 + fi + + # Create packages before tests, to be able to get them if some unexpected test failure happens + # Build the package only once, as we don't have binary dependency this can be used on Linux + # and macOS as long as the dependencies are available + - name: Build wheel + id: build-wheel + if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }} + run: | + cd frontends/concrete-python + rm -rf dist + poetry build -f wheel + + - name: Upload wheel artifact + if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }} + uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb + with: + name: py3-wheel + path: frontends/concrete-python/dist/*.whl + + - name: PyTest Source Code + id: pytest + if: ${{ steps.conformance.outcome == 'success' && !cancelled() }} + run: | + cd frontends/concrete-python + make pytest + + # Compute coverage only on reference build + - name: Test coverage + id: coverage + if: ${{ always() && fromJSON(env.IS_REF_BUILD) && steps.pytest.outcome != 'skipped' && !cancelled() }} + run: | + cd frontends/concrete-python + ./script/actions_utils/coverage.sh .global-coverage.json + + - name: Comment with coverage + uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd + if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }} + continue-on-error: true + with: + path: frontends/concrete-python/diff-coverage.txt + recreate: true + + stop-runner-linux: + name: Stop EC2 runner + needs: [build-linux, start-runner-linux] + runs-on: ubuntu-22.04 + if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }} + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 + with: + aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }} + + - name: Stop EC2 runner python 37 + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-37 }} + with: + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + label: ${{ needs.start-runner-linux.outputs.label-37 }} + ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-37 }} + mode: stop + + - name: Stop EC2 runner python 38 + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }} + with: + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + label: ${{ needs.start-runner-linux.outputs.label-38 }} + ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }} + mode: stop + + - name: Stop EC2 runner python 39 + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-39 }} + with: + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + label: ${{ needs.start-runner-linux.outputs.label-39 }} + ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-39 }} + mode: stop + + - name: Stop EC2 runner python 310 + uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea + if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-310 }} + with: + github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }} + label: ${{ needs.start-runner-linux.outputs.label-310 }} + ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }} + mode: stop + + build-macos: + needs: [matrix-preparation] + if: ${{ needs.matrix-preparation.outputs.macos-matrix != '[]' }} + + runs-on: ${{ matrix.runs_on }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.matrix-preparation.outputs.macos-matrix)) }} + steps: + - name: Checkout Code + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b + + - name: Set up Python ${{ matrix.python_version }} + uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912 + with: + python-version: ${{ matrix.python_version }} + + - name: Install dependencies + id: install-deps + run: | + cd frontends/concrete-python + ./script/make_utils/setup_os_deps.sh + + PATH="/usr/local/opt/make/libexec/gnubin:$PATH" + echo "PATH=${PATH}" >> "$GITHUB_ENV" + + make setup_env + + - name: PyTest Source Code + run: | + cd frontends/concrete-python + make pytest \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml deleted file mode 100644 index d11593b52..000000000 --- a/.github/workflows/continuous-integration.yaml +++ /dev/null @@ -1,749 +0,0 @@ -name: concrete-numpy CI Pipeline -on: - pull_request: - push: - branches: - - main - tags: - - "v*" - - schedule: - # * is a special character in YAML so you have to quote this string - # At 22:00 on Sunday - # Timezone is UTC, so Paris time is +2 during the summer and +1 during winter - - cron: '0 22 * * 0' - -concurrency: - group: ${{ github.ref }} - cancel-in-progress: true - -env: - ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - IS_PR: ${{ github.event_name == 'pull_request' }} - IS_WEEKLY: ${{ github.event_name == 'schedule' }} - IS_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - IS_PUSH_TO_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - RUNNER_TOOL_CACHE: /opt/hostedtoolcache - -jobs: - matrix-preparation: - runs-on: ubuntu-22.04 - outputs: - linux-matrix: ${{ steps.set-matrix.outputs.linux-matrix }} - macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }} - needs-37-linux-runner: ${{ steps.set-matrix.outputs.needs-37-linux-runner }} - needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }} - needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }} - needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }} - steps: - - name: Checkout code - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Set matrix - id: set-matrix - run: | - BUILD_TYPE= - if [[ "${IS_PR}" == "true" ]]; then - BUILD_TYPE="pr" - elif [[ "${IS_WEEKLY}" == "true" ]]; then - BUILD_TYPE="weekly" - elif [[ "${IS_RELEASE}" == "true" ]]; then - BUILD_TYPE="release" - elif [[ "${IS_PUSH_TO_MAIN}" == "true" ]]; then - BUILD_TYPE="push_to_main" - else - echo "Unknown BUILD_TYPE! Aborting" - exit 1 - fi - MATRIX_JSON=$(mktemp --suffix=.json) - echo "Prepared build matrix:" - python3 ./script/actions_utils/generate_test_matrix.py \ - --output-json "${MATRIX_JSON}" \ - --build-type "${BUILD_TYPE}" - LINUX_MATRIX=$(jq -rc '. | map(select(.os_kind=="linux"))' "${MATRIX_JSON}") - MACOS_MATRIX=$(jq -rc '. | map(select(.os_kind=="macos"))' "${MATRIX_JSON}") - - echo "Linux Matrix:" - echo "${LINUX_MATRIX}" | jq '.' - - echo "macOS Matrix:" - echo "${MACOS_MATRIX}" | jq '.' - - echo "::set-output name=linux-matrix::${LINUX_MATRIX}" - echo "::set-output name=macos-matrix::${MACOS_MATRIX}" - - NEEDS_LINUX_37_RUNNER=$(echo "${LINUX_MATRIX}" | \ - jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.7")) | length > 0') - NEEDS_LINUX_38_RUNNER=$(echo "${LINUX_MATRIX}" | \ - jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.8")) | length > 0') - NEEDS_LINUX_39_RUNNER=$(echo "${LINUX_MATRIX}" | \ - jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0') - NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \ - jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0') - - echo "Needs Linux 3.7 runner:" - echo "${NEEDS_LINUX_37_RUNNER}" - - echo "Needs Linux 3.8 runner:" - echo "${NEEDS_LINUX_38_RUNNER}" - - echo "Needs Linux 3.9 runner:" - echo "${NEEDS_LINUX_39_RUNNER}" - - echo "Needs Linux 3.10 runner:" - echo "${NEEDS_LINUX_310_RUNNER}" - - echo "::set-output name=needs-37-linux-runner::${NEEDS_LINUX_37_RUNNER}" - echo "::set-output name=needs-38-linux-runner::${NEEDS_LINUX_38_RUNNER}" - echo "::set-output name=needs-39-linux-runner::${NEEDS_LINUX_39_RUNNER}" - echo "::set-output name=needs-310-linux-runner::${NEEDS_LINUX_310_RUNNER}" - - start-runner-linux: - needs: [matrix-preparation] - name: Start EC2 runner - runs-on: ubuntu-22.04 - outputs: - label-37: ${{ steps.start-ec2-runner-37.outputs.label }} - ec2-instance-id-37: ${{ steps.start-ec2-runner-37.outputs.ec2-instance-id || '' }} - label-38: ${{ steps.start-ec2-runner-38.outputs.label }} - ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }} - label-39: ${{ steps.start-ec2-runner-39.outputs.label }} - ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }} - label-310: ${{ steps.start-ec2-runner-310.outputs.label }} - ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }} - matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }} - steps: - - name: Checkout Code - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Start EC2 runner python 37 - id: start-ec2-runner-37 - if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-37-linux-runner) }} - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - ec2-image-id: ${{ secrets.AWS_EC2_AMI }} - ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }} - subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} - security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} - - - name: Start EC2 runner python 38 - id: start-ec2-runner-38 - if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-38-linux-runner) }} - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - ec2-image-id: ${{ secrets.AWS_EC2_AMI }} - ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }} - subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} - security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} - - - name: Start EC2 runner python 39 - id: start-ec2-runner-39 - if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-39-linux-runner) }} - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - ec2-image-id: ${{ secrets.AWS_EC2_AMI }} - ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }} - subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} - security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} - - - name: Start EC2 runner python 310 - id: start-ec2-runner-310 - if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-310-linux-runner) }} - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - with: - mode: start - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - ec2-image-id: ${{ secrets.AWS_EC2_AMI }} - ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }} - subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }} - security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }} - - - name: Update Linux runs_on Matrix - id: update-linux-matrix - env: - MATRIX: ${{ needs.matrix-preparation.outputs.linux-matrix }} - run: | - MATRIX=$(echo "${MATRIX}" | jq -rc \ - '(. | map(select(.os_kind=="linux" and .python_version=="3.7") |= . + {"runs_on": "${{ steps.start-ec2-runner-37.outputs.label }}"}) )') - MATRIX=$(echo "${MATRIX}" | jq -rc \ - '(. | map(select(.os_kind=="linux" and .python_version=="3.8") |= . + {"runs_on": "${{ steps.start-ec2-runner-38.outputs.label }}"}) )') - MATRIX=$(echo "${MATRIX}" | jq -rc \ - '(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )') - MATRIX=$(echo "${MATRIX}" | jq -rc \ - '(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )') - - echo "Updated matrix:" - echo "${MATRIX}" - - echo "::set-output name=linux-matrix::${MATRIX}" - - build-linux: - needs: [start-runner-linux] - - runs-on: ${{ matrix.runs_on }} - # Run in a clean container - container: - image: ubuntu:22.04 - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.start-runner-linux.outputs.matrix)) }} - env: - IS_REF_BUILD: ${{ matrix.python_version == '3.8' }} - - steps: - - name: Docker container related setup and git installation - run: | - TZ=Europe/Paris - echo "TZ=${TZ}" >> "$GITHUB_ENV" - ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone - sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list - apt update && apt install git git-lfs -y - - - name: Checkout Code - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - # Fetch-detph 0 to have all commits for changelog generation - with: - fetch-depth: 0 - - - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912 - with: - python-version: ${{ matrix.python_version }} - - - name: Install dependencies - id: install-deps - run: | - ./script/make_utils/setup_os_deps.sh - make setup_env - - - name: Check commits first line format - id: ccfl - if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }} - uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee - with: - pattern: '^((feat|fix|chore|refactor|style|test|docs)(\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\))?\:) .+$' - flags: 'gs' - error: "Your first line has to contain a commit type and scope like \"feat(my_feature): msg\".\ - Pattern: '^((feat|fix|chore|refactor|style|test|docs)(\\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\\))?\\:)'" - excludeDescription: 'true' # optional: this excludes the description body of a pull request - excludeTitle: 'true' # optional: this excludes the title of a pull request - checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request - accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true - - - name: Check commits line length - id: ccll - if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }} - uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee - with: - pattern: '(^.{0,74}$\r?\n?){0,20}' - flags: 'gm' - error: 'The maximum line length of 74 characters is exceeded.' - excludeDescription: 'true' # optional: this excludes the description body of a pull request - excludeTitle: 'true' # optional: this excludes the title of a pull request - checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request - accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true - - - name: Commit conformance - id: commit-conformance - if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }} - env: - CCFL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccfl.outcome == 'success') || steps.ccfl.outcome == 'skipped' }} - CCLL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccll.outcome == 'success') || steps.ccll.outcome == 'skipped' }} - run: | - if [[ "${CCFL_OK}" != "true" || "${CCLL_OK}" != "true" ]]; then - echo "Issues with commits. First line ok: ${CCFL_OK}. Line length ok: ${CCLL_OK}." - exit 1 - fi - - - name: Source code Conformance - id: cs - if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }} - # pcc launches an internal target with proper flags - run: | - make pcc - - - name: Generate release changelog - id: changelog - if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }} - run: | - GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g') - CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md" - echo "::set-output name=changelog-file::${CHANGELOG_FILE}" - poetry run python ./script/make_utils/changelog_helper.py \ - --to-ref "${GIT_TAG}" \ - --to-ref-must-have-tag \ - --ancestor-must-have-tag > "${CHANGELOG_FILE}" - - - name: Conformance status - id: conformance - if: ${{ always() && !cancelled() }} - env: - CONFORMANCE_STATUS: ${{ steps.commit-conformance.outcome == 'success' && steps.cs.outcome == 'success' }} - run: | - if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then - echo "Conformance failed, check logs" - exit 1 - fi - - - name: Upload changelog artifacts - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.changelog.outcome == 'success' && !cancelled() }} - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb - with: - name: changelog - path: ${{ steps.changelog.outputs.changelog-file }} - - # Create packages before tests, to be able to get them if some unexpected test failure happens - # Build the package only once, as we don't have binary dependency this can be used on Linux - # and macOS as long as the dependencies are available - - name: Build wheel - id: build-wheel - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }} - run: | - rm -rf dist - poetry build -f wheel - - - name: Upload wheel artifact - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }} - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb - with: - name: py3-wheel - path: dist/*.whl - - - name: PyTest Source Code - id: pytest - if: ${{ steps.conformance.outcome == 'success' && !cancelled() }} - run: | - make pytest - -# - name: PyTest CodeBlocks -# if: ${{ steps.conformance.outcome == 'success' && !cancelled() }} -# run: | -# make pytest_codeblocks -# -# - name: PyTest Notebooks -# if: ${{ fromJSON(env.IS_WEEKLY) && steps.conformance.outcome == 'success' && !cancelled() }} -# run: | -# make pytest_nb - - # Compute coverage only on reference build - - name: Test coverage - id: coverage - if: ${{ always() && fromJSON(env.IS_REF_BUILD) && steps.pytest.outcome != 'skipped' && !cancelled() }} - run: | - ./script/actions_utils/coverage.sh .global-coverage.json - - - name: Comment with coverage - uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd - if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }} - continue-on-error: true - with: - path: diff-coverage.txt - recreate: true - - # This is to manage build matrices and have a single status point for PRs - # This can be updated to take macOS into account but is impractical for private repos because of - # long builds and therefore expensive macOS testing - linux-build-status: - name: Linux build status - needs: [build-linux] - runs-on: ubuntu-22.04 - if: ${{ always() }} - steps: - - name: Fail on unsuccessful Linux build - shell: bash - run: | - if [[ ${{ needs.build-linux.result }} != "success" ]]; then - exit 1 - fi - - - name: Slack Notification - if: ${{ always() && !success() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ needs.build-linux.result }} - SLACK_MESSAGE: "Build finished with status ${{ needs.build-linux.result }}. (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - stop-runner-linux: - name: Stop EC2 runner - needs: [build-linux, start-runner-linux] - runs-on: ubuntu-22.04 - if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }} - steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Stop EC2 runner python 37 - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-37 }} - with: - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - label: ${{ needs.start-runner-linux.outputs.label-37 }} - ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-37 }} - mode: stop - - - name: Stop EC2 runner python 38 - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }} - with: - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - label: ${{ needs.start-runner-linux.outputs.label-38 }} - ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }} - mode: stop - - - name: Stop EC2 runner python 39 - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-39 }} - with: - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - label: ${{ needs.start-runner-linux.outputs.label-39 }} - ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-39 }} - mode: stop - - - name: Stop EC2 runner python 310 - uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea - if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-310 }} - with: - github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }} - label: ${{ needs.start-runner-linux.outputs.label-310 }} - ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }} - mode: stop - - build-macos: - needs: [matrix-preparation] - if: ${{ needs.matrix-preparation.outputs.macos-matrix != '[]' }} - - runs-on: ${{ matrix.runs_on }} - defaults: - run: - shell: bash - strategy: - fail-fast: false - matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.matrix-preparation.outputs.macos-matrix)) }} - steps: - - name: Checkout Code - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Set up Python ${{ matrix.python_version }} - uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912 - with: - python-version: ${{ matrix.python_version }} - - - name: Install dependencies - id: install-deps - run: | - ./script/make_utils/setup_os_deps.sh - - PATH="/usr/local/opt/make/libexec/gnubin:$PATH" - echo "PATH=${PATH}" >> "$GITHUB_ENV" - - make setup_env - - - name: PyTest Source Code - run: | - make pytest - - weekly-pip-audit: - if: ${{ github.event_name == 'schedule' }} - runs-on: ubuntu-22.04 - steps: - - name: Checkout Code - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Set up Python 3.8 - uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912 - with: - python-version: '3.8' - - - name: Set up env - run: | - python -m pip install --upgrade pip - python -m pip install poetry - sudo apt update && sudo apt install sqlite3 -y - make setup_env - - - name: Run pip-audit - shell: bash - run: | - VULN_OUT="$(mktemp --suffix=.json)" - REPORT_OUT="$(mktemp --suffix=.txt)" - echo "REPORT_OUT=${REPORT_OUT}" >> "$GITHUB_ENV" - poetry run pip-audit -f json > "${VULN_OUT}" - cat "${VULN_OUT}" - poetry run python ./script/actions_utils/parse_pip_audit_vulns.py \ - --vulns-json "${VULN_OUT}" \ - --vulns-report "${REPORT_OUT}" - - # We load the report in a new step if we exited with an error code above to let the workflow fail - - name: Load report in env - if: ${{ always() }} - run: | - cat "${REPORT_OUT}" - REPORT="$(cat "${REPORT_OUT}")" - echo "REPORT=${REPORT}" >> "$GITHUB_ENV" - - - name: Slack Notification - if: ${{ always() && !success() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "${{ env.REPORT || 'Error during pip-audit' }} (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - package-release: - needs: [build-linux] - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} - - outputs: - report: ${{ steps.report.outputs.report || 'Did not run.' }} - - name: Package and artifacts release - runs-on: ubuntu-22.04 - - steps: - - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Install dependencies - run: | - sudo apt-get install --no-install-recommends -y gnome-keyring sqlite3 - python -m pip install --upgrade pip - python -m pip install poetry - make setup_env - - - name: Set common environment variables - run: | - PROJECT_NAME_AND_VERSION=$(poetry version) - PROJECT_VERSION=$(echo "$PROJECT_NAME_AND_VERSION" | cut -d ' ' -f 2) - - GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g') - if [[ "${GIT_TAG}" != "v${PROJECT_VERSION}" ]]; then - echo "Mismatch between tag and version: ${GIT_TAG}, v${PROJECT_VERSION}" - exit 1 - fi - - IMAGE_BASE="${{ secrets.IMAGE_BASE }}" - ALL_IMAGE_TAGS="${IMAGE_BASE}:${GIT_TAG}" - - IS_LATEST=$(poetry run python script/make_utils/is_latest.py "${PROJECT_VERSION}") - if [[ "${IS_LATEST}" == "True" ]]; then - ALL_IMAGE_TAGS="${ALL_IMAGE_TAGS},${IMAGE_BASE}:latest" - fi - - IS_PRERELEASE=$(poetry run python script/make_utils/is_prerelease.py "${PROJECT_VERSION}") - - echo "PROJECT_VERSION=${PROJECT_VERSION}" >> "$GITHUB_ENV" - echo "GIT_TAG=${GIT_TAG}" >> "$GITHUB_ENV" - - echo "IS_LATEST=${IS_LATEST}" >> "$GITHUB_ENV" - echo "IS_PRERELEASE=${IS_PRERELEASE}" >> "$GITHUB_ENV" - - echo "ALL_IMAGE_TAGS=${ALL_IMAGE_TAGS}" >> "$GITHUB_ENV" - echo "VERSIONED_IMAGE_TAG=${IMAGE_BASE}:${GIT_TAG}" >> "$GITHUB_ENV" - - - name: Create directory for artifacts - if: ${{ success() && !cancelled() }} - run: | - ARTIFACTS_RAW_DIR=/tmp/release_artifacts/raw - mkdir -p "${ARTIFACTS_RAW_DIR}" - echo "ARTIFACTS_RAW_DIR=${ARTIFACTS_RAW_DIR}" >> "$GITHUB_ENV" - - ARTIFACTS_PACKAGED_DIR=/tmp/release_artifacts/packaged - mkdir -p "${ARTIFACTS_PACKAGED_DIR}" - echo "ARTIFACTS_PACKAGED_DIR=${ARTIFACTS_PACKAGED_DIR}" >> "$GITHUB_ENV" - - - name: Download changelog - if: ${{ success() && !cancelled() }} - id: download-changelog - uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 - with: - name: changelog - path: ${{ env.ARTIFACTS_RAW_DIR }}/changelog/ - - - name: Download python3 wheel - if: ${{ success() && !cancelled() }} - id: download-wheel - uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7 - with: - name: py3-wheel - path: ${{ env.ARTIFACTS_PACKAGED_DIR }}/ - - - name: Copy wheel to docker build context - if: ${{ success() && !cancelled() }} - run: | - mkdir -p ./pkg - cp "${{ env.ARTIFACTS_PACKAGED_DIR }}"/*.whl ./pkg - - - name: Login to GitHub Container Registry - if: ${{ success() && !cancelled() }} - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - with: - registry: ghcr.io - username: ${{ secrets.BOT_USERNAME }} - password: ${{ secrets.BOT_TOKEN }} - - - name: Login to DockerHub - if: ${{ success() && !cancelled() }} - uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build concrete-numpy Image - if: ${{ success() && !cancelled() }} - uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671 - with: - context: . - file: docker/Dockerfile.release - load: true - push: false - tags: "${{ env.ALL_IMAGE_TAGS }}" - no-cache: true - - - name: Release image sanity check - if: ${{ success() && !cancelled() }} - run: | - echo "Running sanity check for ${VERSIONED_IMAGE_TAG}" - docker run --rm -v "$(pwd)"/docker/release_resources:/data \ - "${VERSIONED_IMAGE_TAG}" /bin/bash -c "python ./sanity_check.py" - - - name: Create ready to upload/packaged artifacts and release body - if: ${{ success() && !cancelled() }} - env: - RAW_CHANGELOG_DIR: ${{ steps.download-changelog.outputs.download-path }} - run: | - cp "${RAW_CHANGELOG_DIR}"/* "${ARTIFACTS_PACKAGED_DIR}" - ls -a "${ARTIFACTS_PACKAGED_DIR}" - - RELEASE_BODY_FILE=RELEASE_BODY.md - echo "RELEASE_BODY_FILE=${RELEASE_BODY_FILE}" >> "$GITHUB_ENV" - - cp ./script/actions_utils/RELEASE_TEMPLATE.md "${RELEASE_BODY_FILE}" - { - echo "Docker Image: ${VERSIONED_IMAGE_TAG}"; - echo "PyPI Package: https://pypi.org/project/concrete-numpy/${PROJECT_VERSION}"; - echo ""; - } >> "${RELEASE_BODY_FILE}" - cat "${RAW_CHANGELOG_DIR}"/* >> "${RELEASE_BODY_FILE}" - - - name: Push release docker image - if: ${{ success() && !cancelled() }} - run: | - docker image push --all-tags "${{ secrets.IMAGE_BASE }}" - - - name: Push package to PyPi - if: ${{ success() && !cancelled() }} - run: | - poetry run twine upload \ - -u "${{ secrets.REPO_USERNAME }}" -p ${{ secrets.REPO_PASSWORD }} \ - ${{ secrets.REPO_DETAILS }} "${{ env.ARTIFACTS_PACKAGED_DIR }}"/*.whl - - - name: Create GitHub release - if: ${{ success() && !cancelled() }} - id: create-release - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 - with: - body_path: ${{ env.RELEASE_BODY_FILE }} - prerelease: ${{ fromJSON(env.IS_PRERELEASE) }} - files: | - ${{ env.ARTIFACTS_PACKAGED_DIR }}/* - tag_name: ${{ env.GIT_TAG }} - fail_on_unmatched_files: true - token: ${{ secrets.BOT_TOKEN }} - - - name: Set notification report - id: report - if: ${{ always() }} - run: | - REPORT="Creating release for ${GIT_TAG} finished with status ${{ job.status }}. \ - GitHub release link: ${{ steps.create-release.outputs.url }}." - echo "${REPORT}" - echo "::set-output name=report::${REPORT}" - echo "REPORT=${REPORT}" >> "$GITHUB_ENV" - - - name: Slack Notification - if: ${{ always() && !success() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - - send-report: - if: ${{ always() }} - needs: - [ - matrix-preparation, - start-runner-linux, - build-linux, - stop-runner-linux, - build-macos, - package-release, - ] - - name: Send Slack notification - runs-on: ubuntu-22.04 - steps: - - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - - - name: Prepare whole job status - if: ${{ always() }} - continue-on-error: true - env: - NEEDS_JSON: ${{ toJSON(needs) }} - run: | - echo "${NEEDS_JSON}" > /tmp/needs_context.json - JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \ - --needs_context_json /tmp/needs_context.json) - echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV" - - - name: Slack Notification - if: ${{ always() }} - continue-on-error: true - uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7 - env: - SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} - SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png - SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }} - SLACK_MESSAGE: "Full run finished with status ${{ env.JOB_STATUS || 'failure' }} \ - (${{ env.ACTION_RUN_URL }})\n\ - - matrix-preparation: ${{ needs.matrix-preparation.result || 'Did not run.'}}\n\n\ - - start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\ - - build-linux: ${{ needs.build-linux.result || 'Did not run.' }}\n\n\ - - stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}\n\n\ - - build-macos: ${{ needs.build-macos.result || 'Did not run.' }}\n\n\ - - package-release: ${{ needs.package-release.outputs.report || 'Did not run.' }}" - SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index eb9edac5c..3769d7e34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -12,6 +12,7 @@ jobs: permissions: pull-requests: write outputs: + concrete-python: ${{ steps.concrete-python.outputs.any_changed }} compiler: ${{ steps.compiler.outputs.any_changed }} optimizer: ${{ steps.optimizer.outputs.any_changed }} concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }} @@ -22,6 +23,12 @@ jobs: fetch-depth: 0 token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }} + - name: Get changed files in the concrete-python directory + id: concrete-python + uses: tj-actions/changed-files@7a453ffa2eb31a7e84f3281f88ef6d774c4d807d + with: + files: frontends/concrete-python + - name: Get changed files in the concrete-compiler directory id: compiler uses: tj-actions/changed-files@7a453ffa2eb31a7e84f3281f88ef6d774c4d807d @@ -40,6 +47,14 @@ jobs: with: files: backends/concrete-cpu +################################################# +# concrete-python jobs ########################## + concrete-python: + needs: file-change + if: needs.file-change.outputs.concrete-python == 'true' + uses: ./.github/workflows/concrete_python_build_and_test.yml + secrets: inherit + ################################################# # Compiler jobs ################################# compiler-compliance: