github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-08 19:44:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

chore(ci): Integrate concrete-python CI

Browse Source
This commit is contained in:
Quentin Bourgerie
2023-03-08 14:13:52 +01:00
parent 9858cf03ba
commit 54eba86c02
3 changed files with 399 additions and 749 deletions
Download Patch File Download Diff File Expand all files Collapse all files

384
.github/workflows/concrete_python_build_and_test.yml vendored Normal file
View File

@@ -0,0 +1,384 @@
name: concrete-python
on:
workflow_call:
concurrency:
group: concrete-python-${{ github.ref }}
cancel-in-progress: true
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
IS_PR: ${{ github.event_name == 'pull_request' }}
IS_WEEKLY: ${{ github.event_name == 'schedule' }}
IS_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
IS_PUSH_TO_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
RUNNER_TOOL_CACHE: /opt/hostedtoolcache
jobs:
matrix-preparation:
runs-on: ubuntu-22.04
outputs:
linux-matrix: ${{ steps.set-matrix.outputs.linux-matrix }}
macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }}
needs-37-linux-runner: ${{ steps.set-matrix.outputs.needs-37-linux-runner }}
needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }}
needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }}
needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }}
steps:
- name: Checkout code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Set matrix
id: set-matrix
run: |
cd frontends/concrete-python
BUILD_TYPE=
if [[ "${IS_PR}" == "true" ]]; then
BUILD_TYPE="pr"
elif [[ "${IS_WEEKLY}" == "true" ]]; then
BUILD_TYPE="weekly"
elif [[ "${IS_RELEASE}" == "true" ]]; then
BUILD_TYPE="release"
elif [[ "${IS_PUSH_TO_MAIN}" == "true" ]]; then
BUILD_TYPE="push_to_main"
else
echo "Unknown BUILD_TYPE! Aborting"
exit 1
fi
MATRIX_JSON=$(mktemp --suffix=.json)
echo "Prepared build matrix:"
python3 ./script/actions_utils/generate_test_matrix.py \
--output-json "${MATRIX_JSON}" \
--build-type "${BUILD_TYPE}"
LINUX_MATRIX=$(jq -rc '. | map(select(.os_kind=="linux"))' "${MATRIX_JSON}")
MACOS_MATRIX=$(jq -rc '. | map(select(.os_kind=="macos"))' "${MATRIX_JSON}")
echo "Linux Matrix:"
echo "${LINUX_MATRIX}" | jq '.'
echo "macOS Matrix:"
echo "${MACOS_MATRIX}" | jq '.'
echo "::set-output name=linux-matrix::${LINUX_MATRIX}"
echo "::set-output name=macos-matrix::${MACOS_MATRIX}"
NEEDS_LINUX_37_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.7")) | length > 0')
NEEDS_LINUX_38_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.8")) | length > 0')
NEEDS_LINUX_39_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0')
NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0')
echo "Needs Linux 3.7 runner:"
echo "${NEEDS_LINUX_37_RUNNER}"
echo "Needs Linux 3.8 runner:"
echo "${NEEDS_LINUX_38_RUNNER}"
echo "Needs Linux 3.9 runner:"
echo "${NEEDS_LINUX_39_RUNNER}"
echo "Needs Linux 3.10 runner:"
echo "${NEEDS_LINUX_310_RUNNER}"
echo "::set-output name=needs-37-linux-runner::${NEEDS_LINUX_37_RUNNER}"
echo "::set-output name=needs-38-linux-runner::${NEEDS_LINUX_38_RUNNER}"
echo "::set-output name=needs-39-linux-runner::${NEEDS_LINUX_39_RUNNER}"
echo "::set-output name=needs-310-linux-runner::${NEEDS_LINUX_310_RUNNER}"
start-runner-linux:
needs: [matrix-preparation]
name: Start EC2 runner
runs-on: ubuntu-22.04
outputs:
label-37: ${{ steps.start-ec2-runner-37.outputs.label }}
ec2-instance-id-37: ${{ steps.start-ec2-runner-37.outputs.ec2-instance-id || '' }}
label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
label-39: ${{ steps.start-ec2-runner-39.outputs.label }}
ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }}
label-310: ${{ steps.start-ec2-runner-310.outputs.label }}
ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }}
matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }}
steps:
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
with:
aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }}
- name: Start EC2 runner python 37
id: start-ec2-runner-37
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-37-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 38
id: start-ec2-runner-38
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-38-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 39
id: start-ec2-runner-39
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-39-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 310
id: start-ec2-runner-310
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-310-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.CONCRETE_PYTHON_AWS_EC2_SECURITY_GROUP_ID }}
- name: Update Linux runs_on Matrix
id: update-linux-matrix
env:
MATRIX: ${{ needs.matrix-preparation.outputs.linux-matrix }}
run: |
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.7") |= . + {"runs_on": "${{ steps.start-ec2-runner-37.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.8") |= . + {"runs_on": "${{ steps.start-ec2-runner-38.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )')
echo "Updated matrix:"
echo "${MATRIX}"
echo "::set-output name=linux-matrix::${MATRIX}"
build-linux:
needs: [start-runner-linux]
runs-on: ${{ matrix.runs_on }}
# Run in a clean container
container:
image: ubuntu:22.04
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.start-runner-linux.outputs.matrix)) }}
env:
IS_REF_BUILD: ${{ matrix.python_version == '3.8' }}
steps:
- name: Docker container related setup and git installation
run: |
TZ=Europe/Paris
echo "TZ=${TZ}" >> "$GITHUB_ENV"
ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
apt update && apt install git git-lfs -y
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
# Fetch-detph 0 to have all commits for changelog generation
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
with:
python-version: ${{ matrix.python_version }}
- name: Install dependencies
id: install-deps
run: |
cd frontends/concrete-python
./script/make_utils/setup_os_deps.sh
make setup_env
- name: Source code Conformance
id: cs
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
# pcc launches an internal target with proper flags
run: |
cd frontends/concrete-python
make pcc
- name: Generate release changelog
id: changelog
if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }}
run: |
cd frontends/concrete-python
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md"
echo "::set-output name=changelog-file::${CHANGELOG_FILE}"
poetry run python ./script/make_utils/changelog_helper.py \
--to-ref "${GIT_TAG}" \
--to-ref-must-have-tag \
--ancestor-must-have-tag > "${CHANGELOG_FILE}"
- name: Conformance status
id: conformance
if: ${{ always() && !cancelled() }}
env:
CONFORMANCE_STATUS: ${{ steps.cs.outcome == 'success' }}
run: |
cd frontends/concrete-python
if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then
echo "Conformance failed, check logs"
exit 1
fi
# Create packages before tests, to be able to get them if some unexpected test failure happens
# Build the package only once, as we don't have binary dependency this can be used on Linux
# and macOS as long as the dependencies are available
- name: Build wheel
id: build-wheel
if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }}
run: |
cd frontends/concrete-python
rm -rf dist
poetry build -f wheel
- name: Upload wheel artifact
if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }}
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
with:
name: py3-wheel
path: frontends/concrete-python/dist/*.whl
- name: PyTest Source Code
id: pytest
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
run: |
cd frontends/concrete-python
make pytest
# Compute coverage only on reference build
- name: Test coverage
id: coverage
if: ${{ always() && fromJSON(env.IS_REF_BUILD) && steps.pytest.outcome != 'skipped' && !cancelled() }}
run: |
cd frontends/concrete-python
./script/actions_utils/coverage.sh .global-coverage.json
- name: Comment with coverage
uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
continue-on-error: true
with:
path: frontends/concrete-python/diff-coverage.txt
recreate: true
stop-runner-linux:
name: Stop EC2 runner
needs: [build-linux, start-runner-linux]
runs-on: ubuntu-22.04
if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
with:
aws-access-key-id: ${{ secrets.CONCRETE_PYTHON_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.CONCRETE_PYTHON_AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.CONCRETE_PYTHON_AWS_REGION }}
- name: Stop EC2 runner python 37
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-37 }}
with:
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-37 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-37 }}
mode: stop
- name: Stop EC2 runner python 38
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
with:
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-38 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
mode: stop
- name: Stop EC2 runner python 39
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-39 }}
with:
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-39 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-39 }}
mode: stop
- name: Stop EC2 runner python 310
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-310 }}
with:
github-token: ${{ secrets.CONCRETE_PYTHON_EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-310 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }}
mode: stop
build-macos:
needs: [matrix-preparation]
if: ${{ needs.matrix-preparation.outputs.macos-matrix != '[]' }}
runs-on: ${{ matrix.runs_on }}
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.matrix-preparation.outputs.macos-matrix)) }}
steps:
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
with:
python-version: ${{ matrix.python_version }}
- name: Install dependencies
id: install-deps
run: |
cd frontends/concrete-python
./script/make_utils/setup_os_deps.sh
PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
echo "PATH=${PATH}" >> "$GITHUB_ENV"
make setup_env
- name: PyTest Source Code
run: |
cd frontends/concrete-python
make pytest

749
.github/workflows/continuous-integration.yaml vendored
View File

@@ -1,749 +0,0 @@
name: concrete-numpy CI Pipeline
on:
pull_request:
push:
branches:
- main
tags:
- "v*"
schedule:
# * is a special character in YAML so you have to quote this string
# At 22:00 on Sunday
# Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
- cron: '0 22 * * 0'
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
IS_PR: ${{ github.event_name == 'pull_request' }}
IS_WEEKLY: ${{ github.event_name == 'schedule' }}
IS_RELEASE: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
IS_PUSH_TO_MAIN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache
RUNNER_TOOL_CACHE: /opt/hostedtoolcache
jobs:
matrix-preparation:
runs-on: ubuntu-22.04
outputs:
linux-matrix: ${{ steps.set-matrix.outputs.linux-matrix }}
macos-matrix: ${{ steps.set-matrix.outputs.macos-matrix }}
needs-37-linux-runner: ${{ steps.set-matrix.outputs.needs-37-linux-runner }}
needs-38-linux-runner: ${{ steps.set-matrix.outputs.needs-38-linux-runner }}
needs-39-linux-runner: ${{ steps.set-matrix.outputs.needs-39-linux-runner }}
needs-310-linux-runner: ${{ steps.set-matrix.outputs.needs-310-linux-runner }}
steps:
- name: Checkout code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Set matrix
id: set-matrix
run: |
BUILD_TYPE=
if [[ "${IS_PR}" == "true" ]]; then
BUILD_TYPE="pr"
elif [[ "${IS_WEEKLY}" == "true" ]]; then
BUILD_TYPE="weekly"
elif [[ "${IS_RELEASE}" == "true" ]]; then
BUILD_TYPE="release"
elif [[ "${IS_PUSH_TO_MAIN}" == "true" ]]; then
BUILD_TYPE="push_to_main"
else
echo "Unknown BUILD_TYPE! Aborting"
exit 1
fi
MATRIX_JSON=$(mktemp --suffix=.json)
echo "Prepared build matrix:"
python3 ./script/actions_utils/generate_test_matrix.py \
--output-json "${MATRIX_JSON}" \
--build-type "${BUILD_TYPE}"
LINUX_MATRIX=$(jq -rc '. | map(select(.os_kind=="linux"))' "${MATRIX_JSON}")
MACOS_MATRIX=$(jq -rc '. | map(select(.os_kind=="macos"))' "${MATRIX_JSON}")
echo "Linux Matrix:"
echo "${LINUX_MATRIX}" | jq '.'
echo "macOS Matrix:"
echo "${MACOS_MATRIX}" | jq '.'
echo "::set-output name=linux-matrix::${LINUX_MATRIX}"
echo "::set-output name=macos-matrix::${MACOS_MATRIX}"
NEEDS_LINUX_37_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.7")) | length > 0')
NEEDS_LINUX_38_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.8")) | length > 0')
NEEDS_LINUX_39_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.9")) | length > 0')
NEEDS_LINUX_310_RUNNER=$(echo "${LINUX_MATRIX}" | \
jq -rc '. | map(select(.os_kind=="linux" and .python_version=="3.10")) | length > 0')
echo "Needs Linux 3.7 runner:"
echo "${NEEDS_LINUX_37_RUNNER}"
echo "Needs Linux 3.8 runner:"
echo "${NEEDS_LINUX_38_RUNNER}"
echo "Needs Linux 3.9 runner:"
echo "${NEEDS_LINUX_39_RUNNER}"
echo "Needs Linux 3.10 runner:"
echo "${NEEDS_LINUX_310_RUNNER}"
echo "::set-output name=needs-37-linux-runner::${NEEDS_LINUX_37_RUNNER}"
echo "::set-output name=needs-38-linux-runner::${NEEDS_LINUX_38_RUNNER}"
echo "::set-output name=needs-39-linux-runner::${NEEDS_LINUX_39_RUNNER}"
echo "::set-output name=needs-310-linux-runner::${NEEDS_LINUX_310_RUNNER}"
start-runner-linux:
needs: [matrix-preparation]
name: Start EC2 runner
runs-on: ubuntu-22.04
outputs:
label-37: ${{ steps.start-ec2-runner-37.outputs.label }}
ec2-instance-id-37: ${{ steps.start-ec2-runner-37.outputs.ec2-instance-id || '' }}
label-38: ${{ steps.start-ec2-runner-38.outputs.label }}
ec2-instance-id-38: ${{ steps.start-ec2-runner-38.outputs.ec2-instance-id || '' }}
label-39: ${{ steps.start-ec2-runner-39.outputs.label }}
ec2-instance-id-39: ${{ steps.start-ec2-runner-39.outputs.ec2-instance-id || '' }}
label-310: ${{ steps.start-ec2-runner-310.outputs.label }}
ec2-instance-id-310: ${{ steps.start-ec2-runner-310.outputs.ec2-instance-id || '' }}
matrix: ${{ steps.update-linux-matrix.outputs.linux-matrix }}
steps:
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Start EC2 runner python 37
id: start-ec2-runner-37
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-37-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 38
id: start-ec2-runner-38
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-38-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 39
id: start-ec2-runner-39
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-39-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
- name: Start EC2 runner python 310
id: start-ec2-runner-310
if: ${{ !cancelled() && fromJSON(needs.matrix-preparation.outputs.needs-310-linux-runner) }}
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
with:
mode: start
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
ec2-image-id: ${{ secrets.AWS_EC2_AMI }}
ec2-instance-type: ${{ secrets.AWS_EC2_INSTANCE_TYPE }}
subnet-id: ${{ secrets.AWS_EC2_SUBNET_ID }}
security-group-id: ${{ secrets.AWS_EC2_SECURITY_GROUP_ID }}
- name: Update Linux runs_on Matrix
id: update-linux-matrix
env:
MATRIX: ${{ needs.matrix-preparation.outputs.linux-matrix }}
run: |
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.7") |= . + {"runs_on": "${{ steps.start-ec2-runner-37.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.8") |= . + {"runs_on": "${{ steps.start-ec2-runner-38.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.9") |= . + {"runs_on": "${{ steps.start-ec2-runner-39.outputs.label }}"}) )')
MATRIX=$(echo "${MATRIX}" | jq -rc \
'(. | map(select(.os_kind=="linux" and .python_version=="3.10") |= . + {"runs_on": "${{ steps.start-ec2-runner-310.outputs.label }}"}) )')
echo "Updated matrix:"
echo "${MATRIX}"
echo "::set-output name=linux-matrix::${MATRIX}"
build-linux:
needs: [start-runner-linux]
runs-on: ${{ matrix.runs_on }}
# Run in a clean container
container:
image: ubuntu:22.04
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.start-runner-linux.outputs.matrix)) }}
env:
IS_REF_BUILD: ${{ matrix.python_version == '3.8' }}
steps:
- name: Docker container related setup and git installation
run: |
TZ=Europe/Paris
echo "TZ=${TZ}" >> "$GITHUB_ENV"
ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && echo ${TZ} > /etc/timezone
sed -i 's|^deb http://archive|deb http://fr.archive|g' /etc/apt/sources.list
apt update && apt install git git-lfs -y
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
# Fetch-detph 0 to have all commits for changelog generation
with:
fetch-depth: 0
- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
with:
python-version: ${{ matrix.python_version }}
- name: Install dependencies
id: install-deps
run: |
./script/make_utils/setup_os_deps.sh
make setup_env
- name: Check commits first line format
id: ccfl
if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }}
uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee
with:
pattern: '^((feat|fix|chore|refactor|style|test|docs)(\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\))?\:) .+$'
flags: 'gs'
error: "Your first line has to contain a commit type and scope like \"feat(my_feature): msg\".\
Pattern: '^((feat|fix|chore|refactor|style|test|docs)(\\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\\))?\\:)'"
excludeDescription: 'true' # optional: this excludes the description body of a pull request
excludeTitle: 'true' # optional: this excludes the title of a pull request
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
- name: Check commits line length
id: ccll
if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }}
uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee
with:
pattern: '(^.{0,74}$\r?\n?){0,20}'
flags: 'gm'
error: 'The maximum line length of 74 characters is exceeded.'
excludeDescription: 'true' # optional: this excludes the description body of a pull request
excludeTitle: 'true' # optional: this excludes the title of a pull request
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
- name: Commit conformance
id: commit-conformance
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
env:
CCFL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccfl.outcome == 'success') || steps.ccfl.outcome == 'skipped' }}
CCLL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccll.outcome == 'success') || steps.ccll.outcome == 'skipped' }}
run: |
if [[ "${CCFL_OK}" != "true" || "${CCLL_OK}" != "true" ]]; then
echo "Issues with commits. First line ok: ${CCFL_OK}. Line length ok: ${CCLL_OK}."
exit 1
fi
- name: Source code Conformance
id: cs
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
# pcc launches an internal target with proper flags
run: |
make pcc
- name: Generate release changelog
id: changelog
if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }}
run: |
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md"
echo "::set-output name=changelog-file::${CHANGELOG_FILE}"
poetry run python ./script/make_utils/changelog_helper.py \
--to-ref "${GIT_TAG}" \
--to-ref-must-have-tag \
--ancestor-must-have-tag > "${CHANGELOG_FILE}"
- name: Conformance status
id: conformance
if: ${{ always() && !cancelled() }}
env:
CONFORMANCE_STATUS: ${{ steps.commit-conformance.outcome == 'success' && steps.cs.outcome == 'success' }}
run: |
if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then
echo "Conformance failed, check logs"
exit 1
fi
- name: Upload changelog artifacts
if: ${{ fromJSON(env.IS_REF_BUILD) && steps.changelog.outcome == 'success' && !cancelled() }}
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
with:
name: changelog
path: ${{ steps.changelog.outputs.changelog-file }}
# Create packages before tests, to be able to get them if some unexpected test failure happens
# Build the package only once, as we don't have binary dependency this can be used on Linux
# and macOS as long as the dependencies are available
- name: Build wheel
id: build-wheel
if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }}
run: |
rm -rf dist
poetry build -f wheel
- name: Upload wheel artifact
if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }}
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb
with:
name: py3-wheel
path: dist/*.whl
- name: PyTest Source Code
id: pytest
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
run: |
make pytest
# - name: PyTest CodeBlocks
# if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
# run: |
# make pytest_codeblocks
#
# - name: PyTest Notebooks
# if: ${{ fromJSON(env.IS_WEEKLY) && steps.conformance.outcome == 'success' && !cancelled() }}
# run: |
# make pytest_nb
# Compute coverage only on reference build
- name: Test coverage
id: coverage
if: ${{ always() && fromJSON(env.IS_REF_BUILD) && steps.pytest.outcome != 'skipped' && !cancelled() }}
run: |
./script/actions_utils/coverage.sh .global-coverage.json
- name: Comment with coverage
uses: marocchino/sticky-pull-request-comment@3d60a5b2dae89d44e0c6ddc69dd7536aec2071cd
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
continue-on-error: true
with:
path: diff-coverage.txt
recreate: true
# This is to manage build matrices and have a single status point for PRs
# This can be updated to take macOS into account but is impractical for private repos because of
# long builds and therefore expensive macOS testing
linux-build-status:
name: Linux build status
needs: [build-linux]
runs-on: ubuntu-22.04
if: ${{ always() }}
steps:
- name: Fail on unsuccessful Linux build
shell: bash
run: |
if [[ ${{ needs.build-linux.result }} != "success" ]]; then
exit 1
fi
- name: Slack Notification
if: ${{ always() && !success() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ needs.build-linux.result }}
SLACK_MESSAGE: "Build finished with status ${{ needs.build-linux.result }}. (${{ env.ACTION_RUN_URL }})"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
stop-runner-linux:
name: Stop EC2 runner
needs: [build-linux, start-runner-linux]
runs-on: ubuntu-22.04
if: ${{ always() && (needs.start-runner-linux.result != 'skipped') }}
steps:
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: ${{ secrets.AWS_REGION }}
- name: Stop EC2 runner python 37
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-37 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-37 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-37 }}
mode: stop
- name: Stop EC2 runner python 38
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-38 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-38 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-38 }}
mode: stop
- name: Stop EC2 runner python 39
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-39 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-39 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-39 }}
mode: stop
- name: Stop EC2 runner python 310
uses: machulav/ec2-github-runner@4e0303de215db88e1c489e07a15ca4d867f488ea
if: ${{ always() && needs.start-runner-linux.outputs.ec2-instance-id-310 }}
with:
github-token: ${{ secrets.EC2_RUNNER_BOT_TOKEN }}
label: ${{ needs.start-runner-linux.outputs.label-310 }}
ec2-instance-id: ${{ needs.start-runner-linux.outputs.ec2-instance-id-310 }}
mode: stop
build-macos:
needs: [matrix-preparation]
if: ${{ needs.matrix-preparation.outputs.macos-matrix != '[]' }}
runs-on: ${{ matrix.runs_on }}
defaults:
run:
shell: bash
strategy:
fail-fast: false
matrix: ${{ fromJSON(format('{{"include":{0}}}', needs.matrix-preparation.outputs.macos-matrix)) }}
steps:
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Set up Python ${{ matrix.python_version }}
uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
with:
python-version: ${{ matrix.python_version }}
- name: Install dependencies
id: install-deps
run: |
./script/make_utils/setup_os_deps.sh
PATH="/usr/local/opt/make/libexec/gnubin:$PATH"
echo "PATH=${PATH}" >> "$GITHUB_ENV"
make setup_env
- name: PyTest Source Code
run: |
make pytest
weekly-pip-audit:
if: ${{ github.event_name == 'schedule' }}
runs-on: ubuntu-22.04
steps:
- name: Checkout Code
uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Set up Python 3.8
uses: actions/setup-python@5ccb29d8773c3f3f653e1705f474dfaa8a06a912
with:
python-version: '3.8'
- name: Set up env
run: |
python -m pip install --upgrade pip
python -m pip install poetry
sudo apt update && sudo apt install sqlite3 -y
make setup_env
- name: Run pip-audit
shell: bash
run: |
VULN_OUT="$(mktemp --suffix=.json)"
REPORT_OUT="$(mktemp --suffix=.txt)"
echo "REPORT_OUT=${REPORT_OUT}" >> "$GITHUB_ENV"
poetry run pip-audit -f json > "${VULN_OUT}"
cat "${VULN_OUT}"
poetry run python ./script/actions_utils/parse_pip_audit_vulns.py \
--vulns-json "${VULN_OUT}" \
--vulns-report "${REPORT_OUT}"
# We load the report in a new step if we exited with an error code above to let the workflow fail
- name: Load report in env
if: ${{ always() }}
run: |
cat "${REPORT_OUT}"
REPORT="$(cat "${REPORT_OUT}")"
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
- name: Slack Notification
if: ${{ always() && !success() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "${{ env.REPORT || 'Error during pip-audit' }} (${{ env.ACTION_RUN_URL }})"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
package-release:
needs: [build-linux]
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
outputs:
report: ${{ steps.report.outputs.report || 'Did not run.' }}
name: Package and artifacts release
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Install dependencies
run: |
sudo apt-get install --no-install-recommends -y gnome-keyring sqlite3
python -m pip install --upgrade pip
python -m pip install poetry
make setup_env
- name: Set common environment variables
run: |
PROJECT_NAME_AND_VERSION=$(poetry version)
PROJECT_VERSION=$(echo "$PROJECT_NAME_AND_VERSION" | cut -d ' ' -f 2)
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
if [[ "${GIT_TAG}" != "v${PROJECT_VERSION}" ]]; then
echo "Mismatch between tag and version: ${GIT_TAG}, v${PROJECT_VERSION}"
exit 1
fi
IMAGE_BASE="${{ secrets.IMAGE_BASE }}"
ALL_IMAGE_TAGS="${IMAGE_BASE}:${GIT_TAG}"
IS_LATEST=$(poetry run python script/make_utils/is_latest.py "${PROJECT_VERSION}")
if [[ "${IS_LATEST}" == "True" ]]; then
ALL_IMAGE_TAGS="${ALL_IMAGE_TAGS},${IMAGE_BASE}:latest"
fi
IS_PRERELEASE=$(poetry run python script/make_utils/is_prerelease.py "${PROJECT_VERSION}")
echo "PROJECT_VERSION=${PROJECT_VERSION}" >> "$GITHUB_ENV"
echo "GIT_TAG=${GIT_TAG}" >> "$GITHUB_ENV"
echo "IS_LATEST=${IS_LATEST}" >> "$GITHUB_ENV"
echo "IS_PRERELEASE=${IS_PRERELEASE}" >> "$GITHUB_ENV"
echo "ALL_IMAGE_TAGS=${ALL_IMAGE_TAGS}" >> "$GITHUB_ENV"
echo "VERSIONED_IMAGE_TAG=${IMAGE_BASE}:${GIT_TAG}" >> "$GITHUB_ENV"
- name: Create directory for artifacts
if: ${{ success() && !cancelled() }}
run: |
ARTIFACTS_RAW_DIR=/tmp/release_artifacts/raw
mkdir -p "${ARTIFACTS_RAW_DIR}"
echo "ARTIFACTS_RAW_DIR=${ARTIFACTS_RAW_DIR}" >> "$GITHUB_ENV"
ARTIFACTS_PACKAGED_DIR=/tmp/release_artifacts/packaged
mkdir -p "${ARTIFACTS_PACKAGED_DIR}"
echo "ARTIFACTS_PACKAGED_DIR=${ARTIFACTS_PACKAGED_DIR}" >> "$GITHUB_ENV"
- name: Download changelog
if: ${{ success() && !cancelled() }}
id: download-changelog
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7
with:
name: changelog
path: ${{ env.ARTIFACTS_RAW_DIR }}/changelog/
- name: Download python3 wheel
if: ${{ success() && !cancelled() }}
id: download-wheel
uses: actions/download-artifact@9782bd6a9848b53b110e712e20e42d89988822b7
with:
name: py3-wheel
path: ${{ env.ARTIFACTS_PACKAGED_DIR }}/
- name: Copy wheel to docker build context
if: ${{ success() && !cancelled() }}
run: |
mkdir -p ./pkg
cp "${{ env.ARTIFACTS_PACKAGED_DIR }}"/*.whl ./pkg
- name: Login to GitHub Container Registry
if: ${{ success() && !cancelled() }}
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
registry: ghcr.io
username: ${{ secrets.BOT_USERNAME }}
password: ${{ secrets.BOT_TOKEN }}
- name: Login to DockerHub
if: ${{ success() && !cancelled() }}
uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build concrete-numpy Image
if: ${{ success() && !cancelled() }}
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
with:
context: .
file: docker/Dockerfile.release
load: true
push: false
tags: "${{ env.ALL_IMAGE_TAGS }}"
no-cache: true
- name: Release image sanity check
if: ${{ success() && !cancelled() }}
run: |
echo "Running sanity check for ${VERSIONED_IMAGE_TAG}"
docker run --rm -v "$(pwd)"/docker/release_resources:/data \
"${VERSIONED_IMAGE_TAG}" /bin/bash -c "python ./sanity_check.py"
- name: Create ready to upload/packaged artifacts and release body
if: ${{ success() && !cancelled() }}
env:
RAW_CHANGELOG_DIR: ${{ steps.download-changelog.outputs.download-path }}
run: |
cp "${RAW_CHANGELOG_DIR}"/* "${ARTIFACTS_PACKAGED_DIR}"
ls -a "${ARTIFACTS_PACKAGED_DIR}"
RELEASE_BODY_FILE=RELEASE_BODY.md
echo "RELEASE_BODY_FILE=${RELEASE_BODY_FILE}" >> "$GITHUB_ENV"
cp ./script/actions_utils/RELEASE_TEMPLATE.md "${RELEASE_BODY_FILE}"
{
echo "Docker Image: ${VERSIONED_IMAGE_TAG}";
echo "PyPI Package: https://pypi.org/project/concrete-numpy/${PROJECT_VERSION}";
echo "";
} >> "${RELEASE_BODY_FILE}"
cat "${RAW_CHANGELOG_DIR}"/* >> "${RELEASE_BODY_FILE}"
- name: Push release docker image
if: ${{ success() && !cancelled() }}
run: |
docker image push --all-tags "${{ secrets.IMAGE_BASE }}"
- name: Push package to PyPi
if: ${{ success() && !cancelled() }}
run: |
poetry run twine upload \
-u "${{ secrets.REPO_USERNAME }}" -p ${{ secrets.REPO_PASSWORD }} \
${{ secrets.REPO_DETAILS }} "${{ env.ARTIFACTS_PACKAGED_DIR }}"/*.whl
- name: Create GitHub release
if: ${{ success() && !cancelled() }}
id: create-release
uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844
with:
body_path: ${{ env.RELEASE_BODY_FILE }}
prerelease: ${{ fromJSON(env.IS_PRERELEASE) }}
files: |
${{ env.ARTIFACTS_PACKAGED_DIR }}/*
tag_name: ${{ env.GIT_TAG }}
fail_on_unmatched_files: true
token: ${{ secrets.BOT_TOKEN }}
- name: Set notification report
id: report
if: ${{ always() }}
run: |
REPORT="Creating release for ${GIT_TAG} finished with status ${{ job.status }}. \
GitHub release link: ${{ steps.create-release.outputs.url }}."
echo "${REPORT}"
echo "::set-output name=report::${REPORT}"
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
- name: Slack Notification
if: ${{ always() && !success() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
send-report:
if: ${{ always() }}
needs:
[
matrix-preparation,
start-runner-linux,
build-linux,
stop-runner-linux,
build-macos,
package-release,
]
name: Send Slack notification
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- name: Prepare whole job status
if: ${{ always() }}
continue-on-error: true
env:
NEEDS_JSON: ${{ toJSON(needs) }}
run: |
echo "${NEEDS_JSON}" > /tmp/needs_context.json
JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
--needs_context_json /tmp/needs_context.json)
echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"
- name: Slack Notification
if: ${{ always() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
env:
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
SLACK_MESSAGE: "Full run finished with status ${{ env.JOB_STATUS || 'failure' }} \
(${{ env.ACTION_RUN_URL }})\n\
- matrix-preparation: ${{ needs.matrix-preparation.result || 'Did not run.'}}\n\n\
- start-runner-linux: ${{ needs.start-runner-linux.result || 'Did not run.'}}\n\n\
- build-linux: ${{ needs.build-linux.result || 'Did not run.' }}\n\n\
- stop-runner-linux: ${{ needs.stop-runner-linux.result || 'Did not run.'}}\n\n\
- build-macos: ${{ needs.build-macos.result || 'Did not run.' }}\n\n\
- package-release: ${{ needs.package-release.outputs.report || 'Did not run.' }}"
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}

15
.github/workflows/main.yml vendored
View File

@@ -12,6 +12,7 @@ jobs:
permissions:
pull-requests: write
outputs:
concrete-python: ${{ steps.concrete-python.outputs.any_changed }}
compiler: ${{ steps.compiler.outputs.any_changed }}
optimizer: ${{ steps.optimizer.outputs.any_changed }}
concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }}
@@ -22,6 +23,12 @@ jobs:
fetch-depth: 0
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Get changed files in the concrete-python directory
id: concrete-python
uses: tj-actions/changed-files@7a453ffa2eb31a7e84f3281f88ef6d774c4d807d
with:
files: frontends/concrete-python
- name: Get changed files in the concrete-compiler directory
id: compiler
uses: tj-actions/changed-files@7a453ffa2eb31a7e84f3281f88ef6d774c4d807d
@@ -40,6 +47,14 @@ jobs:
with:
files: backends/concrete-cpu
#################################################
# concrete-python jobs ##########################
concrete-python:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true'
uses: ./.github/workflows/concrete_python_build_and_test.yml
secrets: inherit
#################################################
# Compiler jobs #################################
compiler-compliance: