mirror of
https://github.com/zama-ai/concrete.git
synced 2026-02-09 03:55:04 -05:00
ae841cf77f
Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2.3.0 to 2.3.1.
- [Release notes](https://github.com/actions/setup-python/releases)
- [Commits](0066b88440...f382193329)
---
updated-dependencies:
- dependency-name: actions/setup-python
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
823 lines
36 KiB
YAML
823 lines
36 KiB
YAML
name: concretefhe CI Pipeline
|
|
on:
|
|
pull_request:
|
|
push:
|
|
branches:
|
|
- main
|
|
tags:
|
|
- "v*"
|
|
|
|
# Allows you to run this workflow manually from the Actions tab
|
|
workflow_dispatch:
|
|
inputs:
|
|
rebuild-env-docker:
|
|
description: "String to indicate whether the docker should be rebuilt, true or false"
|
|
required: true
|
|
default: "false"
|
|
|
|
# Allows external webhook trigger
|
|
repository_dispatch:
|
|
types:
|
|
- rebuild-env-docker
|
|
|
|
schedule:
|
|
# * is a special character in YAML so you have to quote this string
|
|
# At 22:00 on Sunday
|
|
# Timezone is UTC, so Paris time is +2 during the summer and +1 during winter
|
|
- cron: '0 22 * * 0'
|
|
|
|
env:
|
|
FORCE_REBUILD_DOCKER: ${{ (github.event_name == 'workflow_dispatch' && fromJSON(github.event.inputs.rebuild-env-docker)) || (github.event_name == 'repository_dispatch' && github.event.action == 'rebuild-env-docker') }}
|
|
ENV_DOCKERFILE: docker/Dockerfile.concretefhe-env
|
|
PREFLIGHT_IMAGE_BASE: ghcr.io/zama-ai/concretefhe-env:preflight
|
|
LATEST_IMAGE: ghcr.io/zama-ai/concretefhe-env:latest
|
|
BASE_IMAGE: ghcr.io/zama-ai/concretefhe-env
|
|
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
IS_PR: ${{ github.event_name == 'pull_request' }}
|
|
|
|
jobs:
|
|
build-preflight-docker:
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
name: Build & Push the concretefhe-env preflight Docker Image
|
|
runs-on: ubuntu-20.04
|
|
outputs:
|
|
image: ${{ steps.set_image.outputs.image || env.LATEST_IMAGE }}
|
|
compiler-tag: ${{ steps.set_image.outputs.compiler-tag }}
|
|
needs-push: ${{ env.BUILD_DOCKER }}
|
|
force-rebuild-docker: ${{ env.FORCE_REBUILD_DOCKER }}
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
env:
|
|
WHEEL: concretefhe_compiler-0.1.0-cp38-cp38-manylinux_2_24_x86_64.whl
|
|
|
|
steps:
|
|
- uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Get changed files
|
|
if: ${{ (github.event_name == 'push' && !startsWith(github.ref, 'refs/tags/')) || github.event_name == 'pull_request' }}
|
|
id: files
|
|
run: |
|
|
CURRENT_SHA="${{ github.sha }}"
|
|
if [[ "${IS_PR}" == "true" ]]; then
|
|
BEFORE_SHA="${{ github.event.pull_request.base.sha }}"
|
|
else
|
|
BEFORE_SHA="$(./script/actions_utils/get_latest_run.sh \
|
|
--token ${{ secrets.BOT_TOKEN }} \
|
|
--org-repo ${{ github.repository }} \
|
|
--event-types 'push workflow_dispatch repository_dispatch')"
|
|
fi
|
|
ALL_CHANGED_FILES=$(git diff --name-only "${CURRENT_SHA}" "${BEFORE_SHA}")
|
|
|
|
echo "Before sha1 ${BEFORE_SHA}"
|
|
echo "Current sha1 ${CURRENT_SHA}"
|
|
|
|
echo "Changed files:"
|
|
echo "${ALL_CHANGED_FILES}"
|
|
|
|
echo "::set-output name=all::${ALL_CHANGED_FILES//$'\n'/ }"
|
|
|
|
- name: Should rebuild docker check
|
|
run: |
|
|
set +e
|
|
ALL_CHANGED_FILES="${{ steps.files.outputs.all }}"
|
|
ALL_CHANGED_FILES="${ALL_CHANGED_FILES// /$'\n'}"
|
|
echo "${ALL_CHANGED_FILES}" | grep "${ENV_DOCKERFILE}$"
|
|
DOCKERFILE_CHANGED=$?
|
|
if [[ "${DOCKERFILE_CHANGED}" == "0" || "${FORCE_REBUILD_DOCKER}" == "true" ]]; then
|
|
echo "Should rebuild docker image!"
|
|
echo "BUILD_DOCKER=true" >> "$GITHUB_ENV"
|
|
else
|
|
echo "Docker image up to date."
|
|
echo "BUILD_DOCKER=false" >> "$GITHUB_ENV"
|
|
fi
|
|
# https://github.com/zama-ai/concretefhe-internal/issues/809
|
|
# Remove gh_dl_release call once package is on PyPi
|
|
- name: Set prefligh Docker image download compiler
|
|
id: set_image
|
|
if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
run: |
|
|
PREFLIGHT_IMAGE_TAG=$(echo ${{ github.ref }} | sed -e 's/\//-/g')
|
|
PREFLIGHT_IMAGE="${PREFLIGHT_IMAGE_BASE}-${PREFLIGHT_IMAGE_TAG}"
|
|
LABEL_SHA1=$(git rev-parse HEAD)
|
|
echo "::set-output name=image::${PREFLIGHT_IMAGE}"
|
|
echo "PREFLIGHT_IMAGE=${PREFLIGHT_IMAGE}" >> "$GITHUB_ENV"
|
|
echo "LABEL_SHA1=${LABEL_SHA1}" >> "$GITHUB_ENV"
|
|
|
|
mkdir -p pkg
|
|
./script/actions_utils/gh_dl_release.sh \
|
|
--token ${{ secrets.BOT_TOKEN }} \
|
|
--org-repo zama-ai/homomorphizer \
|
|
--file "${WHEEL}" \
|
|
--dest-file "pkg/${WHEEL}" \
|
|
--compiler-tag-output-file compiler-output-tag.txt
|
|
|
|
COMPILER_TAG=$(cat compiler-output-tag.txt)
|
|
echo "::set-output name=compiler-tag::${COMPILER_TAG}"
|
|
# Disabled buildx for now as we are seeing a lot of fails on layer pushes
|
|
# - name: Set up Docker Buildx
|
|
# if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
# id: buildx
|
|
# uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
|
|
- name: Login to GitHub Container Registry
|
|
if: ${{ fromJSON(env.BUILD_DOCKER) }}
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Build concretefhe-env Image
|
|
if: ${{ success() && !cancelled() && fromJSON(env.BUILD_DOCKER) }}
|
|
uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
|
|
with:
|
|
context: .
|
|
# builder: ${{ steps.buildx.outputs.name }}
|
|
build-args: |
|
|
WHEEL=${{ env.WHEEL }}
|
|
file: docker/Dockerfile.concretefhe-env
|
|
no-cache: true
|
|
push: true
|
|
tags: "${{ env.PREFLIGHT_IMAGE }}"
|
|
labels: |
|
|
concretefhe_sha=${{ env.LABEL_SHA1 }}
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Docker image preflight build ${{ env.PREFLIGHT_IMAGE }} finished with \
|
|
status ${{ job.status }}. Rebuilt image: ${{ env.BUILD_DOCKER || 'false' }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
build:
|
|
needs: [build-preflight-docker]
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
runs-on: ubuntu-20.04
|
|
container:
|
|
image: ${{ needs.build-preflight-docker.outputs.image }}
|
|
credentials:
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
defaults:
|
|
run:
|
|
shell: '/usr/bin/bash -e {0}'
|
|
strategy:
|
|
matrix:
|
|
# YAML footgun : https://twitter.com/webology/status/1445394072492023811?s=20
|
|
# Versions need to be quoted or risk being interpreted as floating point numbers
|
|
python-version: ["3.8"]
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
steps:
|
|
- name: Checkout Code
|
|
uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
|
|
with:
|
|
fetch-depth: 0
|
|
- name: Set up Python ${{ matrix.python-version }}
|
|
uses: actions/setup-python@f38219332975fe8f9c04cca981d674bf22aea1d3
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
# - name: Cache Installation Files
|
|
# uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
|
|
# with:
|
|
# # Paths are Unix specific for now
|
|
# path: |
|
|
# ~/.cache/pip
|
|
# ~/.cache/pypoetry
|
|
# # Ignore line break in the evaluated double quoted string
|
|
# key: "${{ secrets.CACHE_VERSION }}-${{ runner.os }}-build-${{ matrix.python-version }}-\
|
|
# ${{ hashFiles('poetry.lock') }}"
|
|
# restore-keys: |
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-build-${{ matrix.python-version }}-
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-build-
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-
|
|
# ${{ secrets.CACHE_VERSION }}-
|
|
- name: Install dependencies
|
|
id: install-deps
|
|
run: |
|
|
python -m pip install --upgrade pip
|
|
python -m pip install poetry
|
|
make setup_env
|
|
- name: Check commits first line format
|
|
id: ccfl
|
|
if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
uses: gsactions/commit-message-checker@f27f413dcf8ebcb469d2ce4ae4e45e131d105de6
|
|
with:
|
|
pattern: '^((feat|fix|chore|refactor|style|test|docs)(\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\))?\:) .+$'
|
|
flags: 'gs'
|
|
error: "Your first line has to contain a commit type and scope like \"feat(my_feature): msg\".\
|
|
Pattern: '^((feat|fix|chore|refactor|style|test|docs)(\\((bounds|helpers|data_types|debugging|extensions|fhe_circuit|mlir|graph|optimization|representation|tracing|values|benchmarks|ci|scripts|compilation|execution|deps)\\))?\\:)'"
|
|
excludeDescription: 'true' # optional: this excludes the description body of a pull request
|
|
excludeTitle: 'true' # optional: this excludes the title of a pull request
|
|
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
|
|
accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
|
|
- name: Check commits line length
|
|
id: ccll
|
|
if: ${{ fromJSON(env.IS_PR) && steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
uses: gsactions/commit-message-checker@f27f413dcf8ebcb469d2ce4ae4e45e131d105de6
|
|
with:
|
|
pattern: '(^.{0,74}$\r?\n?){0,20}'
|
|
flags: 'gm'
|
|
error: 'The maximum line length of 74 characters is exceeded.'
|
|
excludeDescription: 'true' # optional: this excludes the description body of a pull request
|
|
excludeTitle: 'true' # optional: this excludes the title of a pull request
|
|
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
|
|
accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
|
|
- name: Commit conformance
|
|
id: commit-conformance
|
|
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
env:
|
|
CCFL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccfl.outcome == 'success') || steps.ccfl.outcome == 'skipped' }}
|
|
CCLL_OK: ${{ (fromJSON(env.IS_PR) && steps.ccll.outcome == 'success') || steps.ccll.outcome == 'skipped' }}
|
|
run: |
|
|
if [[ "${CCFL_OK}" != "true" || "${CCLL_OK}" != "true" ]]; then
|
|
echo "Issues with commits. First line ok: ${CCFL_OK}. Line length ok: ${CCLL_OK}."
|
|
exit 1
|
|
fi
|
|
- name: Source code Conformance
|
|
id: cs
|
|
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
# pcc launches an internal target with proper flags
|
|
run: |
|
|
make pcc
|
|
- name: Build docs
|
|
id: cbd
|
|
if: ${{ steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
make docs
|
|
- name: Generate release changelog
|
|
id: changelog
|
|
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') && steps.install-deps.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
|
|
CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md"
|
|
echo "::set-output name=changelog-file::${CHANGELOG_FILE}"
|
|
poetry run python ./script/make_utils/changelog_helper.py \
|
|
--to-ref "${GIT_TAG}" \
|
|
--to-ref-must-have-tag \
|
|
--ancestor-must-have-tag > "${CHANGELOG_FILE}"
|
|
- name: Conformance status
|
|
id: conformance
|
|
if: ${{ always() && !cancelled() }}
|
|
env:
|
|
CONFORMANCE_STATUS: ${{ steps.commit-conformance.outcome == 'success' && steps.cs.outcome == 'success' && steps.cbd.outcome == 'success' }}
|
|
run: |
|
|
if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then
|
|
echo "Conformance failed, check logs"
|
|
exit 1
|
|
fi
|
|
- name: Archive docs artifacts
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
|
with:
|
|
name: html-docs
|
|
path: docs/_build/html
|
|
- name: Upload changelog artifacts
|
|
if: ${{ steps.changelog.outcome == 'success' && !cancelled() }}
|
|
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
|
with:
|
|
name: changelog
|
|
path: ${{ steps.changelog.outputs.changelog-file }}
|
|
- name: PyTest Source Code
|
|
id: pytest
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
make pytest
|
|
- name: Test CodeBlocks
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
make test_codeblocks
|
|
- name: PyTest Notebooks
|
|
if: ${{ github.event_name == 'schedule' && steps.conformance.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
make pytest_nb
|
|
- name: PyTest Progress Tracker
|
|
if: ${{ steps.conformance.outcome == 'success' && !cancelled() }}
|
|
run: |
|
|
make pytest_progress_tracker
|
|
- name: Test coverage
|
|
id: coverage
|
|
if: ${{ always() && steps.pytest.outcome != 'skipped' && !cancelled() }}
|
|
run: |
|
|
./script/actions_utils/coverage.sh global-coverage-infos.json
|
|
- name: Archive test coverage
|
|
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
|
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
|
|
with:
|
|
name: coverage
|
|
path: coverage.html
|
|
- name: Comment with coverage
|
|
uses: marocchino/sticky-pull-request-comment@39c5b5dc7717447d0cba270cd115037d32d28443
|
|
if: ${{ steps.coverage.outcome != 'skipped' && !cancelled() }}
|
|
continue-on-error: true
|
|
with:
|
|
path: diff-coverage.txt
|
|
recreate: true
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Build finished with status ${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
publish-docs:
|
|
needs: [build]
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
runs-on: ubuntu-20.04
|
|
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
|
|
|
|
steps:
|
|
- name: Prepare docs push
|
|
id: docs-push-infos
|
|
run: |
|
|
if [[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }} != "" ]] && \
|
|
[[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }} != "" ]]; then
|
|
REF_NAME=$(echo "${{ github.ref }}" | sed 's/refs\/heads\///g')
|
|
echo "::set-output name=has-preprod::true"
|
|
echo "::set-output name=aws-bucket::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }}"
|
|
echo "::set-output name=aws-distribution::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }}"
|
|
echo "::set-output name=dest-dir::concretefhe/${REF_NAME}"
|
|
else
|
|
echo "::set-output name=has-preprod::false"
|
|
fi
|
|
- name: Download Documentation
|
|
if: ${{ fromJSON(steps.docs-push-infos.outputs.has-preprod) }}
|
|
id: download
|
|
uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
|
|
with:
|
|
name: html-docs
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@ea7b857d8a33dc2fb4ef5a724500044281b49a5e
|
|
with:
|
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: ${{ secrets.AWS_REGION }}
|
|
|
|
- name: Publish Documentation to S3
|
|
id: publish
|
|
if: ${{ steps.download.outcome == 'success' && !cancelled() }}
|
|
env:
|
|
AWS_S3_BUCKET: ${{ steps.docs-push-infos.outputs.aws-bucket }}
|
|
SOURCE_DIR: '.'
|
|
DEST_DIR: ${{ steps.docs-push-infos.outputs.dest-dir }}
|
|
run: |
|
|
aws s3 sync "${SOURCE_DIR}" s3://"${AWS_S3_BUCKET}/${DEST_DIR}" --delete --acl public-read
|
|
|
|
- name: Invalidate CloudFront Cache
|
|
if: ${{ steps.publish.outcome == 'success' }}
|
|
env:
|
|
SOURCE_PATH: "/${{ steps.docs-push-infos.outputs.dest-dir }}/*"
|
|
DISTRIBUTION_ID: ${{ steps.docs-push-infos.outputs.aws-distribution }}
|
|
run: |
|
|
aws cloudfront create-invalidation \
|
|
--distribution-id "${DISTRIBUTION_ID}" \
|
|
--paths "${SOURCE_PATH}"
|
|
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Publishing documentation finished with status ${{ job.status }}. \
|
|
Pushed to preprod: ${{ steps.docs-push-infos.outputs.has-preprod }}"
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
push-docker-image:
|
|
needs: [build-preflight-docker, build]
|
|
if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main' && fromJSON(needs.build-preflight-docker.outputs.needs-push)) || fromJSON(needs.build-preflight-docker.outputs.force-rebuild-docker) }}
|
|
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
name: Push env docker image
|
|
runs-on: ubuntu-20.04
|
|
env:
|
|
PREFLIGHT_IMAGE: ${{ needs.build-preflight-docker.outputs.image }}
|
|
COMPILER_TAG: ${{ needs.build-preflight-docker.outputs.compiler-tag }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Pull preflight image
|
|
run: |
|
|
docker pull "${PREFLIGHT_IMAGE}"
|
|
# https://github.com/zama-ai/concretefhe-internal/issues/809
|
|
# update once release workflow is ok on the compiler side
|
|
- name: Retag to latest and concretefhe_compiler_version-concretefhe_sha1 and push
|
|
run: |
|
|
SHA1=$(git rev-parse HEAD)
|
|
TAGGED_IMAGE="${BASE_IMAGE}:${COMPILER_TAG}-${SHA1}"
|
|
docker tag "${PREFLIGHT_IMAGE}" "${LATEST_IMAGE}"
|
|
docker tag "${PREFLIGHT_IMAGE}" "${TAGGED_IMAGE}"
|
|
docker push "${LATEST_IMAGE}"
|
|
docker push "${TAGGED_IMAGE}"
|
|
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Pushing docker image ${{ env.BASE_IMAGE }} finished with status \
|
|
${{ job.status }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
package-release:
|
|
needs: [build]
|
|
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }}
|
|
|
|
concurrency:
|
|
group: ${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
outputs:
|
|
report: ${{ steps.report.outputs.report || 'Did not run.' }}
|
|
|
|
name: Package and artifacts release
|
|
runs-on: ubuntu-20.04
|
|
|
|
env:
|
|
RELEASE_IMAGE_BASE: ghcr.io/zama-ai/concretefhe
|
|
|
|
steps:
|
|
- uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
|
|
# To be removed once poetry 1.2 is released to manage dependencies with groups
|
|
# - name: Cache Installation Files
|
|
# uses: actions/cache@c64c572235d810460d0d6876e9c705ad5002b353
|
|
# with:
|
|
# # Paths are Unix specific for now
|
|
# path: |
|
|
# ~/.cache/pip
|
|
# ~/.cache/pypoetry
|
|
# # Use python 3.8 as it is the version available in ubuntu 20.04 and we develop with it
|
|
# key: "$${{ secrets.CACHE_VERSION }}-{{ runner.os }}-build-3.8-\
|
|
# ${{ hashFiles('poetry.lock') }}"
|
|
# restore-keys: |
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-build-3.8-
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-build-
|
|
# ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-
|
|
# ${{ secrets.CACHE_VERSION }}-
|
|
# See #570 To be updated to only install required dependencies group with poetry 1.2 and
|
|
# remove graphviz installs which are only required for the actual package and not dev tools
|
|
- name: Install dependencies
|
|
run: |
|
|
sudo apt-get install --no-install-recommends -y graphviz*
|
|
python -m pip install --upgrade pip
|
|
python -m pip install poetry
|
|
make setup_env
|
|
- name: Set tag in env
|
|
# 'poetry version' cannot be piped properly so do it in 2 steps
|
|
# the project version does not have the leading v to be semver compatible
|
|
run: |
|
|
PROJECT_VERSION=$(poetry version)
|
|
PROJECT_VERSION=$(echo "$PROJECT_VERSION" | cut -d ' ' -f 2)
|
|
GIT_TAG=$(echo "${{ github.ref }}" | sed 's/refs\/tags\///g')
|
|
|
|
if [[ "v${PROJECT_VERSION}" != "${GIT_TAG}" ]]; then
|
|
echo "Mismatch between tag and version: ${GIT_TAG}, v${PROJECT_VERSION}"
|
|
exit 1
|
|
fi
|
|
|
|
echo "PROJECT_VERSION=${PROJECT_VERSION}" >> "$GITHUB_ENV"
|
|
echo "GIT_TAG=${GIT_TAG}" >> "$GITHUB_ENV"
|
|
RELEASE_IMG_GIT_TAG="${RELEASE_IMAGE_BASE}:${GIT_TAG}"
|
|
echo "RELEASE_IMG_GIT_TAG=${RELEASE_IMG_GIT_TAG}" >> "$GITHUB_ENV"
|
|
RELEASE_IMG_TAGS_TO_PUSH="${RELEASE_IMG_GIT_TAG}"
|
|
|
|
EXISTING_TAGS=$(curl \
|
|
-X GET \
|
|
-H "Authorization: Bearer $(echo ${{ secrets.BOT_TOKEN }} | base64)" \
|
|
https://ghcr.io/v2/zama-ai/concretefhe/tags/list | jq -rc '.tags | join(" ")')
|
|
|
|
# We want the space separated list of versions to be expanded
|
|
# shellcheck disable=SC2086
|
|
IS_LATEST_INFO=$(poetry run python script/make_utils/version_utils.py \
|
|
islatest \
|
|
--new-version "${GIT_TAG}" \
|
|
--existing-versions $EXISTING_TAGS)
|
|
|
|
IS_LATEST=$(echo "${IS_LATEST_INFO}" | jq -rc '.is_latest')
|
|
echo "IS_LATEST=${IS_LATEST}" >> "$GITHUB_ENV"
|
|
IS_PRERELEASE=$(echo "${IS_LATEST_INFO}" | jq -rc '.is_prerelease')
|
|
echo "IS_PRERELEASE=${IS_PRERELEASE}" >> "$GITHUB_ENV"
|
|
|
|
if [[ "${IS_LATEST}" == "true" ]]; then
|
|
RELEASE_IMG_LATEST_TAG="${RELEASE_IMAGE_BASE}:latest"
|
|
RELEASE_IMG_TAGS_TO_PUSH="${RELEASE_IMG_TAGS_TO_PUSH},${RELEASE_IMG_LATEST_TAG}"
|
|
fi
|
|
|
|
echo "RELEASE_IMG_TAGS_TO_PUSH=${RELEASE_IMG_TAGS_TO_PUSH}" >> "$GITHUB_ENV"
|
|
# Disabled buildx for now as we are seeing a lot of fails on layer pushes
|
|
# - name: Set up Docker Buildx
|
|
# id: buildx
|
|
# uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25
|
|
- name: Login to GitHub Container Registry
|
|
uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
|
|
with:
|
|
registry: ghcr.io
|
|
username: ${{ secrets.BOT_USERNAME }}
|
|
password: ${{ secrets.BOT_TOKEN }}
|
|
- name: Build concretefhe Image
|
|
if: ${{ success() && !cancelled() }}
|
|
uses: docker/build-push-action@a66e35b9cbcf4ad0ea91ffcaf7bbad63ad9e0229
|
|
with:
|
|
context: .
|
|
# builder: ${{ steps.buildx.outputs.name }}
|
|
file: docker/Dockerfile.release
|
|
load: true
|
|
push: false
|
|
tags: "${{ env.RELEASE_IMG_TAGS_TO_PUSH }}"
|
|
no-cache: true
|
|
- name: Release image sanity check
|
|
if: ${{ success() && !cancelled() }}
|
|
run: |
|
|
echo "Running sanity check for ${RELEASE_IMG_GIT_TAG}"
|
|
docker run --rm -v "$(pwd)"/docker/release_resources:/data \
|
|
"${RELEASE_IMG_GIT_TAG}" /bin/bash -c "python ./sanity_check.py"
|
|
- name: Create directory for artifacts
|
|
if: ${{ success() && !cancelled() }}
|
|
run: |
|
|
ARTIFACTS_RAW_DIR=/tmp/release_artifacts/raw
|
|
mkdir -p "${ARTIFACTS_RAW_DIR}"
|
|
echo "ARTIFACTS_RAW_DIR=${ARTIFACTS_RAW_DIR}" >> "$GITHUB_ENV"
|
|
|
|
ARTIFACTS_PACKAGED_DIR=/tmp/release_artifacts/packaged
|
|
mkdir -p "${ARTIFACTS_PACKAGED_DIR}"
|
|
echo "ARTIFACTS_PACKAGED_DIR=${ARTIFACTS_PACKAGED_DIR}" >> "$GITHUB_ENV"
|
|
- name: Download Documentation
|
|
if: ${{ success() && !cancelled() }}
|
|
id: download-docs
|
|
uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
|
|
with:
|
|
name: html-docs
|
|
path: ${{ env.ARTIFACTS_RAW_DIR }}/html_docs/
|
|
- name: Download changelog
|
|
if: ${{ success() && !cancelled() }}
|
|
id: download-changelog
|
|
uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
|
|
with:
|
|
name: changelog
|
|
path: ${{ env.ARTIFACTS_RAW_DIR }}/changelog/
|
|
- name: Prepare docs push
|
|
id: docs-push-infos
|
|
run: |
|
|
if [[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }} != "" ]] && \
|
|
[[ ${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }} != "" ]] && \
|
|
[[ "${IS_PRERELEASE}" == "true" ]]; then
|
|
echo "::set-output name=aws-bucket::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_BUCKET_NAME }}"
|
|
echo "::set-output name=aws-distribution::${{ secrets.AWS_REPO_PREPROD_DOCUMENTATION_DISTRIBUTION_ID }}"
|
|
else
|
|
echo "::set-output name=aws-bucket::${{ secrets.AWS_REPO_DOCUMENTATION_BUCKET_NAME }}"
|
|
echo "::set-output name=aws-distribution::${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}"
|
|
fi
|
|
- name: Configure AWS credentials
|
|
uses: aws-actions/configure-aws-credentials@ea7b857d8a33dc2fb4ef5a724500044281b49a5e
|
|
with:
|
|
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
aws-region: ${{ secrets.AWS_REGION }}
|
|
- name: Update versions.json for docs
|
|
if: ${{ success() && !cancelled() }}
|
|
env:
|
|
RAW_DOCS_DIR: ${{ steps.download-docs.outputs.download-path }}
|
|
run: |
|
|
DOWNLOADED_VERSIONS_JSON_FILE=$(mktemp --suffix=.json)
|
|
OUTPUT_VERSIONS_JSON_FILE=$(mktemp --suffix=.json)
|
|
OPTS=""
|
|
if [[ $IS_LATEST = "true" ]]; then
|
|
OPTS="${OPTS} --latest "
|
|
fi
|
|
if [[ $IS_PRERELEASE = "true" ]]; then
|
|
OPTS="${OPTS} --prerelease "
|
|
fi
|
|
|
|
aws s3api get-object \
|
|
--bucket ${{ steps.docs-push-infos.outputs.aws-bucket }} \
|
|
--key concretefhe/versions.json "${DOWNLOADED_VERSIONS_JSON_FILE}"
|
|
|
|
poetry run python ./script/actions_utils/generate_versions_json.py \
|
|
--add-version "${PROJECT_VERSION}" \
|
|
--versions-json-file "${DOWNLOADED_VERSIONS_JSON_FILE}" \
|
|
--output-json "${OUTPUT_VERSIONS_JSON_FILE}" \
|
|
$OPTS
|
|
|
|
|
|
echo "OUTPUT_VERSIONS_JSON_FILE=${OUTPUT_VERSIONS_JSON_FILE}" >> "$GITHUB_ENV"
|
|
|
|
# Copy to docs to keep a version in docs artifacts
|
|
cp "${OUTPUT_VERSIONS_JSON_FILE}" "${RAW_DOCS_DIR}"/versions.json
|
|
- name: Create ready to upload/packaged artifacts and release body
|
|
if: ${{ success() && !cancelled() }}
|
|
env:
|
|
RAW_DOCS_DIR: ${{ steps.download-docs.outputs.download-path }}
|
|
RAW_CHANGELOG_DIR: ${{ steps.download-changelog.outputs.download-path }}
|
|
run: |
|
|
pushd "${RAW_DOCS_DIR}"
|
|
zip -r "${ARTIFACTS_PACKAGED_DIR}/html-docs.zip" ./*
|
|
tar -cvzf "${ARTIFACTS_PACKAGED_DIR}/html-docs.tar.gz" ./*
|
|
# Remove the versions.json to avoid pushing it to S3 but have it in release artifacts
|
|
rm versions.json
|
|
popd
|
|
cp "${RAW_CHANGELOG_DIR}"/* "${ARTIFACTS_PACKAGED_DIR}"
|
|
ls -a "${ARTIFACTS_PACKAGED_DIR}"
|
|
|
|
RELEASE_BODY_FILE=RELEASE_BODY.md
|
|
echo "RELEASE_BODY_FILE=${RELEASE_BODY_FILE}" >> "$GITHUB_ENV"
|
|
|
|
cp ./script/actions_utils/RELEASE_TEMPLATE.md "${RELEASE_BODY_FILE}"
|
|
{
|
|
echo "Docker Image: ${RELEASE_IMG_GIT_TAG}";
|
|
echo "Documentation: https://${{ steps.docs-push-infos.outputs.aws-bucket }}/concretefhe/${PROJECT_VERSION}";
|
|
echo "";
|
|
} >> "${RELEASE_BODY_FILE}"
|
|
cat "${RAW_CHANGELOG_DIR}"/* >> "${RELEASE_BODY_FILE}"
|
|
- name: Push release docker image
|
|
if: ${{ success() && !cancelled() }}
|
|
run: |
|
|
docker image push --all-tags "${RELEASE_IMAGE_BASE}"
|
|
- name: Push release documentation
|
|
if: ${{ success() && !cancelled() }}
|
|
env:
|
|
AWS_S3_BUCKET: ${{ steps.docs-push-infos.outputs.aws-bucket }}
|
|
SOURCE_DIR: ${{ steps.download-docs.outputs.download-path }}
|
|
DEST_DIR: 'concretefhe/${{ env.PROJECT_VERSION }}'
|
|
run: |
|
|
aws s3 sync "${SOURCE_DIR}" s3://"${AWS_S3_BUCKET}/${DEST_DIR}" --delete --acl public-read
|
|
|
|
- name: Push release documentation as stable
|
|
if: ${{ success() && !cancelled() && !fromJSON(env.IS_PRERELEASE) && fromJSON(env.IS_LATEST) }}
|
|
env:
|
|
AWS_S3_BUCKET: ${{ steps.docs-push-infos.outputs.aws-bucket }}
|
|
SOURCE_DIR: ${{ steps.download-docs.outputs.download-path }}
|
|
DEST_DIR: 'concretefhe/stable'
|
|
run: |
|
|
aws s3 sync "${SOURCE_DIR}" s3://"${AWS_S3_BUCKET}/${DEST_DIR}" --delete --acl public-read
|
|
- name: Invalidate CloudFront Cache for stable
|
|
if: ${{ success() && !fromJSON(env.IS_PRERELEASE) && fromJSON(env.IS_LATEST) }}
|
|
env:
|
|
SOURCE_PATH: "/concretefhe/stable/*"
|
|
DISTRIBUTION_ID: ${{ steps.docs-push-infos.outputs.aws-distribution }}
|
|
run: |
|
|
aws cloudfront create-invalidation \
|
|
--distribution-id "${DISTRIBUTION_ID}" \
|
|
--paths "${SOURCE_PATH}"
|
|
- name: Create GitHub release
|
|
if: ${{ success() && !cancelled() }}
|
|
id: create-release
|
|
uses: softprops/action-gh-release@1e07f4398721186383de40550babbdf2b84acfc5
|
|
with:
|
|
body_path: ${{ env.RELEASE_BODY_FILE }}
|
|
prerelease: ${{ fromJSON(env.IS_PRERELEASE) }}
|
|
files: |
|
|
${{ env.ARTIFACTS_PACKAGED_DIR }}/*
|
|
tag_name: ${{ env.GIT_TAG }}
|
|
fail_on_unmatched_files: true
|
|
token: ${{ secrets.BOT_TOKEN }}
|
|
- name: Push updated versions.html
|
|
if: ${{ success() }}
|
|
run: |
|
|
aws s3 cp "${OUTPUT_VERSIONS_HTML_FILE}" \
|
|
s3://${{ steps.docs-push-infos.outputs.aws-bucket }}/concretefhe/versions.html \
|
|
--acl public-read
|
|
|
|
aws cloudfront create-invalidation \
|
|
--distribution-id ${{ steps.docs-push-infos.outputs.aws-distribution }} \
|
|
--paths /concretefhe/versions.html
|
|
- name: Set notification report
|
|
id: report
|
|
if: ${{ always() }}
|
|
run: |
|
|
REPORT="Creating release for ${GIT_TAG} finished with status ${{ job.status }}. \
|
|
GitHub release link: ${{ steps.create-release.outputs.url }}."
|
|
echo "${REPORT}"
|
|
echo "::set-output name=report::${REPORT}"
|
|
echo "REPORT=${REPORT}" >> "$GITHUB_ENV"
|
|
- name: Slack Notification
|
|
if: ${{ always() && !success() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ job.status }}
|
|
SLACK_MESSAGE: "${{ env.REPORT }} (${{ env.ACTION_RUN_URL }})"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|
|
|
|
send-report:
|
|
if: ${{ always() }}
|
|
needs:
|
|
[
|
|
build-preflight-docker,
|
|
build,
|
|
publish-docs,
|
|
push-docker-image,
|
|
package-release,
|
|
]
|
|
|
|
name: Send Slack notification
|
|
runs-on: ubuntu-20.04
|
|
steps:
|
|
- uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579
|
|
- name: Prepare whole job status
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
env:
|
|
NEEDS_JSON: ${{ toJSON(needs) }}
|
|
run: |
|
|
echo "${NEEDS_JSON}" > /tmp/needs_context.json
|
|
JOB_STATUS=$(python3 ./script/actions_utils/actions_combine_status.py \
|
|
--needs_context_json /tmp/needs_context.json)
|
|
echo "JOB_STATUS=${JOB_STATUS}" >> "$GITHUB_ENV"
|
|
|
|
- name: Slack Notification
|
|
if: ${{ always() }}
|
|
continue-on-error: true
|
|
uses: rtCamp/action-slack-notify@12e36fc18b0689399306c2e0b3e0f2978b7f1ee7
|
|
env:
|
|
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
|
|
SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png
|
|
SLACK_COLOR: ${{ env.JOB_STATUS || 'failure' }}
|
|
SLACK_MESSAGE: "Full run finished with status ${{ env.JOB_STATUS || 'failure' }} \
|
|
(${{ env.ACTION_RUN_URL }})\n\
|
|
- build-preflight-docker: ${{ needs.build-preflight-docker.outputs.report || 'Did not run.' }}\n\n\
|
|
- build: ${{ needs.build.outputs.report || 'Did not run.' }}\n\n\
|
|
- publish-docs: ${{ needs.publish-docs.outputs.report || 'Did not run.' }}\n\n\
|
|
- push-docker-image: ${{ needs.push-docker-image.outputs.report || 'Did not run.' }}\n\n\
|
|
- package-release: ${{ needs.package-release.outputs.report || 'Did not run.' }}"
|
|
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
|
|
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
|