mirror of
https://github.com/zama-ai/concrete.git
synced 2026-02-09 03:55:04 -05:00
e3e1602fe8
default is now 3.11 which we don't support yet, mainly due to our version of pybind11. 2.10.1 is the first pybind11 version to supports it
770 lines
28 KiB
YAML
770 lines
28 KiB
YAML
name: Continuous Integration Pipeline
|
|
|
|
on:
|
|
push:
|
|
branches: [main, test-ci]
|
|
tags:
|
|
- 'v*'
|
|
pull_request:
|
|
types: [opened, synchronize, reopened]
|
|
|
|
# DOCKER_IMAGE variables aren't used in BuildAndPushDockerImages because of https://github.com/actions/runner/issues/480
|
|
env:
|
|
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
|
|
|
|
jobs:
|
|
########################
|
|
# Tests and formating #
|
|
########################
|
|
|
|
FormattingAndLinting:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
- name: Format with clang-format (Cpp)
|
|
run: sudo apt install moreutils && .github/workflows/scripts/format_cpp.sh
|
|
- name: Format with black (Python)
|
|
run: |
|
|
cd compiler
|
|
pip install -r lib/Bindings/Python/requirements_dev.txt
|
|
make check-python-format
|
|
- name: Lint with pylint (Python)
|
|
run: |
|
|
cd compiler
|
|
# compiler requirements to lint
|
|
pip install numpy
|
|
make python-lint
|
|
- name: Linelint
|
|
uses: fernandrone/linelint@0.0.4
|
|
id: linelint
|
|
|
|
CheckLicense:
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
- name: Check if sources include the license header
|
|
run: .github/workflows/scripts/check_for_license.sh
|
|
|
|
BuildAndTest:
|
|
runs-on: ubuntu-latest
|
|
# We want to always run the tests unless cancelled, but after the docker image build job.
|
|
# If there is no new build then we use the old image, else we use the new image.
|
|
if: ${{ !cancelled() }}
|
|
needs: [BuildAndPushDockerImages]
|
|
steps:
|
|
# Free 4Gb of workspace
|
|
- name: Freeing space
|
|
run: |
|
|
df -h
|
|
for image in ubuntu:{16,18}.04 \
|
|
node:{12,14,16}{-alpine,} \
|
|
buildpack-deps:{stretch,buster,bullseye} \
|
|
debian:{9,10,11} alpine:3.{12,13,14} \
|
|
moby/buildkit:latest docker:20.10
|
|
do
|
|
docker image rm $image || echo Please clean remove it from this step
|
|
done
|
|
df -h
|
|
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.5.2
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Concrete-Optimizer
|
|
run: |
|
|
cd compiler
|
|
make concrete-optimizer-lib
|
|
|
|
- name: Download KeySetCache
|
|
if: ${{ !contains(github.head_ref, 'newkeysetcache') }}
|
|
continue-on-error: true
|
|
run: |
|
|
cd compiler
|
|
GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated
|
|
|
|
- name: Mark KeySetCache
|
|
run: |
|
|
touch keysetcache.timestamp
|
|
|
|
- name: Build and test compiler
|
|
uses: addnab/docker-run-action@v3
|
|
id: build-compiler
|
|
with:
|
|
registry: ghcr.io
|
|
image: ${{ env.DOCKER_IMAGE_TEST }}
|
|
username: ${{ secrets.GHCR_LOGIN }}
|
|
password: ${{ secrets.GHCR_PASSWORD }}
|
|
options: >-
|
|
-v ${{ github.workspace }}/llvm-project:/llvm-project
|
|
-v ${{ github.workspace }}/compiler:/compiler
|
|
-v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache
|
|
shell: bash
|
|
run: |
|
|
set -e
|
|
cd /compiler
|
|
rm -rf /build
|
|
pip install pytest
|
|
sed "s/pytest/python -m pytest/g" -i Makefile
|
|
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all run-tests run-end-to-end-dataflow-tests run-rust-tests
|
|
echo "Debug: ccache statistics (after the build):"
|
|
ccache -s
|
|
chmod -R ugo+rwx /tmp/KeySetCache
|
|
|
|
- name: Prune KeySetCache
|
|
run: |
|
|
TO_CLEAN=$(find KeySetCache/* -maxdepth 1 -mindepth 1 -not -newer keysetcache.timestamp -type d)
|
|
if [ -n "${TO_CLEAN}" ]
|
|
then
|
|
echo "Cleaning ${TO_CLEAN}"
|
|
rm -rf "${TO_CLEAN}"
|
|
echo New cache size is
|
|
du -sh KeySetCache
|
|
else
|
|
echo Nothing to clean
|
|
fi
|
|
|
|
- name: Upload KeySetCache
|
|
if: ${{ github.ref == 'refs/heads/main' }}
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: KeySetCacheV2
|
|
path: KeySetCache
|
|
retention-days: 90
|
|
|
|
- name: Cleanup Old KeySetCache
|
|
uses: Remagpie/gha-remove-artifact@v1
|
|
if: ${{ github.ref == 'refs/heads/main' }}
|
|
with:
|
|
only-name: KeySetCacheV2
|
|
max-count: 1
|
|
|
|
- name: Build the documentation
|
|
id: build-doc
|
|
if: ${{ steps.build-compiler.outcome == 'success' && !cancelled() }}
|
|
uses: addnab/docker-run-action@v3
|
|
with:
|
|
registry: ghcr.io
|
|
image: ${{ env.DOCKER_IMAGE_TEST }}
|
|
username: ${{ secrets.GHCR_LOGIN }}
|
|
password: ${{ secrets.GHCR_PASSWORD }}
|
|
options: >-
|
|
-v ${{ github.workspace }}/compiler:/compiler
|
|
-v ${{ github.workspace }}/llvm-project:/llvm-project
|
|
-v ${{ github.workspace }}/docs:/docs
|
|
shell: bash
|
|
run: |
|
|
set -e
|
|
rm -rf /build
|
|
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build concretecompiler python-bindings doc
|
|
cd /docs
|
|
pip install -r requirements.txt
|
|
pip install -r ../llvm-project/mlir/python/requirements.txt
|
|
dnf install -y doxygen
|
|
sed "s/sphinx-apidoc/python -m sphinx.ext.apidoc/g" -i Makefile
|
|
sed "s/sphinx-build/python -m sphinx.cmd.build/g" -i Makefile
|
|
make COMPILER_BUILD_DIR=/build/ doc
|
|
|
|
- name: Archive docs artifacts
|
|
if: ${{ steps.build-doc.outcome == 'success' && !cancelled() }}
|
|
uses: actions/upload-artifact@v3
|
|
with:
|
|
name: html-docs
|
|
path: docs/_build/html
|
|
|
|
BuildAndTestMacOS:
|
|
runs-on: macos-11
|
|
steps:
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.6.0
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
# MacOS now comes with 3.11 which we don't yet support because of pybind11 (2.10.1 supports it)
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: '3.10'
|
|
|
|
- name: Install Deps
|
|
run: |
|
|
brew install ninja ccache
|
|
pip3 install numpy pybind11==2.8 wheel delocate
|
|
pip3 install pytest
|
|
|
|
- name: Cache compilation (push)
|
|
if: github.event_name == 'push'
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: /Users/runner/Library/Caches/ccache
|
|
key: ${{ runner.os }}-compilation-cache-${{ github.sha }}
|
|
restore-keys: |
|
|
${{ runner.os }}-compilation-cache-
|
|
|
|
- name: Cache compilation (pull_request)
|
|
if: github.event_name == 'pull_request'
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: /Users/runner/Library/Caches/ccache
|
|
key: ${{ runner.os }}-compilation-cache-${{ github.event.pull_request.base.sha }}
|
|
restore-keys: |
|
|
${{ runner.os }}-compilation-cache-
|
|
|
|
- name: Get tmpdir path
|
|
if: github.event_name == 'push'
|
|
id: tmpdir-path
|
|
run: echo "::set-output name=TMPDIR_PATH::$TMPDIR"
|
|
|
|
# We do run run-check-tests as part of the build, as they aren't that costly
|
|
# and will at least give minimum confidence that the compiler works in PRs
|
|
- name: Build
|
|
run: |
|
|
cd compiler
|
|
echo "Debug: ccache statistics (prior to the build):"
|
|
ccache -s
|
|
make Python3_EXECUTABLE=$(which python) all run-check-tests
|
|
echo "Debug: ccache statistics (after the build):"
|
|
ccache -s
|
|
|
|
- name: Download KeySetCache
|
|
continue-on-error: true
|
|
if: ${{ github.event_name == 'push' && !contains(github.head_ref, 'newkeysetcache') }}
|
|
run: |
|
|
cd compiler
|
|
KEYSETCACHECI="$TMPDIR/KeySetCache" GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} make keysetcache_ci_populated
|
|
|
|
- name: Test
|
|
if: github.event_name == 'push'
|
|
run: |
|
|
cd compiler
|
|
echo "Debug: ccache statistics (prior to the tests):"
|
|
ccache -s
|
|
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
|
|
pip3 wheel --no-deps -w ${{ github.workspace }}/wheels .
|
|
delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
|
|
pip3 install $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
|
|
make run-tests
|
|
echo "Debug: ccache statistics (after the tests):"
|
|
ccache -s
|
|
|
|
BlockMerge:
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Check Commit to Squash
|
|
run: |
|
|
set -e
|
|
git log origin/${{ github.base_ref }}..origin/${{ github.head_ref }} --format=%s | ( ! grep -e "^f [0-9a-f]\+" -q )
|
|
|
|
##################################
|
|
# Releasing and Testing Packages #
|
|
##################################
|
|
|
|
PublishDoc:
|
|
needs: [BuildAndTest]
|
|
|
|
runs-on: ubuntu-20.04
|
|
if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') }}
|
|
|
|
steps:
|
|
- name: Set env
|
|
id: vars
|
|
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> "${GITHUB_ENV}"
|
|
- name: Download Documentation
|
|
id: download
|
|
uses: actions/download-artifact@v3
|
|
with:
|
|
name: html-docs
|
|
- name: Publish Documentation to S3
|
|
id: publish
|
|
if: ${{ steps.download.outcome == 'success' && !cancelled() }}
|
|
uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83
|
|
with:
|
|
args: --acl public-read
|
|
env:
|
|
AWS_S3_BUCKET: ${{ secrets.AWS_PREPROD_REPO_DOCUMENTATION_BUCKET_NAME }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
AWS_REGION: ${{ secrets.AWS_REGION }}
|
|
SOURCE_DIR: '.'
|
|
DEST_DIR: 'concrete-compiler/${{ env.RELEASE_VERSION }}'
|
|
- name: Invalidate CloudFront Cache
|
|
if: ${{ steps.publish.outcome == 'success' }}
|
|
uses: awact/cloudfront-action@8bcfabc7b4bbc0cb8e55e48527f0e3a6d681627c
|
|
env:
|
|
SOURCE_PATH: '/concrete-compiler/*'
|
|
AWS_REGION: ${{ secrets.AWS_REGION }}
|
|
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
|
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
|
DISTRIBUTION_ID: ${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}
|
|
|
|
CreateRelease:
|
|
runs-on: ubuntu-latest
|
|
needs: [BuildAndTest, BuildAndTestMacOS]
|
|
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
|
outputs:
|
|
upload_url: ${{ steps.release.outputs.upload_url }}
|
|
release_id: ${{ steps.release.outputs.id }}
|
|
steps:
|
|
- name: Release
|
|
id: release
|
|
uses: softprops/action-gh-release@v1
|
|
with:
|
|
token: ${{ secrets.GH_TOKEN_RELEASE }}
|
|
draft: true
|
|
prerelease: true
|
|
generate_release_notes: true
|
|
|
|
BuildAndPushPythonPackagesLinux:
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python: 37
|
|
python_dir: "cp37-cp37m"
|
|
- python: 38
|
|
python_dir: "cp38-cp38"
|
|
- python: 39
|
|
python_dir: "cp39-cp39"
|
|
- python: 310
|
|
python_dir: "cp310-cp310"
|
|
outputs:
|
|
python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }}
|
|
python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }}
|
|
python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }}
|
|
python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }}
|
|
needs: CreateRelease
|
|
steps:
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.5.2
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install Rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Concrete-Optimizer
|
|
run: |
|
|
cd compiler
|
|
make concrete-optimizer-lib
|
|
|
|
- name: Update Python Version
|
|
run: cd compiler && make update-python-version
|
|
|
|
- name: Login to Github Container Registry
|
|
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
|
|
|
- name: Build Wheel
|
|
uses: addnab/docker-run-action@v3
|
|
with:
|
|
registry: ghcr.io
|
|
image: ${{ env.DOCKER_IMAGE_TEST }}
|
|
username: ${{ secrets.GHCR_LOGIN }}
|
|
password: ${{ secrets.GHCR_PASSWORD }}
|
|
options: >-
|
|
-v ${{ github.workspace }}/llvm-project:/llvm-project
|
|
-v ${{ github.workspace }}/compiler:/compiler
|
|
-v ${{ github.workspace }}/wheels:/wheels
|
|
shell: bash
|
|
run: |
|
|
set -e
|
|
cd /compiler
|
|
rm -rf /build
|
|
export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python
|
|
$PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt
|
|
# setup env variable for wheel building
|
|
export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC
|
|
export CONCRETE_COMPILER_BUILD_DIR=/build
|
|
/opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels .
|
|
# We need to run it twice: the first will generate the directories, so that
|
|
# the second run can find the packages via find_namespace_packages
|
|
/opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels .
|
|
auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels
|
|
echo "Debug: ccache statistics (after the build):"
|
|
ccache -s
|
|
|
|
- name: Set Outputs
|
|
id: set-output-wheel-linux
|
|
run: |
|
|
echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )"
|
|
# used later for python package test
|
|
echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )"
|
|
|
|
- name: Upload Python Package
|
|
uses: actions/upload-release-asset@v1
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
|
with:
|
|
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
|
asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
|
|
asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
|
|
asset_content_type: application/zip
|
|
|
|
BuildAndPushTarballLinux:
|
|
runs-on: ubuntu-latest
|
|
needs: CreateRelease
|
|
steps:
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.6.0
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install Rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Concrete-Optimizer
|
|
run: |
|
|
cd compiler
|
|
make concrete-optimizer-lib
|
|
|
|
- name: Login to Github Container Registry
|
|
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
|
|
|
- name: Build
|
|
id: build-tarball
|
|
run: |
|
|
cd compiler
|
|
make release-tarballs
|
|
|
|
TAG="$(git describe --tags --abbrev=0)"
|
|
|
|
sudo cp "${{ github.workspace }}/tarballs/concretecompiler.tar.gz" "${{ github.workspace }}/tarballs/concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz"
|
|
echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz"
|
|
|
|
- name: Upload Tarball
|
|
uses: actions/upload-release-asset@v1
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
|
with:
|
|
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
|
asset_path: ${{ github.workspace }}/tarballs/${{ steps.build-tarball.outputs.ASSET_NAME }}
|
|
asset_name: ${{ steps.build-tarball.outputs.ASSET_NAME }}
|
|
asset_content_type: application/tar+gzip
|
|
|
|
BuildAndPushPackagesMacOS:
|
|
needs: CreateRelease
|
|
runs-on: macos-11
|
|
strategy:
|
|
matrix:
|
|
python: ['3.8', '3.9', '3.10']
|
|
outputs:
|
|
python-package-name-macos-py38: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY38 }}
|
|
python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }}
|
|
python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }}
|
|
steps:
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.5.0
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install Rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Concrete-Optimizer
|
|
run: |
|
|
cd compiler
|
|
make concrete-optimizer-lib
|
|
|
|
- name: Install rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
|
|
- name: Install Deps
|
|
run: |
|
|
brew install ninja ccache
|
|
pip install numpy pybind11==2.8 wheel delocate
|
|
|
|
- name: Update Python Version
|
|
run: cd compiler && make update-python-version
|
|
|
|
- name: Use Compilation Cache
|
|
uses: actions/cache@v3
|
|
with:
|
|
path: /Users/runner/Library/Caches/ccache
|
|
key: ${{ runner.os }}-compilation-cache-${{ github.sha }}
|
|
restore-keys: |
|
|
${{ runner.os }}-compilation-cache-
|
|
|
|
- name: Build
|
|
id: build-wheel-macos
|
|
run: |
|
|
cd compiler
|
|
make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings
|
|
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
|
|
pip wheel --no-deps -w ${{ github.workspace }}/wheels .
|
|
delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
|
|
echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )"
|
|
# used later for python package test
|
|
echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )"
|
|
|
|
- name: Upload Python Package
|
|
uses: actions/upload-release-asset@v1
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
|
with:
|
|
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
|
asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
|
|
asset_name: ${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
|
|
asset_content_type: application/zip
|
|
|
|
- name: Build tarball
|
|
if: matrix.python == '3.8'
|
|
id: build-mac-tarball
|
|
run: |
|
|
cd compiler
|
|
make concretecompiler
|
|
mkdir -p tarballs/concretecompiler/lib tarballs/concretecompiler/bin
|
|
cp build/bin/concretecompiler tarballs/concretecompiler/bin
|
|
cp build/lib/libConcretelangRuntime.dylib tarballs/concretecompiler/lib
|
|
cp ../.github/workflows/assets/Installation.md tarballs/concretecompiler/
|
|
|
|
TAG=$(git describe --tags --abbrev=0)
|
|
|
|
cd tarballs && tar -czvf "concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" concretecompiler
|
|
echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz"
|
|
|
|
- name: Upload Tarball
|
|
if: matrix.python == '3.8'
|
|
uses: actions/upload-release-asset@v1
|
|
env:
|
|
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
|
with:
|
|
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
|
asset_path: ${{ github.workspace }}/compiler/tarballs/${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
|
|
asset_name: ${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
|
|
asset_content_type: application/tar+gzip
|
|
|
|
TestPythonPackageLinux:
|
|
runs-on: ubuntu-latest
|
|
needs: [BuildAndPushPythonPackagesLinux, CreateRelease]
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python: '3.7'
|
|
filename-index: 'python-package-name-linux-py37'
|
|
- python: '3.8'
|
|
filename-index: 'python-package-name-linux-py38'
|
|
- python: '3.9'
|
|
filename-index: 'python-package-name-linux-py39'
|
|
- python: '3.10'
|
|
filename-index: 'python-package-name-linux-py310'
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
|
|
- name: Extract Package Filename
|
|
id: extract-filename
|
|
run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPythonPackagesLinux.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )"
|
|
|
|
- name: Download and Install Package
|
|
run: |
|
|
FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \
|
|
https://api.github.com/repos/${{ github.repository }}/releases | \
|
|
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
|
|
jq '.[0].assets' | \
|
|
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
|
|
jq '.[].id')
|
|
|
|
wget --auth-no-challenge --header='Accept:application/octet-stream' \
|
|
"https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \
|
|
-O ${{ steps.extract-filename.outputs.FILE_NAME }}
|
|
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
|
|
|
|
- name: Test
|
|
run: |
|
|
cd compiler
|
|
pip install pytest
|
|
pytest -vs tests/python
|
|
|
|
TestPythonPackageMacOS:
|
|
runs-on: macos-11
|
|
needs: [BuildAndPushPackagesMacOS, CreateRelease]
|
|
env:
|
|
SYSTEM_VERSION_COMPAT: 0
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- python: '3.8'
|
|
filename-index: 'python-package-name-macos-py38'
|
|
- python: '3.9'
|
|
filename-index: 'python-package-name-macos-py39'
|
|
- python: '3.10'
|
|
filename-index: 'python-package-name-macos-py310'
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v4
|
|
with:
|
|
python-version: ${{ matrix.python }}
|
|
|
|
- name: Extract Package Filename
|
|
id: extract-filename
|
|
run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPackagesMacOS.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )"
|
|
|
|
- name: Download and Install Package
|
|
run: |
|
|
FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \
|
|
https://api.github.com/repos/${{ github.repository }}/releases | \
|
|
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
|
|
jq '.[0].assets' | \
|
|
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
|
|
jq '.[].id')
|
|
|
|
wget --auth-no-challenge --header='Accept:application/octet-stream' \
|
|
"https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \
|
|
-O ${{ steps.extract-filename.outputs.FILE_NAME }}
|
|
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
|
|
|
|
- name: Test
|
|
run: |
|
|
cd compiler
|
|
pip install pytest
|
|
pytest -vs -m "not parallel" tests/python
|
|
|
|
#################
|
|
# Docker Images #
|
|
#################
|
|
|
|
BuildAndPushDockerImages:
|
|
if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') || needs.BuildAndPublishHPXDockerImage.outputs.image_built == 'true' }}
|
|
needs: [BuildAndPublishHPXDockerImage]
|
|
name: Build & Publish Docker Images
|
|
runs-on: ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
include:
|
|
- name: test-env
|
|
image: ghcr.io/zama-ai/concrete-compiler
|
|
dockerfile: builders/Dockerfile.concrete-compiler-env
|
|
|
|
steps:
|
|
# A SSH private key is required as some dependencies are from private repos
|
|
- uses: webfactory/ssh-agent@v0.6.0
|
|
with:
|
|
ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }}
|
|
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
submodules: recursive
|
|
token: ${{ secrets.GH_TOKEN }}
|
|
|
|
- name: Install rust
|
|
uses: actions-rs/toolchain@v1
|
|
with:
|
|
toolchain: stable
|
|
|
|
- name: Concrete-Optimizer
|
|
run: |
|
|
cd compiler
|
|
make concrete-optimizer-lib
|
|
|
|
- name: Login to Registry
|
|
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
|
|
|
# label was initially a need from the frontend CI
|
|
- name: Build Image
|
|
run: docker image build --no-cache --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} .
|
|
|
|
- name: Tag and Publish Image
|
|
run: |
|
|
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.sha }}
|
|
docker image push ${{ matrix.image }}:latest
|
|
docker image push ${{ matrix.image }}:${{ github.sha }}
|
|
|
|
- name: Tag and Publish Release Image
|
|
if: startsWith(github.ref, 'refs/tags/v')
|
|
run: |
|
|
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.ref_name }}
|
|
docker image push ${{ matrix.image }}:${{ github.ref_name }}
|
|
|
|
BuildAndPublishHPXDockerImage:
|
|
name: Build & Publish HPX Docker Image
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
IMAGE: ghcr.io/zama-ai/hpx
|
|
outputs:
|
|
image_built: ${{ steps.is-built.outputs.IMAGE_BUILT }}
|
|
|
|
steps:
|
|
- uses: actions/checkout@v3
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Get changed files
|
|
id: changed-files
|
|
uses: tj-actions/changed-files@v32
|
|
|
|
- name: Login
|
|
id: login
|
|
if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.hpx-env')
|
|
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
|
|
|
- name: Build Tag and Publish
|
|
if: ${{ steps.login.conclusion != 'skipped' }}
|
|
run: |
|
|
docker build -t "${IMAGE}" -f builders/Dockerfile.hpx-env .
|
|
docker push "${IMAGE}:latest"
|
|
|
|
- name: Is Image Built
|
|
id: is-built
|
|
run: echo "::set-output name=IMAGE_BUILT::${{ contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.hpx-env') }}"
|