mirror of
https://github.com/zama-ai/concrete.git
synced 2026-02-09 12:15:09 -05:00
771 lines
29 KiB
YAML
771 lines
29 KiB
YAML
name: Continuous Integration Pipeline
|
||
|
||
on:
|
||
push:
|
||
branches: [master, test-ci]
|
||
tags:
|
||
- 'v*'
|
||
pull_request:
|
||
types: [opened, synchronize, reopened]
|
||
|
||
# DOCKER_IMAGE variables aren't used in BuildAndPushDockerImages because of https://github.com/actions/runner/issues/480
|
||
env:
|
||
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
|
||
DOCKER_IMAGE_TEST_GCC7: ghcr.io/zama-ai/concrete-compiler-gcc7
|
||
DOCKER_IMAGE_TEST_DF: ghcr.io/zama-ai/concrete-compiler-df
|
||
|
||
jobs:
|
||
########################
|
||
# Tests and formatiing #
|
||
########################
|
||
|
||
Formatting:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: true
|
||
- name: Format with clang-format
|
||
run: sudo apt install moreutils && .github/workflows/scripts/format_cpp.sh
|
||
|
||
CheckLicense:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
- name: Check if sources include the license header
|
||
run: .github/workflows/scripts/check_for_license.sh
|
||
|
||
BuildAndTest:
|
||
runs-on: ubuntu-latest
|
||
strategy:
|
||
matrix:
|
||
compiler: [gcc7, latest]
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- name: "KeySetCache"
|
||
if: ${{ matrix.compiler == 'gcc7' }}
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: ${{ github.workspace }}/KeySetCache
|
||
# actions/cache does not permit to update a cache entry
|
||
key: Linux-KeySetCache-2022-02-28
|
||
|
||
- name: Build and test compiler
|
||
if: ${{ matrix.compiler == 'gcc7' }}
|
||
id: build-compiler
|
||
uses: addnab/docker-run-action@v3
|
||
with:
|
||
registry: ghcr.io
|
||
image: ${{ env.DOCKER_IMAGE_TEST_GCC7 }}
|
||
username: ${{ secrets.GHCR_LOGIN }}
|
||
password: ${{ secrets.GHCR_PASSWORD }}
|
||
options: -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache
|
||
shell: bash
|
||
run: |
|
||
set -e
|
||
echo "Debug: ccache statistics (prior to the build):"
|
||
ccache -s
|
||
cd /compiler
|
||
rm -rf /build
|
||
/opt/python/cp38-cp38/bin/pip install pytest
|
||
sed "s/pytest/\/opt\/python\/cp38-cp38\/bin\/python -m pytest/g" -i Makefile
|
||
make CXX_COMPILER=/gcc7/bin/g++-7.5.0 CC_COMPILER=/gcc7/bin/gcc-7.5.0 CCACHE=ON Python3_EXECUTABLE=/opt/python/cp38-cp38/bin/python BUILD_DIR=/build test doc
|
||
echo "Debug: ccache statistics (after the build):"
|
||
ccache -s
|
||
chmod -R ugo+rwx /tmp/KeySetCache
|
||
|
||
- name: Build compiler
|
||
if: ${{ matrix.compiler == 'latest' }}
|
||
id: build-compiler-latest
|
||
uses: addnab/docker-run-action@v3
|
||
with:
|
||
registry: ghcr.io
|
||
image: ${{ env.DOCKER_IMAGE_TEST }}
|
||
username: ${{ secrets.GHCR_LOGIN }}
|
||
password: ${{ secrets.GHCR_PASSWORD }}
|
||
options: -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/build:/build
|
||
shell: bash
|
||
run: |
|
||
set -e
|
||
echo "Debug: ccache statistics (prior to the build):"
|
||
ccache -s
|
||
cd /compiler
|
||
export PYTHONPATH=""
|
||
make CCACHE=ON BUILD_DIR=/build concretecompiler python-bindings doc
|
||
echo "Debug: ccache statistics (after the build):"
|
||
ccache -s
|
||
|
||
- name: Build the documentation
|
||
id: build-doc
|
||
if: ${{ steps.build-compiler-latest.outcome == 'success' && !cancelled() }}
|
||
uses: addnab/docker-run-action@v3
|
||
with:
|
||
registry: ghcr.io
|
||
image: ${{ env.DOCKER_IMAGE_TEST }}
|
||
username: ${{ secrets.GHCR_LOGIN }}
|
||
password: ${{ secrets.GHCR_PASSWORD }}
|
||
options: -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/docs:/docs -v ${{ github.workspace }}/build:/compiler/build
|
||
shell: bash
|
||
run: |
|
||
set -e
|
||
make CCACHE=ON BUILD_DIR=/build concretecompiler python-bindings doc
|
||
cd /docs
|
||
pip install -r requirements.txt
|
||
pip install -r ../llvm-project/mlir/python/requirements.txt
|
||
DEBIAN_FRONTEND="noninteractive" apt-get install -y doxygen
|
||
make CCACHE=ON doc
|
||
|
||
- name: Archive docs artifacts
|
||
if: ${{ steps.build-doc.outcome == 'success' && !cancelled() }}
|
||
uses: actions/upload-artifact@27121b0bdffd731efa15d66772be8dc71245d074
|
||
with:
|
||
name: html-docs
|
||
path: docs/_build/html
|
||
|
||
BuildAndTestMacOS:
|
||
runs-on: macos-10.15
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
repository: zama-ai/concrete_internal
|
||
ref: engine_c_api
|
||
path: concrete
|
||
token: ${{ secrets.GH_TOKEN }}
|
||
|
||
- name: Get ConcreteLib commit hash
|
||
id: concretelib-hash
|
||
run: cd ${{ github.workspace }}/concrete && echo "::set-output name=COMMIT_SHA::`git rev-parse HEAD`"
|
||
|
||
- name: Cache ConcreteLib
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: ${{ github.workspace }}/concrete/target
|
||
key: ${{ runner.os }}-concrete-lib-${{ steps.concretelib-hash.outputs.COMMIT_SHA }}
|
||
|
||
- name: Install Deps
|
||
run: |
|
||
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
||
brew install ninja ccache
|
||
pip3 install numpy pybind11==2.6.2 wheel delocate
|
||
pip3 install pytest
|
||
cd ${{ github.workspace }}/concrete/concrete-ffi
|
||
RUSTFLAGS="-C target-cpu=native" cargo build --release
|
||
|
||
- name: Cache compilation (push)
|
||
if: github.event_name == 'push'
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: /Users/runner/Library/Caches/ccache
|
||
key: ${{ runner.os }}-compilation-cache-${{ github.sha }}
|
||
|
||
- name: Cache compilation (pull_request)
|
||
if: github.event_name == 'pull_request'
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: /Users/runner/Library/Caches/ccache
|
||
key: ${{ runner.os }}-compilation-cache-${{ github.event.pull_request.base.sha }}
|
||
|
||
- name: Get tmpdir path
|
||
if: github.event_name == 'push'
|
||
id: tmpdir-path
|
||
run: echo "::set-output name=TMPDIR_PATH::`echo $TMPDIR`"
|
||
|
||
- name: KeySetCache
|
||
if: github.event_name == 'push'
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: ${{ steps.tmpdir-path.outputs.TMPDIR_PATH }}/KeySetCache
|
||
# The cache is not specific to Linux and can be used on MacOS
|
||
key: Linux-KeySetCache-2022-02-28
|
||
|
||
# We do run test-check as part of the build, as they aren't that costly
|
||
# and will at least give minimum confidence that the compiler works in PRs
|
||
- name: Build
|
||
run: |
|
||
cd compiler
|
||
echo "Debug: ccache statistics (prior to the build):"
|
||
ccache -s
|
||
export CONCRETE_PROJECT=${{ github.workspace }}/concrete
|
||
make python-bindings build-tests test-check
|
||
echo "Debug: ccache statistics (after the build):"
|
||
ccache -s
|
||
|
||
- name: Test
|
||
if: github.event_name == 'push'
|
||
run: |
|
||
cd compiler
|
||
echo "Debug: ccache statistics (prior to the tests):"
|
||
ccache -s
|
||
export CONCRETE_PROJECT=${{ github.workspace }}/concrete
|
||
make install_runtime_lib
|
||
make test
|
||
echo "Debug: ccache statistics (after the tests):"
|
||
ccache -s
|
||
|
||
BuildAndTestDF:
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- name: "KeySetCache"
|
||
uses: actions/cache@v2
|
||
with:
|
||
path: ${{ github.workspace }}/KeySetCache
|
||
# actions/cache does not permit to update a cache entry
|
||
key: Linux-KeySetCache-2022-02-28
|
||
|
||
- name: Build and test compiler (dataflow)
|
||
uses: addnab/docker-run-action@v3
|
||
with:
|
||
registry: ghcr.io
|
||
image: ${{ env.DOCKER_IMAGE_TEST_DF }}
|
||
username: ${{ secrets.GHCR_LOGIN }}
|
||
password: ${{ secrets.GHCR_PASSWORD }}
|
||
options: -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/KeySetCache:/tmp/KeySetCache
|
||
shell: bash
|
||
run: |
|
||
set -e
|
||
echo "Debug: ccache statistics (prior to the build):"
|
||
ccache -s
|
||
cd /compiler
|
||
pip install pytest
|
||
rm -rf /build
|
||
export PYTHONPATH=""
|
||
make PARALLEL_EXECUTION_ENABLED=ON CCACHE=ON BUILD_DIR=/build test test-dataflow
|
||
echo "Debug: ccache statistics (after the build):"
|
||
ccache -s
|
||
chmod -R ugo+rwx /tmp/KeySetCache
|
||
|
||
BlockMerge:
|
||
if: github.event_name == 'pull_request'
|
||
runs-on: ubuntu-latest
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Check Commit to Squash
|
||
run: |
|
||
set -e
|
||
git log origin/${{ github.base_ref }}..origin/${{ github.head_ref }} --format=%s | ( ! grep -e "^f [0-9a-f]\+" -q )
|
||
|
||
##################################
|
||
# Releasing and Testing Packages #
|
||
##################################
|
||
|
||
PublishDoc:
|
||
needs: [BuildAndTest]
|
||
|
||
runs-on: ubuntu-20.04
|
||
if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }}
|
||
|
||
steps:
|
||
- name: Set env
|
||
id: vars
|
||
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
|
||
- name: Download Documentation
|
||
id: download
|
||
uses: actions/download-artifact@3be87be14a055c47b01d3bd88f8fe02320a9bb60
|
||
with:
|
||
name: html-docs
|
||
- name: Publish Documentation to S3
|
||
id: publish
|
||
if: ${{ steps.download.outcome == 'success' && !cancelled() }}
|
||
uses: jakejarvis/s3-sync-action@be0c4ab89158cac4278689ebedd8407dd5f35a83
|
||
with:
|
||
args: --acl public-read
|
||
env:
|
||
AWS_S3_BUCKET: ${{ secrets.AWS_PREPROD_REPO_DOCUMENTATION_BUCKET_NAME }}
|
||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||
SOURCE_DIR: '.'
|
||
DEST_DIR: 'concrete-compiler/${{ env.RELEASE_VERSION }}'
|
||
- name: Invalidate CloudFront Cache
|
||
if: ${{ steps.publish.outcome == 'success' }}
|
||
uses: awact/cloudfront-action@8bcfabc7b4bbc0cb8e55e48527f0e3a6d681627c
|
||
env:
|
||
SOURCE_PATH: '/concrete-compiler/*'
|
||
AWS_REGION: ${{ secrets.AWS_REGION }}
|
||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||
DISTRIBUTION_ID: ${{ secrets.AWS_REPO_DOCUMENTATION_DISTRIBUTION_ID }}
|
||
|
||
CreateRelease:
|
||
runs-on: ubuntu-latest
|
||
needs: [BuildAndTest, BuildAndTestDF, BuildAndTestMacOS]
|
||
if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
|
||
outputs:
|
||
upload_url: ${{ steps.release.outputs.upload_url }}
|
||
release_id: ${{ steps.release.outputs.id }}
|
||
steps:
|
||
- name: Release
|
||
id: release
|
||
uses: softprops/action-gh-release@v1
|
||
with:
|
||
token: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
draft: true
|
||
prerelease: true
|
||
generate_release_notes: true
|
||
|
||
BuildAndPushPythonPackagesLinux:
|
||
runs-on: ubuntu-latest
|
||
strategy:
|
||
matrix:
|
||
python: [38, 39, 310]
|
||
outputs:
|
||
python-package-name-linux-py38: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY38 }}
|
||
python-package-name-linux-py39: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY39 }}
|
||
python-package-name-linux-py310: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY310 }}
|
||
needs: CreateRelease
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- name: Login to Github Container Registry
|
||
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
||
|
||
- name: Build
|
||
id: build-wheel-linux
|
||
run: |
|
||
cd compiler
|
||
make package_py${{ matrix.python }}
|
||
echo "::set-output name=ASSET_NAME::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
# used later for python package test
|
||
echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
|
||
- name: Upload Python Package
|
||
uses: actions/upload-release-asset@v1
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
with:
|
||
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
||
asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
|
||
asset_name: ${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
|
||
asset_content_type: application/zip
|
||
|
||
BuildAndPushPythonPackagesLinuxWithParallelization:
|
||
runs-on: ubuntu-latest
|
||
needs: CreateRelease
|
||
strategy:
|
||
matrix:
|
||
python: ['3.8', '3.9', '3.10']
|
||
outputs:
|
||
python-package-name-linux-py38: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY38 }}
|
||
python-package-name-linux-py39: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY39 }}
|
||
python-package-name-linux-py310: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY310 }}
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
repository: zama-ai/concrete_internal
|
||
ref: engine_c_api
|
||
path: concrete
|
||
token: ${{ secrets.GH_TOKEN }}
|
||
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
repository: STEllAR-GROUP/hpx.git
|
||
ref: 1.7.1
|
||
path: hpx
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python }}
|
||
|
||
- name: Build Concrete
|
||
run: |
|
||
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
||
cd ${{ github.workspace }}/concrete/concrete-ffi
|
||
RUSTFLAGS="-C target-cpu=native" cargo build --release
|
||
|
||
- name: Build HPX
|
||
run: |
|
||
sudo apt install -y libboost-filesystem-dev libhwloc-dev
|
||
cd ${{ github.workspace }}/hpx
|
||
mkdir build
|
||
cd build
|
||
cmake \
|
||
-DHPX_WITH_FETCH_ASIO=on \
|
||
-DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \
|
||
-DHPX_WITH_MALLOC=system ..
|
||
make
|
||
|
||
- name: Update Python Version
|
||
run: cd compiler && make update_python_version
|
||
|
||
- name: Build Parallel Python Package
|
||
id: build-wheel-linux
|
||
run: |
|
||
set -e
|
||
export HPX=${{ github.workspace }}/hpx
|
||
export HPX_INSTALL_DIR=$HPX/build
|
||
export CONCRETE_PROJECT=${{ github.workspace }}/concrete
|
||
sudo apt install -y ninja-build
|
||
pip install numpy pybind11==2.6.2 wheel auditwheel patchelf
|
||
cd compiler
|
||
mkdir build
|
||
make PARALLEL_EXECUTION_ENABLED=ON python-bindings
|
||
pip wheel --no-deps -w ./wheels .
|
||
auditwheel repair ./wheels/*.whl --plat linux_x86_64 -w ./wheels/
|
||
echo "::set-output name=ASSET_NAME::`find ./wheels/ -name *linux_x86_64*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
# used later for python package test
|
||
echo "::set-output name=ASSET_NAME_PY`echo ${{ matrix.python }} |tr -d '.'`::`find ./wheels/ -name *linux_x86_64*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
|
||
- name: Upload Python Package
|
||
uses: actions/upload-release-asset@v1
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
with:
|
||
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
||
asset_path: ${{ github.workspace }}/compiler/wheels/${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
|
||
asset_name: ${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
|
||
asset_content_type: application/zip
|
||
|
||
BuildAndPushTarballLinux:
|
||
runs-on: ubuntu-latest
|
||
needs: CreateRelease
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- name: Login to Github Container Registry
|
||
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
||
|
||
- name: Build
|
||
id: build-tarball
|
||
run: |
|
||
cd compiler
|
||
make release_tarballs
|
||
sudo cp ${{ github.workspace }}/tarballs/concretecompiler.tar.gz ${{ github.workspace }}/tarballs/concretecompiler-`git describe --tags --abbrev=0`-x86_64-linux-gnu.tar.gz
|
||
echo "::set-output name=ASSET_NAME::concretecompiler-`git describe --tags --abbrev=0`-x86_64-linux-gnu.tar.gz"
|
||
|
||
- name: Upload Tarball
|
||
uses: actions/upload-release-asset@v1
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
with:
|
||
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
||
asset_path: ${{ github.workspace }}/tarballs/${{ steps.build-tarball.outputs.ASSET_NAME }}
|
||
asset_name: ${{ steps.build-tarball.outputs.ASSET_NAME }}
|
||
asset_content_type: application/tar+gzip
|
||
|
||
BuildAndPushPackagesMacOS:
|
||
needs: CreateRelease
|
||
runs-on: macos-10.15
|
||
strategy:
|
||
matrix:
|
||
python: ['3.8', '3.9', '3.10']
|
||
outputs:
|
||
python-package-name-macos-py38: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY38 }}
|
||
python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }}
|
||
python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }}
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
repository: zama-ai/concrete_internal
|
||
ref: engine_c_api
|
||
path: concrete
|
||
token: ${{ secrets.GH_TOKEN }}
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python }}
|
||
|
||
- name: Install Deps
|
||
run: |
|
||
curl https://sh.rustup.rs -sSf | sh -s -- -y
|
||
brew install ninja
|
||
pip install numpy pybind11==2.6.2 wheel delocate
|
||
cd ${{ github.workspace }}/concrete/concrete-ffi
|
||
RUSTFLAGS="-C target-cpu=native" cargo build --release
|
||
|
||
- name: Update Python Version
|
||
run: cd compiler && make update_python_version
|
||
|
||
- name: Build
|
||
id: build-wheel-macos
|
||
run: |
|
||
cd compiler
|
||
export CONCRETE_PROJECT=${{ github.workspace }}/concrete
|
||
make Python3_EXECUTABLE=`which python` python-bindings
|
||
pip wheel --no-deps -w ${{ github.workspace }}/wheels .
|
||
delocate-wheel -v `find ${{ github.workspace }}/wheels/ -name *macosx*.whl`
|
||
echo "::set-output name=ASSET_NAME::`find ${{ github.workspace }}/wheels/ -name *macosx*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
# used later for python package test
|
||
echo "::set-output name=ASSET_NAME_PY`echo ${{ matrix.python }} |tr -d '.'`::`find ${{ github.workspace }}/wheels/ -name *macosx*.whl | rev |cut -d "/" -f 1 |rev `"
|
||
|
||
- name: Upload Python Package
|
||
uses: actions/upload-release-asset@v1
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
with:
|
||
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
||
asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
|
||
asset_name: ${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
|
||
asset_content_type: application/zip
|
||
|
||
- name: Build tarball
|
||
if: matrix.python == '3.8'
|
||
id: build-mac-tarball
|
||
run: |
|
||
cd compiler
|
||
export CONCRETE_PROJECT=${{ github.workspace }}/concrete
|
||
make concretecompiler
|
||
mkdir -p tarballs/concretecompiler/lib tarballs/concretecompiler/bin
|
||
cp build/bin/concretecompiler tarballs/concretecompiler/bin
|
||
cp build/lib/libConcretelangRuntime.dylib tarballs/concretecompiler/lib
|
||
cp ../.github/workflows/assets/Installation.md tarballs/concretecompiler/
|
||
cd tarballs && tar -czvf concretecompiler-`git describe --tags --abbrev=0`-x86_64-macos-catalina.tar.gz concretecompiler
|
||
echo "::set-output name=ASSET_NAME::concretecompiler-`git describe --tags --abbrev=0`-x86_64-macos-catalina.tar.gz"
|
||
|
||
- name: Upload Tarball
|
||
if: matrix.python == '3.8'
|
||
uses: actions/upload-release-asset@v1
|
||
env:
|
||
GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
|
||
with:
|
||
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
|
||
asset_path: ${{ github.workspace }}/compiler/tarballs/${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
|
||
asset_name: ${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
|
||
asset_content_type: application/tar+gzip
|
||
|
||
TestPythonPackageLinux:
|
||
runs-on: ubuntu-latest
|
||
needs: [BuildAndPushPythonPackagesLinux, CreateRelease]
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- python: '3.8'
|
||
filename-index: 'python-package-name-linux-py38'
|
||
- python: '3.9'
|
||
filename-index: 'python-package-name-linux-py39'
|
||
- python: '3.10'
|
||
filename-index: 'python-package-name-linux-py310'
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python }}
|
||
|
||
- name: Extract Package Filename
|
||
id: extract-filename
|
||
run: echo "::set-output name=FILE_NAME::`echo '${{ toJson(needs.BuildAndPushPythonPackagesLinux.outputs) }}' | jq '.[\"${{ matrix.filename-index }}\"]' | tr -d '\"' `"
|
||
|
||
- name: Download and Install Package
|
||
run: |
|
||
wget --auth-no-challenge --header='Accept:application/octet-stream' \
|
||
"https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/`curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \
|
||
https://api.github.com/repos/${{ github.repository }}/releases | \
|
||
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
|
||
jq '.[0].assets' | \
|
||
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
|
||
jq '.[].id'`" -O ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
|
||
- name: Test
|
||
run: |
|
||
cd compiler
|
||
pip install pytest
|
||
pytest -vs -m "not parallel" --ignore=tests/python/test_compiler_file_output/ tests/python
|
||
|
||
TestPythonPackageLinuxWithParallelization:
|
||
runs-on: ubuntu-latest
|
||
needs: [BuildAndPushPythonPackagesLinuxWithParallelization, CreateRelease]
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- python: '3.8'
|
||
filename-index: 'python-package-name-linux-py38'
|
||
- python: '3.9'
|
||
filename-index: 'python-package-name-linux-py39'
|
||
- python: '3.10'
|
||
filename-index: 'python-package-name-linux-py310'
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python }}
|
||
|
||
- name: Extract Package Filename
|
||
id: extract-filename
|
||
run: echo "::set-output name=FILE_NAME::`echo '${{ toJson(needs.BuildAndPushPythonPackagesLinuxWithParallelization.outputs) }}' | jq '.[\"${{ matrix.filename-index }}\"]' | tr -d '\"' `"
|
||
|
||
- name: Download and Install Package
|
||
run: |
|
||
wget --auth-no-challenge --header='Accept:application/octet-stream' \
|
||
"https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/`curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \
|
||
https://api.github.com/repos/${{ github.repository }}/releases | \
|
||
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
|
||
jq '.[0].assets' | \
|
||
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
|
||
jq '.[].id'`" -O ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
|
||
- name: Test
|
||
run: |
|
||
cd compiler
|
||
pip install pytest
|
||
pytest -vs --ignore=tests/python/test_compiler_file_output/ tests/python
|
||
|
||
TestPythonPackageMacOS:
|
||
runs-on: macos-10.15
|
||
needs: [BuildAndPushPackagesMacOS, CreateRelease]
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- python: '3.8'
|
||
filename-index: 'python-package-name-macos-py38'
|
||
- python: '3.9'
|
||
filename-index: 'python-package-name-macos-py39'
|
||
- python: '3.10'
|
||
filename-index: 'python-package-name-macos-py310'
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
|
||
- name: Set up Python
|
||
uses: actions/setup-python@v2
|
||
with:
|
||
python-version: ${{ matrix.python }}
|
||
|
||
- name: Extract Package Filename
|
||
id: extract-filename
|
||
run: echo "::set-output name=FILE_NAME::`echo '${{ toJson(needs.BuildAndPushPackagesMacOS.outputs) }}' | jq '.[\"${{ matrix.filename-index }}\"]' | tr -d '\"' `"
|
||
|
||
- name: Download and Install Package
|
||
run: |
|
||
wget --auth-no-challenge --header='Accept:application/octet-stream' \
|
||
"https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/`curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \
|
||
https://api.github.com/repos/${{ github.repository }}/releases | \
|
||
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
|
||
jq '.[0].assets' | \
|
||
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
|
||
jq '.[].id'`" -O ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
|
||
|
||
- name: Test
|
||
run: |
|
||
cd compiler
|
||
pip install pytest
|
||
pytest -vs -m "not parallel" --ignore=tests/python/test_compiler_file_output/ tests/python
|
||
|
||
#################
|
||
# Docker Images #
|
||
#################
|
||
|
||
BuildAndPushDockerImages:
|
||
if: ${{ github.event_name == 'push' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/master') }}
|
||
needs: [BuildAndTest]
|
||
name: Build & Publish Docker Images
|
||
runs-on: ubuntu-latest
|
||
strategy:
|
||
matrix:
|
||
include:
|
||
- name: test-env
|
||
image: ghcr.io/zama-ai/concrete-compiler
|
||
dockerfile: builders/Dockerfile.concrete-compiler-env
|
||
- name: test-env-gcc7
|
||
image: ghcr.io/zama-ai/concrete-compiler-gcc7
|
||
dockerfile: builders/Dockerfile.concrete-compiler-gcc7-env
|
||
- name: test-df
|
||
image: ghcr.io/zama-ai/concrete-compiler-df
|
||
dockerfile: builders/Dockerfile.concrete-compiler-df-env
|
||
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
submodules: recursive
|
||
|
||
- name: Login to Registry
|
||
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
||
|
||
# label was initially a need from the frontend CI
|
||
- name: Build Image
|
||
run: docker image build --no-cache --label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} .
|
||
|
||
- name: Tag and Publish Image
|
||
run: |
|
||
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.sha }}
|
||
docker image push ${{ matrix.image }}:latest
|
||
docker image push ${{ matrix.image }}:${{ github.sha }}
|
||
|
||
- name: Tag and Publish Release Image
|
||
if: startsWith(github.ref, 'refs/tags/v')
|
||
run: |
|
||
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.ref_name }}
|
||
docker image push ${{ matrix.image }}:${{ github.ref_name }}
|
||
|
||
BuildAndPublishHPXDockerImage:
|
||
name: Build & Publish HPX Docker Image
|
||
runs-on: ubuntu-latest
|
||
env:
|
||
IMAGE: ghcr.io/zama-ai/hpx
|
||
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Get changed files
|
||
id: changed-files
|
||
uses: tj-actions/changed-files@v2.0.0
|
||
|
||
- name: Login
|
||
if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.hpx-env')
|
||
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
||
|
||
- name: Build Tag and Publish
|
||
if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.hpx-env')
|
||
run: |
|
||
docker build -t $IMAGE -f builders/Dockerfile.hpx-env .
|
||
docker push $IMAGE:latest
|
||
|
||
BuildAndPublishGCC7DockerImage:
|
||
name: Build & Publish GCC7 Docker Image
|
||
runs-on: ubuntu-latest
|
||
env:
|
||
IMAGE: ghcr.io/zama-ai/gcc7
|
||
|
||
steps:
|
||
- uses: actions/checkout@v2
|
||
with:
|
||
fetch-depth: 0
|
||
|
||
- name: Get changed files
|
||
id: changed-files
|
||
uses: tj-actions/changed-files@v2.0.0
|
||
|
||
- name: Login
|
||
if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.gcc7-env')
|
||
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
|
||
|
||
- name: Build Tag and Publish
|
||
if: contains(steps.changed-files.outputs.modified_files, 'builders/Dockerfile.gcc7-env')
|
||
run: |
|
||
docker build -t $IMAGE -f builders/Dockerfile.gcc7-env .
|
||
docker push $IMAGE:latest
|