github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-01-10 05:18:00 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(ci): Refactoring CI workflows

Browse Source
This commit is contained in:
Bourgerie Quentin
2024-10-18 18:35:32 +02:00
committed by Quentin Bourgerie
parent bd9332ba61
commit 84cb1b5b73
45 changed files with 1699 additions and 2101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
.github/workflows/action-pin.yaml vendored
View File

@@ -1,19 +0,0 @@
name: Action Pin
on:
pull_request:
push:
branches:
- main
jobs:
check-action-pin:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Ensure SHA pinned actions
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12
with:
allowlist: |
slsa-framework/slsa-github-generator

32
.github/workflows/action_compliance.yaml vendored Normal file
View File

@@ -0,0 +1,32 @@
name: check action compliance
on:
pull_request:
paths:
- .github/workflows/**
push:
branches:
- main
- 'release/*'
jobs:
action-pin:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Ensure SHA pinned actions
uses: zgosalvez/github-actions-ensure-sha-pinned-actions@0901cf7b71c7ea6261ec69a3dc2bd3f9264f893e # v3.0.12
with:
allowlist: |
slsa-framework/slsa-github-generator
action-lint:
runs-on: ubuntu-latest
steps:
- name: checkout
uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: check-missing-teardown
run: .github/workflows/scripts/teardown-check.sh
- name: actionlint
uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0

16
.github/workflows/actionlint.yml vendored
View File

@@ -1,16 +0,0 @@
name: Action Lint
on:
pull_request:
push:
branches:
- main
jobs:
action-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: actionlint
uses: raven-actions/actionlint@01fce4f43a270a612932cb1c64d40505a029f821 # v2.0.0

11
.github/workflows/check_commit_signature.yml vendored
View File

@@ -1,11 +0,0 @@
name: Check Commit Signatures
on:
pull_request:
jobs:
check-commit-signatures:
runs-on: ubuntu-latest
steps:
- name: Check commit signatures
uses: 1Password/check-signed-commits-action@ed2885f3ed2577a4f5d3c3fe895432a557d23d52

19
.github/workflows/block_merge.yml → .github/workflows/commit_compliance.yml vendored
View File

@@ -1,13 +1,10 @@
# Check commit and PR compliance
name: Check commit message compliance
name: check commit compliance
on:
pull_request:
types: [opened, synchronize, reopened]
jobs:
check-commit-pr:
name: Check commit and PR
runs-on: ubuntu-20.04
format:
runs-on: ubuntu-latest
steps:
- name: Check first line
uses: gsactions/commit-message-checker@16fa2d5de096ae0d35626443bcd24f1e756cafee # v2.0.0
@@ -19,3 +16,13 @@ jobs:
excludeTitle: 'true' # optional: this excludes the title of a pull request
checkAllCommitMessages: 'true' # optional: this checks all commits associated with a pull request
accessToken: ${{ secrets.GITHUB_TOKEN }} # github access token is only required if checkAllCommitMessages is true
- name: checkout
uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
- name: linelint
uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4
id: linelint
- name: markdown-link-check
uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1
with:
use-quiet-mode: 'yes'
use-verbose-mode: 'yes'

142
.github/workflows/compiler_benchmark.yml vendored
View File

@@ -1,142 +0,0 @@
# Run benchmarks on an AWS instance for compiler and return parsed results to Slab CI bot.
name: Compiler - Performance benchmarks
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
# concurrency:
# group: ${{ github.workflow }}-${{ github.ref }}
# cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
CARGO_TERM_COLOR: always
RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json
CUDA_PATH: /usr/local/cuda-11.8
GCC_VERSION: 8
jobs:
run-benchmarks:
name: Execute end-to-end benchmarks in EC2
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
steps:
- name: Instance configuration used
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
- name: Get benchmark date
run: |
echo "BENCH_DATE=$(date --iso-8601=seconds)" >> "${GITHUB_ENV}"
- name: Fetch submodules
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Set up home
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Export specific variables (CPU)
if: ${{ !startswith(inputs.instance_type, 'p3.') }}
run: |
echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}"
echo "BENCHMARK_TARGET=run-cpu-benchmarks" >> "${GITHUB_ENV}"
- name: Export specific variables (GPU)
if: ${{ startswith(inputs.instance_type, 'p3.') }}
run: |
echo "CUDA_SUPPORT=ON" >> "${GITHUB_ENV}"
echo "BENCHMARK_TARGET=run-gpu-benchmarks" >> "${GITHUB_ENV}"
echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}"
echo "$CUDA_PATH/bin" >> "${GITHUB_PATH}"
echo "LD_LIBRARY_PATH=$CUDA_PATH/lib:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}"
echo "CC=/usr/bin/gcc-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
echo "CXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
echo "CUDAHOSTCXX=/usr/bin/g++-${{ env.GCC_VERSION }}" >> "${GITHUB_ENV}"
echo "CUDACXX=$CUDA_PATH/bin/nvcc" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Build compiler benchmarks
run: |
set -e
git config --global --add safe.directory '*'
cd compilers/concrete-compiler/compiler
make BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-benchmarks
- name: Run end-to-end benchmarks
run: |
set -e
cd compilers/concrete-compiler/compiler
make ${{ env.BENCHMARK_TARGET }}
- name: Upload raw results artifact
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: compiler_${{ github.sha }}_raw
path: compilers/concrete-compiler/compiler/benchmarks_results.json
- name: Parse results
shell: bash
run: |
COMMIT_DATE="$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})"
COMMIT_HASH="$(git describe --tags --dirty)"
python3 ./ci/benchmark_parser.py compilers/concrete-compiler/compiler/benchmarks_results.json ${{ env.RESULTS_FILENAME }} \
--database compiler_benchmarks \
--hardware ${{ inputs.instance_type }} \
--project-version ${COMMIT_HASH} \
--branch ${{ github.ref_name }} \
--commit-date ${COMMIT_DATE} \
--bench-date "${{ env.BENCH_DATE }}" \
--throughput
- name: Upload parsed results artifact
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: compiler_${{ github.sha }}
path: ${{ env.RESULTS_FILENAME }}
- name: Checkout Slab repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: zama-ai/slab
path: slab
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Send data to Slab
shell: bash
run: |
echo "Computing HMac on downloaded artifact"
SIGNATURE="$(slab/scripts/hmac_calculator.sh ${{ env.RESULTS_FILENAME }} '${{ secrets.JOB_SECRET }}')"
echo "Sending results to Slab..."
curl -v -k \
-H "Content-Type: application/json" \
-H "X-Slab-Repository: ${{ github.repository }}" \
-H "X-Slab-Command: store_data" \
-H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-d @${{ env.RESULTS_FILENAME }} \
${{ secrets.SLAB_URL }}

158
.github/workflows/compiler_build_and_test_cpu.yml vendored
View File

@@ -1,158 +0,0 @@
name: Compiler - Build and Test (CPU)
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
matrix_item:
description: 'Build matrix item'
type: string
# concurrency:
# group: compiler_build_and_test_cpu-${{ github.ref }}
# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
jobs:
BuildAndTest:
name: Build and test compiler in EC2
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
steps:
- name: Instance configuration used
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "Matrix item: ${{ inputs.matrix_item }}"
- name: Set up env
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
#echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Create build dir
run: mkdir build
- name: Build compiler
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
rustup toolchain install nightly-2024-09-30
pip install mypy
set -e
cd /concrete/compilers/concrete-compiler/compiler
rm -rf /build/*
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Build compiler Dialects docs and check diff
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-docs
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
set -e
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build doc
cd /build/tools/concretelang/docs/concretelang/
sed -i -e 's/\[TOC\]//' *Dialect.md
for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done;
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
- name: Test compiler
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
set -e
rustup toolchain install nightly-2024-09-30
cd /concrete/compilers/concrete-compiler/compiler
pip install pytest
pip install mypy
dnf install -y libzstd libzstd-devel
sed "s/pytest/python -m pytest/g" -i Makefile
mkdir -p /tmp/concrete_compiler/gpu_tests/
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests
chmod -R ugo+rwx /tmp/KeySetCache
- name: Analyze logs
run: |
cd build/gtest-parallel-logs/passed
ls -1 | xargs grep -H "WARNING RETRY" | sed -e "s/.log.*//g" | uniq -c | sed -re "s/ *([0-9]*) (.*)/::warning ::Test \2 retried \1 times/g" | cat
# - name: Archive python package
# uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
# with:
# name: concrete-compiler.whl
# path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl
# retention-days: 14

89
.github/workflows/compiler_build_and_test_cpu_distributed.yml vendored
View File

@@ -1,89 +0,0 @@
name: Compiler - Distributed Build and Test (CPU)
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
matrix_item:
description: 'Build matrix item'
type: string
env:
GLIB_VER: 2_28
jobs:
BuildAndTest:
name: Build and test compiler on Slurm cluster in EC2
runs-on: distributed-ci
if: ${{ !cancelled() }}
steps:
- name: Instance configuration used
run: |
echo "ID: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "Matrix item: ${{ inputs.matrix_item }}"
- name: Instance cleanup
run: |
sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete
mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete
docker system prune -af
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Set up home
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/shared" >> "${GITHUB_ENV}"
- name: Export specific variables (CPU)
if: ${{ !startswith(inputs.instance_type, 'p3.') }}
run: |
echo "CUDA_SUPPORT=OFF" >> "${GITHUB_ENV}"
echo "DATAFLOW_EXECUTION_ENABLED=ON" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Build compiler benchmarks
run: |
set -e
git config --global --add safe.directory '*'
cd compilers/concrete-compiler/compiler
rm -rf /shared/build
make HPX_DIR=/shared/hpx install-hpx-from-source
make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF CUDA_SUPPORT=${{ env.CUDA_SUPPORT }} build-end-to-end-tests
- name: Run end-to-end benchmarks
run: |
set -e
cd compilers/concrete-compiler/compiler
rm -rf /shared/KeyCache
make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests
- name: Instance cleanup
run: |
sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete/*
docker system prune -af

91
.github/workflows/compiler_build_and_test_gpu.yml vendored
View File

@@ -1,91 +0,0 @@
name: Compiler - Build and Test (GPU)
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
matrix_item:
description: 'Build matrix item'
type: string
# concurrency:
# group: compiler_build_and_test_gpu-${{ github.ref }}
# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
CUDA_PATH: /usr/local/cuda-11.8
GCC_VERSION: 11
jobs:
BuildAndTest:
name: Build and test compiler in EC2 with CUDA support
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
steps:
- name: Instance configuration used
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "Matrix item: ${{ inputs.matrix_item }}"
- name: Set up env
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Create build dir
run: mkdir build
- name: Build and test compiler
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
--gpus all
shell: bash
run: |
rustup toolchain install nightly-2024-09-30
pip install mypy
set -e
cd /concrete/compilers/concrete-compiler/compiler
rm -rf /build/*
mkdir -p /tmp/concrete_compiler/gpu_tests/
make BINDINGS_PYTHON_ENABLED=OFF CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu
echo "Debug: ccache statistics (after the build):"
ccache -s

39
.github/workflows/compiler_format_and_linting.yml vendored
View File

@@ -1,39 +0,0 @@
name: Compiler - Compliance
on:
workflow_call:
workflow_dispatch:
jobs:
FormattingAndLinting:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Format with clang-format (Cpp)
run: |
sudo apt install moreutils
cd compilers/concrete-compiler/compiler
./scripts/format_cpp.sh
- name: Format with cmake-format (Cmake)
run: |
pip3 install cmakelang
cd compilers/concrete-compiler/compiler
./scripts/format_cmake.sh
- name: Format with black (Python)
run: |
cd compilers/concrete-compiler/compiler
pip install -r lib/Bindings/Python/requirements_dev.txt
make check-python-format
- name: Lint with pylint (Python)
run: |
cd compilers/concrete-compiler/compiler
# compiler requirements to lint
pip install numpy
make python-lint
CheckLicense:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Check if sources include the license header
run: .github/workflows/scripts/check_for_license.sh

104
.github/workflows/compiler_macos_build_and_test.yml vendored
View File

@@ -1,104 +0,0 @@
# Perform a build on MacOS platform with M1 chip.
name: Compiler - Build and Test (MacOS)
on:
workflow_call:
workflow_dispatch:
secrets:
CONCRETE_CI_SSH_PRIVATE:
required: true
CONCRETE_ACTIONS_TOKEN:
required: true
concurrency:
group: compiler_macos_build_and_test-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
BuildAndTestMacOS:
strategy:
# if a failure happens, we want to know if it's specific
# to the architecture or the operating system
fail-fast: false
matrix:
runson: ["aws-mac1-metal", "aws-mac2-metal"]
runs-on: ${{ matrix.runson }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Install Deps
run: |
brew install ninja ccache
pip3.10 install numpy pybind11==2.8 wheel delocate
pip3.10 install pytest
pip3.10 install mypy
- name: Cache compilation (push)
if: github.event_name == 'push'
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: /Users/runner/Library/Caches/ccache
key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-compilation-cache-
- name: Cache compilation (pull_request)
if: github.event_name == 'pull_request'
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
path: /Users/runner/Library/Caches/ccache
key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-compilation-cache-
- name: Get tmpdir path
if: github.event_name == 'push'
id: tmpdir-path
run: echo "::set-output name=TMPDIR_PATH::$TMPDIR"
# We do run run-check-tests as part of the build, as they aren't that costly
# and will at least give minimum confidence that the compiler works in PRs
- name: Build
run: |
set -e
cd compilers/concrete-compiler/compiler
echo "Debug: ccache statistics (prior to the build):"
ccache -s
make Python3_EXECUTABLE=$(which python3.10) all run-check-tests python-package
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
- name: Test
run: |
set -e
export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}"
mkdir $KEY_CACHE_DIRECTORY
cd compilers/concrete-compiler/compiler
echo "Debug: ccache statistics (prior to the tests):"
ccache -s
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
pip3.10 install build/wheels/*macosx*.whl
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$(which python3.10) run-tests
echo "Debug: ccache statistics (after the tests):"
ccache -s
- name: Cleanup host
if: success() || failure()
run: |
rm -rf $KEY_CACHE_DIRECTORY

191
.github/workflows/compiler_publish_docker_images.yml vendored
View File

@@ -1,191 +0,0 @@
# Build and publish Docker images for different applications using AWS EC2.
name: Compiler - Docker images build & publish
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
matrix_item:
description: 'Build matrix item'
type: string
# concurrency:
# group: compiler_publish_docker_images-${{ github.ref }}
# cancel-in-progress: true
env:
THIS_FILE: .github/workflows/compiler_publish_docker_images.yml
jobs:
BuildAndPushDockerImages:
needs: [BuildAndPublishHPXDockerImage, BuildAndPublishCUDADockerImage]
name: Build & Publish Docker Images
runs-on: ${{ github.event.inputs.runner_name }}
strategy:
matrix:
include:
- name: test-env
image: ghcr.io/zama-ai/concrete-compiler
dockerfile: docker/Dockerfile.concrete-compiler-env
steps:
- name: Instance configuration used
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Login to Registry
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
# label was initially a need from the frontend CI
- name: Build Image
run: |
DOCKER_BUILDKIT=1 docker build --no-cache \
--label "commit-sha=${{ github.sha }}" -t ${{ matrix.image }} -f ${{ matrix.dockerfile }} .
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ matrix.image }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Tag and Publish Image
run: |
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.sha }}
docker image push ${{ matrix.image }}:latest
docker image push ${{ matrix.image }}:${{ github.sha }}
- name: Tag and Publish Release Image
if: startsWith(github.ref, 'refs/tags/v')
run: |
docker image tag ${{ matrix.image }} ${{ matrix.image }}:${{ github.ref_name }}
docker image push ${{ matrix.image }}:${{ github.ref_name }}
BuildAndPublishHPXDockerImage:
name: Build & Publish HPX Docker Image
runs-on: ${{ github.event.inputs.runner_name }}
env:
IMAGE: ghcr.io/zama-ai/hpx
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Set up env
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
- name: Login
id: login
if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.hpx-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build
if: ${{ steps.login.conclusion != 'skipped' }}
run: docker build -t "${IMAGE}" -f docker/Dockerfile.hpx-env .
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# if: ${{ steps.login.conclusion != 'skipped' }}
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.IMAGE }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Publish
if: ${{ steps.login.conclusion != 'skipped' }}
run: docker push "${IMAGE}:latest"
BuildAndPublishCUDADockerImage:
name: Build & Publish CUDA Docker Image
runs-on: ${{ github.event.inputs.runner_name }}
env:
IMAGE: ghcr.io/zama-ai/cuda
strategy:
matrix:
include:
- name: cuda-12-3
tag: 12-3
dockerfile: docker/Dockerfile.cuda-123-env
- name: cuda-11-8
tag: 11-8
dockerfile: docker/Dockerfile.cuda-118-env
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Set up env
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
- name: Login
id: login
# from the docs: The jobs.<job_id>.if condition is evaluated before jobs.<job_id>.strategy.matrix is applied. So we can't just use matrix.dockerfile
# so we have to build both images if one of the two files change, or we will have to split this into two
# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif
if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-118-env') || contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-123-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build Tag and Publish
if: ${{ steps.login.conclusion != 'skipped' }}
run: |
docker build -t "${IMAGE}" -f ${{ matrix.dockerfile }} .
docker image tag "${IMAGE}" "${IMAGE}:${{ matrix.tag }}"
docker push "${IMAGE}:${{ matrix.tag }}"
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# if: ${{ steps.login.conclusion != 'skipped' }}
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.IMAGE }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Push Latest Image
if: ${{ steps.login.conclusion != 'skipped' && matrix.tag == '11-8' }}
run: docker push "${IMAGE}:latest"

167
.github/workflows/concrete_compiler_benchmark.yml vendored Normal file
View File

@@ -0,0 +1,167 @@
name: concrete-compiler benchmark linux-cpu
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_compiler_benchmark.yml
- compilers/**
- backends/**
- tools/**
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_compiler_benchmark_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: cpu-bench
build-and-run-benchmarks:
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
outputs:
bench_date: ${{ steps.benchmark-details.outputs.bench_date }}
commit_date: ${{ steps.benchmark-details.outputs.commit_date }}
commit_hash: ${{ steps.benchmark-details.outputs.commit_hash }}
steps:
- name: Checkout concrete
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
fetch-depth: 0
- name: Ouput benchmark details
id: benchmark-details
run: |
echo "bench_date=$(date --iso-8601=seconds)" >> "$GITHUB_OUTPUT"
echo "commit_date=$(git --no-pager show -s --format=%cd --date=iso8601-strict ${{ github.sha }})" >> "$GITHUB_OUTPUT"
echo "commit_hash=$(git describe --tags --dirty)" >> "$GITHUB_OUTPUT"
- name: Set up home
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Build compiler benchmarks
run: |
set -e
git config --global --add safe.directory '*'
cd compilers/concrete-compiler/compiler
make BINDINGS_PYTHON_ENABLED=OFF build-benchmarks
- name: Run compiler benchmarks
run: |
set -e
cd compilers/concrete-compiler/compiler
make run-cpu-benchmarks
- name: Upload raw results artifact
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: compiler-benchmarks-result
path: compilers/concrete-compiler/compiler/benchmarks_results.json
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-run-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
parse-and-send-results:
name: Parse and send results
needs: [setup-instance, build-and-run-benchmarks]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Download compiler-benchmarks-result
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: compiler-benchmarks-result
- name: Parse results
shell: bash
run: |
# TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38)
python3 ./ci/benchmark_parser.py benchmarks_results.json parsed_benchmark_results.json \
--database compiler_benchmarks \
--hardware "hpc7a.96xlarge" \
--project-version ${{ needs.build-and-run-benchmarks.outputs.commit_hash}} \
--branch ${{ github.ref_name }} \
--commit-date "${{ needs.build-and-run-benchmarks.outputs.commit_date }}" \
--bench-date "${{ needs.build-and-run-benchmarks.outputs.bench_date }}" \
--throughput
- name: Upload parsed results artifact
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: compiler-benchmarks-parsed-result
path: parsed_benchmark_results.json
- name: Checkout Slab repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: zama-ai/slab
path: slab
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Send data to Slab
shell: bash
run: |
echo "Computing HMac on downloaded artifact"
SIGNATURE="$(slab/scripts/hmac_calculator.sh $parsed_benchmark_results.json '${{ secrets.JOB_SECRET }}')"
echo "Sending results to Slab..."
curl -v -k \
-H "Content-Type: application/json" \
-H "X-Slab-Repository: ${{ github.repository }}" \
-H "X-Slab-Command: store_data" \
-H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-d @parsed_benchmark_results.json \
${{ secrets.SLAB_URL }}
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "parse-and-send-results finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
name: Teardown instance
needs: [ setup-instance, parse-and-send-results ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

218
.github/workflows/concrete_compiler_publish_docker_images.yml vendored Normal file
View File

@@ -0,0 +1,218 @@
name: concrete-compiler publish docker images
on:
workflow_dispatch:
push:
branches:
- 'main'
- 'force-docker-images'
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
THIS_FILE: .github/workflows/concrete_compiler_publish_docker_images.yml
concurrency:
group: concrete_compiler_publish_docker_images
cancel-in-progress: true
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: cpu-test
hpx-image:
needs: [setup-instance]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
env:
image: ghcr.io/zama-ai/hpx
dockerfile: docker/Dockerfile.hpx-env
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
- name: Login
id: login
if: contains(steps.changed-files.outputs.modified_files, env.dockerfile) || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build
if: ${{ steps.login.conclusion != 'skipped' }}
run: docker build -t "${{ env.image }}" -f ${{ env.dockerfile }} .
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# if: ${{ steps.login.conclusion != 'skipped' }}
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.IMAGE }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Publish
if: ${{ steps.login.conclusion != 'skipped' }}
run: docker push "${{ env.image }}:latest"
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "hpx-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
cuda-image:
needs: [setup-instance]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
env:
image: ghcr.io/zama-ai/cuda
strategy:
matrix:
include:
- name: cuda-12-3
tag: 12-3
dockerfile: docker/Dockerfile.cuda-123-env
- name: cuda-11-8
tag: 11-8
dockerfile: docker/Dockerfile.cuda-118-env
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Set up env
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
- name: Login
id: login
# from the docs: The jobs.<job_id>.if condition is evaluated before jobs.<job_id>.strategy.matrix is applied. So we can't just use matrix.dockerfile
# so we have to build both images if one of the two files change, or we will have to split this into two
# https://docs.github.com/en/actions/writing-workflows/workflow-syntax-for-github-actions#jobsjob_idif
if: contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-118-env') || contains(steps.changed-files.outputs.modified_files, 'docker/Dockerfile.cuda-123-env') || contains(steps.changed-files.outputs.modified_files, env.THIS_FILE)
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build Tag and Publish
if: ${{ steps.login.conclusion != 'skipped' }}
run: |
docker build -t "${{ env.image }}" -f ${{ matrix.dockerfile }} .
docker image tag "${{ env.image }}" "${{ env.image }}:${{ matrix.tag }}"
docker push "${{ env.image }}:${{ matrix.tag }}"
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# if: ${{ steps.login.conclusion != 'skipped' }}
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.image }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Push Latest Image
if: ${{ steps.login.conclusion != 'skipped' && matrix.tag == '11-8' }}
run: docker push "${{ env.image }}:latest"
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "cuda-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
compiler-image:
needs: [setup-instance, hpx-image, cuda-image]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
env:
image: ghcr.io/zama-ai/concrete-compiler
dockerfile: docker/Dockerfile.concrete-compiler-env
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
submodules: recursive
- name: Get changed files
id: changed-files
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275 # v44.5.24
with:
files: |
backends/**
compilers/**
third_party/**
tools/**
- name: Login to Registry
id: login
if: steps.changed-files.outputs.any_changed == 'true'
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build Image
if: steps.login.conclusion != 'skipped'
run: |
DOCKER_BUILDKIT=1 docker build --no-cache \
--label "commit-sha=${{ github.sha }}" -t ${{ env.image }} -f ${{ env.dockerfile }} .
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ matrix.image }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Tag and Publish Image
if: steps.login.conclusion != 'skipped'
run: |
docker image tag ${{ env.image }} ${{ env.image }}:${{ github.sha }}
docker image push ${{ env.image }}:latest
docker image push ${{ env.image }}:${{ github.sha }}
- name: Tag and Publish Release Image
if: steps.login.conclusion != 'skipped' && startsWith(github.ref, 'refs/tags/v')
run: |
docker image tag ${{ env.image }} ${{ env.image }}:${{ github.ref_name }}
docker image push ${{ env.image }}:${{ github.ref_name }}
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "compiler-image finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
name: Teardown instance
needs: [ setup-instance, compiler-image ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

181
.github/workflows/concrete_compiler_test_cpu.yml vendored Normal file
View File

@@ -0,0 +1,181 @@
name: concrete-compiler test linux-cpu
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_compiler_test_cpu.yml
- compilers/**
- backends/concrete-cpu/**
- tools/**
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_compiler_test_cpu_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: cpu-test
format-and-lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Format with clang-format (Cpp)
run: |
sudo apt install moreutils
cd compilers/concrete-compiler/compiler
./scripts/format_cpp.sh
- name: Format with cmake-format (Cmake)
run: |
pip3 install cmakelang
cd compilers/concrete-compiler/compiler
./scripts/format_cmake.sh
- name: Format with black (Python)
run: |
cd compilers/concrete-compiler/compiler
pip install -r lib/Bindings/Python/requirements_dev.txt
make check-python-format
- name: Lint with pylint (Python)
run: |
cd compilers/concrete-compiler/compiler
# compiler requirements to lint
pip install numpy
make python-lint
- name: Check if sources include the license header
run: .github/workflows/scripts/check_for_license.sh
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "format-and-lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
build-and-run-test:
needs: [ setup-instance ]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
fetch-depth: 0
- name: Create build dir
run: mkdir build
- name: Build compiler
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
shell: bash
run: |
set -e
cd /concrete/compilers/concrete-compiler/compiler
rm -rf /build/*
make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Check compiler dialects docs is up to date
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-docs
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
shell: bash
run: |
set -e
cd /build/tools/concretelang/docs/concretelang/
sed -i -e 's/\[TOC\]//' *Dialect.md
for i in `ls *Dialect.md`; do diff $i /concrete/docs/explanations/$i; done;
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> ${GITHUB_ENV}
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> ${GITHUB_ENV}
- name: Run compiler tests
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
shell: bash
run: |
set -e
cd /concrete/compilers/concrete-compiler/compiler
mkdir -p /tmp/concrete_compiler/gpu_tests/
pip install pytest
sed "s/pytest/python -m pytest/g" -i Makefile
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
needs: [ setup-instance, build-and-run-test ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

109
.github/workflows/concrete_compiler_test_cpu_distributed.yml vendored Normal file
View File

@@ -0,0 +1,109 @@
name: concrete-compiler test linux-cpu-distributed
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_compiler_test_cpu_distributed.yml
- compilers/concrete-compiler/**
push:
branches:
- 'main'
- 'release/*'
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_compiler_test_cpu_distributed_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: slurm-cluster
build-and-run-test:
# The distributed-ci runner is registered on the instance configured in the slurm-cluster profile.
# It's why we need to setup-instance
needs: setup-instance
runs-on: distributed-ci
steps:
- name: Instance cleanup
run: |
sudo rm -rf /home/ubuntu/actions-runner/_work/concrete/concrete
mkdir -p /home/ubuntu/actions-runner/_work/concrete/concrete
docker system prune -af
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
submodules: recursive
- name: Set up home
# "Install rust" step require root user to have a HOME directory which is not set.
run: |
echo "HOME=/shared" >> "${GITHUB_ENV}"
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Build end-to-end distributed test
run: |
cd compilers/concrete-compiler/compiler
rm -rf /shared/build
make HPX_DIR=/shared/hpx install-hpx-from-source
make HPX_DIR=/shared/hpx BUILD_DIR=/shared/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON BINDINGS_PYTHON_ENABLED=OFF build-end-to-end-tests
- name: Run end-to-end distributed test
run: |
cd compilers/concrete-compiler/compiler
rm -rf /shared/KeyCache
make BUILD_DIR=/shared/build KEY_CACHE_DIRECTORY=/shared/KeyCache run-end-to-end-distributed-tests
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
needs: [ setup-instance, build-and-run-test ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

106
.github/workflows/concrete_compiler_test_gpu.yml vendored Normal file
View File

@@ -0,0 +1,106 @@
name: concrete-compiler test linux-gpu
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_compiler_test_gpu.yml
- compilers/**
- backends/concrete-cuda/**
- tools/**
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
CUDA_PATH: /usr/local/cuda-11.8
concurrency:
group: concrete_compiler_test_gpu_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: gpu-test
build-and-test:
needs: [ setup-instance ]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
if: ${{ !cancelled() }}
steps:
- name: Fetch repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
submodules: recursive
- name: Create build dir
run: mkdir build
- name: Build and test compiler
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
--gpus all
shell: bash
run: |
set -e
cd /concrete/compilers/concrete-compiler/compiler
rm -rf /build/*
mkdir -p /tmp/concrete_compiler/gpu_tests/
make BINDINGS_PYTHON_ENABLED=OFF Python3_EXECUTABLE=$PYTHON_EXEC CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} run-end-to-end-tests-gpu
echo "Debug: ccache statistics (after the build):"
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-run-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
needs: [ setup-instance, build-and-test ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

89
.github/workflows/concrete_compiler_test_macos_cpu.yml vendored Normal file
View File

@@ -0,0 +1,89 @@
name: concrete-compiler test macos-cpu
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_compiler_test_macos_cpu.yml
- compilers/**
- backends/**
- tools/**
push:
branches:
- 'main'
- 'release/*'
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_compiler_test_macos_cpu_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
build-and-test:
strategy:
# if a failure happens, we want to know if it's specific
# to the architecture or the operating system
fail-fast: false
matrix:
runson: ["aws-mac1-metal", "aws-mac2-metal"]
python-version: ["3.10"]
runs-on: ${{ matrix.runson }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
fetch-depth: 0
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Set python variables
run: |
set -e
echo "PIP=${{ format('pip{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
echo "PYTHON=${{ format('python{0}', matrix.python-version) }}" >> "${GITHUB_ENV}"
echo "PYTHON_EXEC=$(which ${{ format('python{0}', matrix.python-version) }})" >> "${GITHUB_ENV}"
- name: Install dependencies
run: |
brew install ninja ccache
${PIP} install pytest
- name: Build compiler
run: |
set -e
cd compilers/concrete-compiler/compiler
echo "Debug: ccache statistics (prior to the build):"
ccache -s
make Python3_EXECUTABLE=$PYTHON_EXEC all
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Enable complete tests on push to main
if: github.ref == 'refs/heads/main'
run: echo "MINIMAL_TESTS=OFF" >> $GITHUB_ENV
- name: Enable minimal tests otherwise
if: github.ref != 'refs/heads/main'
run: echo "MINIMAL_TESTS=ON" >> $GITHUB_ENV
- name: Create keyset cache directory
run: |
export KEY_CACHE_DIRECTORY=$(mktemp -d)/KeySetCache
echo "KEY_CACHE_DIRECTORY=$KEY_CACHE_DIRECTORY" >> "${GITHUB_ENV}"
mkdir $KEY_CACHE_DIRECTORY
- name: Test
run: |
set -e
cd compilers/concrete-compiler/compiler
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
make MINIMAL_TESTS=${{ env.MINIMAL_TESTS }} Python3_EXECUTABLE=$PYTHON_EXEC run-tests
- name: Cleanup host
if: success() || failure()
run: |
rm -rf $KEY_CACHE_DIRECTORY
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-test finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

53
.github/workflows/concrete_cpu_test.yml vendored
View File

@@ -1,17 +1,28 @@
name: Concrete CPU - Tests
name: concrete-cpu test
on:
workflow_call:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_cpu_test.yml
- backends/concrete-cpu/**
push:
branches:
- 'main'
- 'release/*'
concurrency:
group: concrete_cpu_test-${{ github.ref }}
group: concrete_cpu_test_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
env:
CARGO_TERM_COLOR: always
jobs:
tests-linux:
strategy:
fail-fast: false
matrix:
runson: ["ubuntu-20.04"]
runs-on: ubuntu-20.04
env:
RUSTFLAGS: -D warnings
@@ -57,39 +68,3 @@ jobs:
run: |
cd backends/concrete-cpu/implementation
cargo test --no-fail-fast --all-targets --features=nightly
tests-mac_x86:
runs-on: macos-11
env:
RUSTFLAGS: -D warnings
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Download cargo cache
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
- name: Tests
run: |
cd backends/concrete-cpu/implementation
cargo test --no-fail-fast --all-targets
tests-mac-m1:
runs-on: "aws-mac2-metal"
env:
RUSTFLAGS: -D warnings
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Download cargo cache
uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
- name: Tests
run: |
cd backends/concrete-cpu/implementation
cargo test --no-fail-fast --all-targets

138
.github/workflows/concrete_ml_test.yml vendored Normal file
View File

@@ -0,0 +1,138 @@
name: concrete-ml test
on:
workflow_dispatch:
pull_request:
paths:
- .github/workflows/concrete_ml_test.yml
- frontends/concrete-python/**
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_ml_test_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: cpu-test
build-and-run-tests:
strategy:
matrix:
python-version: ["3.8"]
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
- name: Set release version
run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Create build directory
run: mkdir build
- name: Build wheel
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-bindings
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
shell: bash
run: |
set -e
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
cd /concrete/frontends/concrete-python
export COMPILER_BUILD_DIRECTORY="/build"
make whl
deactivate
- name: Setup Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: ${{ matrix.python-version }}
- name: ML Tests
run: |
export HOME="/home/ubuntu"
export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl
apt update
apt install git git-lfs -y
pip install poetry==1.7.1
./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-and-run-tests finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
needs: [ setup-instance, build-and-run-tests ]
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

112
.github/workflows/concrete_ml_tests.yml vendored
View File

@@ -1,112 +0,0 @@
name: Concrete ML Tests
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
env:
DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler
jobs:
linux-x86:
strategy:
matrix:
python-version: ["3.8"]
runs-on: ${{ github.event.inputs.runner_name }}
steps:
- name: Log instance configuration
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "User Inputs: ${{ inputs.user_inputs }}"
- name: Set up GitHub environment
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
#echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Set release version
run: echo "__version__ = \"$(date +"%Y.%m.%d")\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Create build directory
run: mkdir build
- name: Build wheel
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-bindings
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
set -e
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
cd /concrete/frontends/concrete-python
export COMPILER_BUILD_DIRECTORY="/build"
make whl
deactivate
- name: Setup Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: ${{ matrix.python-version }}
- name: ML Tests
run: |
export CONCRETE_PYTHON_WHEEL=$(pwd)/frontends/concrete-python/dist/*manylinux*.whl
apt update
apt install git git-lfs -y
pip install poetry==1.7.1
./ci/scripts/test_cml.sh --use-wheel $CONCRETE_PYTHON_WHEEL --verbose

63
.github/workflows/optimizer.yml → .github/workflows/concrete_optimizer.yml vendored
View File

@@ -1,73 +1,72 @@
name: Optimizer - Tests
name: concrete-optimizer test
on:
workflow_call:
workflow_dispatch:
secrets:
CONCRETE_CI_SSH_PRIVATE:
required: true
CONCRETE_ACTIONS_TOKEN:
required: true
concurrency:
group: optimizer-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
pull_request:
paths:
- .github/workflows/concrete_optimizer.yml
- compilers/concrete-optimizer/**
- backends/**
- tools/**
push:
branches:
- 'main'
- 'release/*'
env:
CARGO_TERM_COLOR: always
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_optimizer-${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
tests:
strategy:
matrix:
os: [ubuntu-20.04, macos-11]
runs-on: ${{ matrix.os }}
runson: ["ubuntu-latest", "aws-mac1-metal", "aws-mac2-metal"]
runs-on: ${{ matrix.runson }}
env:
RUSTFLAGS: -D warnings
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: "Setup"
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup
uses: ./.github/workflows/optimizer_setup
with:
ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
- name: Formatting
run: |
cargo --version
cd compilers/concrete-optimizer
cargo fmt --check
- name: Build
run: |
cd compilers/concrete-optimizer
cargo build --release --all-targets
- name: Lint
run: |
cd compilers/concrete-optimizer
cargo clippy --release --all-targets
- name: Tests
if: matrix.os == 'ubuntu-20.04'
run: |
cd compilers/concrete-optimizer
cargo test --release --no-fail-fast --all-targets
make -C concrete-optimizer-cpp test-ci
benchmarks:
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: "Setup"
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Setup
uses: ./.github/workflows/optimizer_setup
with:
ssh_private_key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
- name: Run benchmark
run: |
cd compilers/concrete-optimizer
cargo bench -p v0-parameters -- --output-format bencher | tee bench_result.txt
- name: Download PR base benchmark data
if: ${{ github.event_name == 'pull_request' }}
# for artifacts restrictions see https://github.com/actions/download-artifact/issues/3
@@ -81,7 +80,6 @@ jobs:
name: ${{ runner.os }}-benchmark
if_no_artifact_found: warn
path: ./benchmark
- name: Save benchmark result to file
uses: benchmark-action/github-action-benchmark@4de1bed97a47495fc4c5404952da0499e31f5c29 # v1.20.3
with:
@@ -94,7 +92,6 @@ jobs:
comment-always: true
# Enable Job Summary for PRs
summary-always: true
- name: Upload benchmark data
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:

54
.github/workflows/concrete_python_benchmark.yml vendored
View File

@@ -1,17 +1,25 @@
name: Concrete Python Benchmark
name: concrete-python benchmark linux-cpu
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * SAT"
pull_request:
paths:
- .github/workflows/concrete_pyhon_benchmark.yml
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
concurrency:
group: concrete_python_benchmark_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
name: Setup Instance
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
@@ -25,10 +33,9 @@ jobs:
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: m7i-cpu-bench
profile: cpu-bench
concrete-python-benchmarks:
name: Run Concrete Python Benchmarks
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
@@ -49,22 +56,14 @@ jobs:
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
set -e
rustup toolchain install nightly-2024-09-30
pip install mypy
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
dnf -y install graphviz graphviz-devel
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
source .venv/bin/activate
@@ -72,14 +71,12 @@ jobs:
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python) python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
cd /concrete/frontends/concrete-python
export COMPILER_BUILD_DIRECTORY="/build"
export PROGRESS_MACHINE_NAME="m7i.48xlarge"
# TODO output setup-instance (https://github.com/zama-ai/slab-github-runner/issues/38)
export PROGRESS_MACHINE_NAME="hpc7a.96xlarge"
make benchmark
make process-benchmark-results-for-grafana
@@ -97,9 +94,9 @@ jobs:
run: |
echo "Computing HMac on results file"
SIGNATURE="$(slab/scripts/hmac_calculator.sh frontends/concrete-python/progress.processed.json '${{ secrets.JOB_SECRET }}')"
cd frontends/concrete-python
echo "Sending results to Slab..."
curl -v -k \
-H "Content-Type: application/json" \
@@ -108,9 +105,15 @@ jobs:
-H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-d @progress.processed.json \
${{ secrets.SLAB_URL }}
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "concrete-python-benchmarks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
name: Teardown Instance
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
needs: [ setup-instance, concrete-python-benchmarks ]
runs-on: ubuntu-latest
@@ -124,3 +127,10 @@ jobs:
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

16
.github/workflows/concrete_python_checks.yml vendored
View File

@@ -1,16 +0,0 @@
name: Concrete Python Checks
on:
workflow_call:
jobs:
Checks:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install Platform Dependencies
run: |
sudo apt install -y graphviz libgraphviz-dev
- name: Pre-Commit Checks
run: |
./frontends/concrete-python/scripts/checks/checks.sh

79
.github/workflows/concrete_python_finalize_release.yml vendored Normal file
View File

@@ -0,0 +1,79 @@
# This workflows should be runned after that releases has been validated and ready to push to pypi.org and docker hub.
name: concrete-python finalize-release
on:
workflow_dispatch:
inputs:
version:
description: 'version of concrete-python to push to pypi and docker hub'
required: true
type: string
jobs:
publish-to-pypi:
runs-on: ubuntu-latest
steps:
- name: Pull wheels from S3
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }}
run: |
mkdir wheels
aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
echo "============== Downloaded wheels ==============="
ls -la ./wheels/
- name: Push wheels to public PyPI (public)
run: |
pip install twine==4.0.2
twine upload wheels/concrete_python-${{ inputs.version }}*.whl \
-u "${{ secrets.PUBLIC_PYPI_USER }}" \
-p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \
-r pypi
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
publish-to-dockerhub:
runs-on: ubuntu-latest
env:
DOCKER_IMAGE_NAME: zamafhe/concrete-python
DOCKER_FILE: docker/Dockerfile.concrete-python
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Get version from tag
run: |
# remove leading 'v' and '-' from tag
export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"`
echo "VERSION=$VERSION" >> "${GITHUB_ENV}"
echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}"
- name: Build image
run: |
mkdir empty_context
docker image build -t ${{ env.NAME_TAG }} --build-arg version=${{ env.VERSION }} -f ${{ env.DOCKER_FILE }} empty_context
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.NAME_TAG }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Push image
run: docker image push ${{ env.NAME_TAG }}

55
.github/workflows/concrete_python_push_docker_image.yml vendored
View File

@@ -1,55 +0,0 @@
name: Concrete Python Push Docker Image
on:
workflow_dispatch:
inputs:
tag:
description: 'tag to use for the docker image'
type: string
workflow_call:
inputs:
tag:
description: 'tag to use for the docker image'
type: string
env:
DOCKER_IMAGE_NAME: zamafhe/concrete-python
DOCKER_FILE: docker/Dockerfile.concrete-python
jobs:
build_and_push:
runs-on: ubuntu-22.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Get version from tag
run: |
# remove leading 'v' and '-' from tag
export VERSION=`echo ${{ inputs.tag }} | sed "s/^v*//g" | sed "s/-//g"`
echo "VERSION=$VERSION" >> "${GITHUB_ENV}"
echo "NAME_TAG=${{ env.DOCKER_IMAGE_NAME }}:v$VERSION" >> "${GITHUB_ENV}"
- name: Build image
run: |
mkdir empty_context
docker image build -t ${{ env.NAME_TAG }} --build-arg version=${{ env.VERSION }} -f ${{ env.DOCKER_FILE }} empty_context
# disabled because of https://github.com/aquasecurity/trivy/discussions/7668
# - name: Run Trivy vulnerability scanner
# uses: aquasecurity/trivy-action@915b19bbe73b92a6cf82a1bc12b087c9a19a5fe2 # 0.28.0
# with:
# image-ref: '${{ env.NAME_TAG }}'
# format: 'table'
# exit-code: '1'
# ignore-unfixed: true
# vuln-type: 'os,library'
# severity: 'CRITICAL,HIGH'
- name: Login to Docker Hub
uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Push image
run: docker image push ${{ env.NAME_TAG }}

177
.github/workflows/concrete_python_release.yml → .github/workflows/concrete_python_release_cpu.yml vendored
View File

@@ -1,37 +1,46 @@
name: Concrete Python Release
name: concrete-python release-cpu
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
user_inputs:
description: 'either "nightly" or "public" or "private" to specify the release type'
required: true
default: 'nightly'
type: string
push:
tags:
- 'v[0-9]+.[0-9]+.[0-9]+*'
schedule:
# Nightly Release @ 3AM after each work day
- cron: "0 3 * * 2-6"
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
RELEASE_TYPE: ${{ inputs.user_inputs }}
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }}
concurrency:
group: concrete_python_release_cpu_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: release
release-checks:
runs-on: ${{ github.event.inputs.runner_name }}
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -40,32 +49,29 @@ jobs:
fetch-depth: 0
- name: Check python api doc is up to date
run: ci/scripts/make_apidocs.sh
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "release-checks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
build-linux-x86:
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
runs-on: ${{ github.event.inputs.runner_name }}
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Log instance configuration
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "User Inputs: ${{ inputs.user_inputs }}"
- name: Set up GitHub environment
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
fetch-depth: 0
- name: Set release version (nightly)
if: ${{ env.RELEASE_TYPE == 'nightly' }}
run: |
@@ -75,14 +81,11 @@ jobs:
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
git tag nightly-$NIGHTLY_VERSION || true
git push origin nightly-$NIGHTLY_VERSION || true
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-bindings
@@ -94,15 +97,10 @@ jobs:
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
shell: bash
run: |
set -e
rustup toolchain install nightly-2024-09-30
pip install mypy
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
@@ -126,20 +124,25 @@ jobs:
make whl
deactivate
- name: Upload wheel
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
path: frontends/concrete-python/dist/*manylinux*.whl
retention-days: 3
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-linux-x86 finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
build-macos:
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
runs-on: ["aws-mac1-metal", "aws-mac2-metal"]
runs-on: ${{ matrix.runs-on }}
steps:
- name: Checkout
@@ -147,14 +150,11 @@ jobs:
with:
submodules: recursive
fetch-depth: 0
- name: Install OS Dependencies
run: |
brew install ninja ccache
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Set release version (nightly)
if: ${{ env.RELEASE_TYPE == 'nightly' }}
run: |
@@ -162,14 +162,11 @@ jobs:
NIGHTLY_VERSION_ONE_NUMBER=$(date +"%Y%m%d")
LATEST_RELEASE_VERSION=`git tag -l |grep "v.*" |sort |tail -n 1 | grep -e '[0-9].*' -o`
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
run: |
export CONCRETE_PYTHON=$(pwd)/frontends/concrete-python
@@ -203,13 +200,19 @@ jobs:
delocate-wheel -v dist/*macos*.whl
deactivate
- name: Upload wheel
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: ${{ format('wheel-{0}-{1}', matrix.python-version, matrix.runs-on) }}
path: frontends/concrete-python/dist/*macos*.whl
retention-days: 3
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-macos finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
hash:
# Generate hashes for the wheels, used later for provenance.
@@ -279,28 +282,24 @@ jobs:
aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/cpu/concrete-python
# update indexes and invalidate cloudfront cache
python .github/workflows/scripts/s3_update_html_indexes.py
- name: Start pushing Docker images
if: ${{ env.RELEASE_TYPE == 'public' }}
run: |
export TAG=$(git describe --tags --abbrev=0)
curl -L \
-X POST \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
https://api.github.com/repos/zama-ai/concrete/actions/workflows/concrete_python_push_docker_image.yml/dispatches \
-d "{\"ref\": \"$TAG\", \"inputs\": {\"tag\":\"v$TAG\"}}"
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "push finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
test-linux-x86:
needs: [build-linux-x86]
needs: [setup-instance, build-linux-x86]
continue-on-error: true
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
runs-on: ${{ github.event.inputs.runner_name }}
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1
- name: Install rust
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7
- name: Setup Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
@@ -343,6 +342,36 @@ jobs:
# Running tests
make tfhers-utils
pytest tests -svv -n auto
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-linux-x86 (${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
teardown-instance:
needs: [ setup-instance, test-linux-x86 ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
test-macos:
needs: [build-macos]
@@ -396,8 +425,14 @@ jobs:
make tfhers-utils
mkdir ./KeySetCache
pytest tests -svv -n auto --key-cache "./KeySetCache" -m "not dataflow and not graphviz"
- name: Cleanup host
if: success() || failure()
run: |
rm -rf $TEST_TMP_DIR
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-macos (${{matrix.runs-on}}/${{ matrix.python-version }}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

217
.github/workflows/concrete_python_release_gpu.yml vendored
View File

@@ -1,61 +1,58 @@
name: Concrete Python Release (GPU)
name: concrete-python release-gpu
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
user_inputs:
description: 'either "nightly" or "public" or "private" to specify the release type'
required: true
default: 'nightly'
type: string
push:
tags:
- 'v[0-9]+.[0-9]+.[0-9]+*'
schedule:
# Nightly Release @ 3AM after each work day
- cron: "0 3 * * 2-6"
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
CUDA_PATH: /usr/local/cuda-11.8
GCC_VERSION: 11
RELEASE_TYPE: ${{ inputs.user_inputs }}
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
RELEASE_TYPE: ${{ (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) && 'public' || 'nightly' }}
concurrency:
group: concrete_python_release_gpu_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: release
build-linux-x86:
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
runs-on: ${{ github.event.inputs.runner_name }}
steps:
- name: Log instance configuration
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
echo "User Inputs: ${{ inputs.user_inputs }}"
- name: Set up GitHub environment
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
fetch-depth: 0
- name: Set release version (nightly)
if: ${{ env.RELEASE_TYPE == 'nightly' }}
run: |
@@ -65,14 +62,11 @@ jobs:
echo "__version__ = \"${LATEST_RELEASE_VERSION}-dev${NIGHTLY_VERSION_ONE_NUMBER}\"" >| frontends/concrete-python/version.txt
git tag nightly-$NIGHTLY_VERSION || true
git push origin nightly-$NIGHTLY_VERSION || true
- name: Set release version (public)
if: ${{ env.RELEASE_TYPE == 'public' }}
run: echo "__version__ = \"`git describe --tags --abbrev=0 | grep -e '[0-9].*' -o`\"" >| frontends/concrete-python/version.txt
- name: Expose release version from Python
run: cp frontends/concrete-python/version.txt frontends/concrete-python/concrete/fhe/version.py
- name: Build wheel
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
id: build-compiler-bindings
@@ -84,29 +78,23 @@ jobs:
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
shell: bash
run: |
set -e
rustup toolchain install nightly-2024-09-30
pip install mypy
rm -rf /build/*
export PYTHON=${{ format('python{0}', matrix.python-version) }}
echo "Using $PYTHON"
dnf -y install graphviz graphviz-devel
cd /concrete/frontends/concrete-python
make PYTHON=$PYTHON venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=OFF Python3_EXECUTABLE=$(which python) \
CUDA_SUPPORT=ON TIMING_ENABLED=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
@@ -114,37 +102,62 @@ jobs:
export COMPILER_BUILD_DIRECTORY="/build"
make whl
deactivate
deactivate
- name: Upload wheel
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: ${{ format('wheel-{0}-linux-x86', matrix.python-version) }}
path: frontends/concrete-python/dist/*manylinux*.whl
retention-days: 3
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-linux-x86 (${{matrix.python-version}}) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
push:
teardown-instance:
needs: [ setup-instance, build-linux-x86 ]
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
push-wheels:
needs: [build-linux-x86]
runs-on: ubuntu-latest
outputs:
wheel_version: ${{ steps.version.outputs.wheel_version }}
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: wheels
merge-multiple: true
- name: Install aws-cli if not present
- name: Install aws-cli
run: |
aws --version || (curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
unzip awscliv2.zip && \
sudo ./aws/install)
- name: Upload wheels to S3
if: ${{ env.RELEASE_TYPE == 'public' || env.RELEASE_TYPE == 'nightly' }}
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }}
@@ -157,18 +170,92 @@ jobs:
aws s3 sync ./wheels/ s3://${S3_BUCKET_NAME}/gpu/concrete-python
# update indexes and invalidate cloudfront cache
python .github/workflows/scripts/s3_update_html_indexes.py
- name: Output Wheel Version
id: version
run: |
export VERSION=`ls ./wheels/*manylinux* | head -n1 | cut -d "-" -f2`
echo "VERSION=$VERSION"
echo "wheel_version=$VERSION" >> "$GITHUB_OUTPUT"
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "push-wheels finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
test-gpu-wheel:
needs: [push]
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-test-gpu-wheel
user_inputs: "${{ needs.push.outputs.wheel_version }}"
setup-test-instance:
runs-on: ubuntu-latest
needs: [push-wheels]
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: gpu-test
test-linux-x86:
strategy:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
fail-fast: false
needs: [setup-test-instance, push-wheels]
runs-on: ${{ needs.setup-test-instance.outputs.runner-name }}
steps:
- name: Setup Python
uses: actions/setup-python@f677139bbe7f9c59b41e40162b753c062f5d49a3 # v5.2.0
with:
python-version: ${{ matrix.python-version }}
- name: Install concrete-python
run: pip install --pre --extra-index-url https://pypi.zama.ai/gpu/ "concrete-python==${{ needs.push-wheels.outputs.wheel_version }}"
- name: Checkout the repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: repo
- name: Test wheel
run: |
CONCRETE_PYTHON=$(pwd)/repo/frontends/concrete-python
# Install extra requirements for tests
sudo apt update -y
sudo apt install -y graphviz libgraphviz-dev
pip install -r $CONCRETE_PYTHON/requirements.extra-full.txt
pip install -r $CONCRETE_PYTHON/requirements.dev.txt
# Running tests
cd $CONCRETE_PYTHON
make pytest-gpu
teardown-test-instance:
needs: [ setup-test-instance, test-linux-x86 ]
if: ${{ always() && needs.setup-test-instance.result != 'skipped' }}
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-test-instance.outputs.runner-name }}
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

96
.github/workflows/concrete_python_test_macos.yml vendored
View File

@@ -1,40 +1,37 @@
name: Concrete Python Tests (macOS)
name: concrete-python tests macos
on:
workflow_call:
workflow_dispatch:
secrets:
CONCRETE_CI_SSH_PRIVATE:
required: true
CONCRETE_ACTIONS_TOKEN:
required: true
pull_request:
paths:
- .github/workflows/concrete_python_tests_macos.yml
push:
branches:
- 'main'
- 'release/*'
concurrency:
group: concrete_python_tests_macos-${{ github.ref }}
group: concrete_python_tests_macos_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
BuildConcreteCompilerAndTestConcretePythonInMacOS:
concrete-python-test-pytest:
strategy:
fail-fast: false
matrix:
machine: ["aws-mac1-metal", "aws-mac2-metal"]
runs-on: ${{ matrix.machine }}
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Install OS Dependencies
fetch-depth: 0
- name: Install build dependencies
run: |
brew install ninja ccache
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Cache Compilation (push)
- name: Cache compilation (push)
if: github.event_name == 'push'
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
@@ -42,8 +39,7 @@ jobs:
key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-compilation-cache-
- name: Cache Compilation (pull_request)
- name: Cache compilation (pull_request)
if: github.event_name == 'pull_request'
uses: actions/cache@6849a6489940f00c2f30c0fb92c6274307ccb58a # v4.1.2
with:
@@ -51,88 +47,72 @@ jobs:
key: ${{ runner.os }}-${{ runner.arch }}-compilation-cache-${{ github.event.pull_request.base.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-compilation-cache-
- name: Get tmpdir path
if: github.event_name == 'push'
id: tmpdir-path
run: echo "::set-output name=TMPDIR_PATH::$TMPDIR"
- name: Build
- name: Prepare build environment
run: |
set -e
cd frontends/concrete-python
cd $GITHUB_WORKSPACE/frontends/concrete-python
# Setup pkg-config to find OpenBLAS (scipy need it)
export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig"
rm -rf .venv
python3.10 -m venv .venv
. .venv/bin/activate
. $GITHUB_WORKSPACE/frontends/concrete-python/.venv/bin/activate
pip install -r requirements.dev.txt
pip install -r requirements.txt
- name: Build concrete-compiler python-bindings
run: |
$GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate
cd $GITHUB_WORKSPACE/compilers/concrete-compiler/compiler
echo "Debug: ccache statistics (prior to the build):"
ccache -s
ccache -z
make Python3_EXECUTABLE=$(which python) python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
export COMPILER_BUILD_DIRECTORY=$(pwd)/build
- name: Create wheels
run: |
$GITHUB_WORKSPACE/frontends/concrete-python .venv/bin/activate
cd $GITHUB_WORKSPACE/frontends/concrete-python
rm -rf dist
mkdir -p dist
export COMPILER_BUILD_DIRECTORY=$GITHUB_WORKSPACE/compilers/concrete-compiler/compiler
rm -rf dist && mkdir -p dist
pip wheel -v --no-deps -w dist .
delocate-wheel -v dist/*macos*.whl
deactivate
deactivate
- name: Prepare test environment
run: |
set -e
export TEST_TMP_DIR=$(mktemp -d)
echo "TEST_TMP_DIR=$TEST_TMP_DIR" >> "${GITHUB_ENV}"
cd $TEST_TMP_DIR
python3.10 -m venv .testenv
. .testenv/bin/activate
pip install $GITHUB_WORKSPACE/frontends/concrete-python/dist/*macos*.whl
pip install -r $GITHUB_WORKSPACE/frontends/concrete-python/requirements.dev.txt
# MacOS x86 have conflict between our OpenMP library, and one from torch
# we fix it by using a single one (from torch)
# see discussion: https://discuss.python.org/t/conflicting-binary-extensions-in-different-packages/25332/8
find .testenv/lib/python3.10/site-packages -not \( -path .testenv/lib/python3.10/site-packages/concrete -prune \) -name 'lib*omp5.dylib' -or -name 'lib*omp.dylib' | xargs -n 1 ln -f -s $(pwd)/.testenv/lib/python3.10/site-packages/concrete/.dylibs/libomp.dylib
cp -R $GITHUB_WORKSPACE/frontends/concrete-python/examples ./examples
cp -R $GITHUB_WORKSPACE/frontends/concrete-python/tests ./tests
cp $GITHUB_WORKSPACE/frontends/concrete-python/Makefile .
- name: Test
- name: Run pytest-macos
run: |
set -e
export TEST_TMP_DIR="testing_concrete_python"
cd $TEST_TMP_DIR && . .testenv/bin/activate
KEY_CACHE_DIRECTORY=./KeySetCache PYTEST_MARKERS="not dataflow and not graphviz" make pytest-macos
- name: Test notebooks
- name: Run test-notebooks
run: |
set -e
export TEST_TMP_DIR="testing_concrete_python"
cd $TEST_TMP_DIR && . .testenv/bin/activate
make test-notebooks
- name: Cleanup host
if: success() || failure()
run: |

290
.github/workflows/concrete_python_tests_linux.yml vendored
View File

@@ -1,69 +1,72 @@
name: Concrete Python Tests (Linux)
name: concrete-python tests linux-cpu
on:
workflow_dispatch:
inputs:
instance_id:
description: 'Instance ID'
type: string
instance_image_id:
description: 'Instance AMI ID'
type: string
instance_type:
description: 'Instance product type'
type: string
runner_name:
description: 'Action runner name'
type: string
request_id:
description: 'Slab request ID'
type: string
# concurrency:
# group: concrete_python_tests_linux-${{ github.ref }}
# cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
pull_request:
paths:
- .github/workflows/concrete_python_tests_linux.yml
- frontends/concrete-python/**
push:
branches:
- 'main'
- 'release/*'
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
CUDA_PATH: /usr/local/cuda-11.8
GCC_VERSION: 11
GLIB_VER: 2_28
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }}
SLACK_USERNAME: ${{ secrets.BOT_USERNAME }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
concurrency:
group: concrete_python_tests_linux_${{ github.ref }}
cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
jobs:
BuildConcreteCompilerAndTestConcretePythonInLinux:
name: Build Concrete Compiler and Test Concrete Python in Linux
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
setup-instance:
runs-on: ubuntu-latest
outputs:
runner-name: ${{ steps.start-instance.outputs.label }}
steps:
- name: Log instance configuration
run: |
echo "IDs: ${{ inputs.instance_id }}"
echo "AMI: ${{ inputs.instance_image_id }}"
echo "Type: ${{ inputs.instance_type }}"
echo "Request ID: ${{ inputs.request_id }}"
- name: Start instance
id: start-instance
uses: zama-ai/slab-github-runner@447a2d0fd2d1a9d647aa0d0723a6e9255372f261
with:
mode: start
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
backend: aws
profile: cpu-test
- name: Set up GitHub environment
pre-commit-check:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Install platform dependencies
run: |
echo "HOME=/home/ubuntu" >> "${GITHUB_ENV}"
#echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
- name: Checkout
sudo apt install -y graphviz libgraphviz-dev
- name: Pre-commit Checks
run: |
cd frontends/concrete-python
make venv
source .venv/bin/activate
make pcc
build-python-bindings:
needs: setup-instance
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Checkout concrete
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Create build directory
fetch-depth: 0
- name: Create concrete build directory
run: mkdir build
- name: Setup rust toolchain for concrete-cpu
uses: ./.github/workflows/setup_rust_toolchain_for_concrete_cpu
- name: Build bindings
- name: Build concrete-compiler python bindings
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ !contains(inputs.instance_type, 'p3') }}
id: build-compiler-bindings
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
@@ -72,30 +75,56 @@ jobs:
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
shell: bash
run: |
rustup toolchain install nightly-2024-09-30
pip install mypy
set -e
rm -rf /build/*
rustup toolchain install nightly-2024-09-30
dnf -y install graphviz graphviz-devel
cd /concrete/frontends/concrete-python
make venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$(which python3) python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Create artifact archive
run: |
cd build
tar czvf artifacts.tgz lib/libConcretelangRuntime.so tools/concretelang/python_packages
- name: Upload concrete-compiler python-bindings
uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
with:
name: concrete-compiler-python-bindings
include-hidden-files: true
retention-days: 3
path: build/artifacts.tgz
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "build-python-bindings finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
- name: Prepare test environment
test-pytest:
needs: [setup-instance, build-python-bindings]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Download concrete-compiler python-bindings
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: concrete-compiler-python-bindings
path: compiler-artifacts
- name: Extract artifacts archive
run: |
cd compiler-artifacts
tar xzvf artifacts.tgz
- name: Run pytest
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ !contains(inputs.instance_type, 'p3') }}
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
@@ -103,42 +132,39 @@ jobs:
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts
shell: bash
run: |
set -e
dnf -y install graphviz graphviz-devel
cd /concrete/frontends/concrete-python
make venv
- name: Test
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ !contains(inputs.instance_type, 'p3') }}
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
shell: bash
run: |
set -e
export COMPILER_BUILD_DIRECTORY=/compiler-artifacts
cd /concrete/frontends/concrete-python
source .venv/bin/activate
export KEY_CACHE_DIRECTORY=./key-set-cache
mkdir $KEY_CACHE_DIRECTORY
make pytest
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-pytest finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
export COMPILER_BUILD_DIRECTORY=/build
mkdir ./KeySetCache
KEY_CACHE_DIRECTORY=./KeySetCache make pytest
- name: Test notebooks
test-notebooks:
needs: [setup-instance, build-python-bindings]
runs-on: ${{ needs.setup-instance.outputs.runner-name }}
steps:
- name: Download concrete-compiler python-bindings
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: concrete-compiler-python-bindings
path: compiler-artifacts
- name: Extract artifacts archive
run: |
cd compiler-artifacts
tar xzvf artifacts.tgz
- name: Run pytest
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ !contains(inputs.instance_type, 'p3') }}
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
@@ -146,73 +172,41 @@ jobs:
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/compiler-artifacts:/compiler-artifacts
shell: bash
run: |
set -e
export COMPILER_BUILD_DIRECTORY=/compiler-artifacts
cd /concrete/frontends/concrete-python
source .venv/bin/activate
export COMPILER_BUILD_DIRECTORY=/build
make test-notebooks
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "test-notebooks finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"
- name: Build bindings gpu
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ contains(inputs.instance_type, 'p3') }}
id: build-compiler-bindings-gpu
teardown-instance:
if: ${{ always() && needs.setup-instance.result != 'skipped' }}
needs: [ setup-instance, test-pytest, test-notebooks ]
runs-on: ubuntu-latest
steps:
- name: Stop instance
id: stop-instance
uses: zama-ai/slab-github-runner@c0e7168795bd78f61f61146951ed9d0c73c9b701
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
--gpus all
shell: bash
run: |
set -e
rm -rf /build/*
mode: stop
github-token: ${{ secrets.SLAB_ACTION_TOKEN }}
slab-url: ${{ secrets.SLAB_BASE_URL }}
job-secret: ${{ secrets.JOB_SECRET }}
label: ${{ needs.setup-instance.outputs.runner-name }}
cd /concrete/frontends/concrete-python
make venv
source .venv/bin/activate
cd /concrete/compilers/concrete-compiler/compiler
make BUILD_DIR=/build CCACHE=ON DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=$(which python3) CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} python-bindings
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Test gpu
uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3
if: ${{ contains(inputs.instance_type, 'p3') }}
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}:/concrete
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
--gpus all
shell: bash
run: |
set -e
cd /concrete/frontends/concrete-python
make venv
source .venv/bin/activate
export COMPILER_BUILD_DIRECTORY=/build
KEY_CACHE_DIRECTORY=/tmp/KeySetCache mkdir ./KeySetCache
make pytest-gpu
chmod -R ugo+rwx /tmp/KeySetCache
- name: Slack Notification
if: ${{ failure() }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Instance teardown finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

18
.github/workflows/docker-lint.yml vendored
View File

@@ -1,18 +0,0 @@
name: Lint Dockerfiles
on:
pull_request:
push:
branches:
- main
jobs:
lint:
runs-on: ubuntu-latest
container:
image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Lint All Dockerfiles (except third_party)
run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs `

29
.github/workflows/docker_compliance.yml vendored Normal file
View File

@@ -0,0 +1,29 @@
name: check docker files compliance
on:
pull_request:
paths:
- .github/workflows/docker_compliance.yml
- '**Dockerfile**'
push:
branches:
- main
- 'release/*'
jobs:
lint:
runs-on: ubuntu-latest
container:
image: hadolint/hadolint@sha256:27173fe25e062448490a32de410c08491c626a0bef360aa2ce5d5bdd9384b50d #2.12.0-debian
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Lint All Dockerfiles (except third_party)
run: hadolint -V `find -name "*Dockerfile*" -not -path "./third_party/*" |xargs `
- name: Slack Notification
if: ${{ failure() && github.ref == 'refs/heads/main' }}
continue-on-error: true
uses: rtCamp/action-slack-notify@4e5fb42d249be6a45a298f3c9543b111b02f7907
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "lint finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})"

18
.github/workflows/linelint.yml vendored
View File

@@ -1,18 +0,0 @@
# This job is the main jobs will dispatch build and test for every modules of our mono repo.
name: Linelint
on:
pull_request:
push:
branches:
- 'main'
jobs:
linelint:
runs-on: ubuntu-20.04
steps:
- name: Checkout
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- name: Linelint
uses: fernandrone/linelint@8136e0fa9997122d80f5f793e0bb9a45e678fbb1 # 0.0.4
id: linelint

303
.github/workflows/main.yml vendored
View File

@@ -1,303 +0,0 @@
# This job is the main jobs will dispatch build and test for every modules of our mono repo.
name: Main
on:
pull_request:
push:
branches:
- 'main'
- 'release/*'
- 'force-docker-images'
- 'private_release/*'
tags:
- 'v[0-9]+.[0-9]+.[0-9]+*'
schedule:
# Nightly Release @ 3AM after each work day
- cron: "0 3 * * 2-6"
jobs:
# This jobs outputs for each modules of our mono-repo if it changed,
# in order to launch jobs only for the changed modules
file-change:
if: ${{ github.event_name != 'schedule' }}
runs-on: ubuntu-latest
outputs:
compiler: ${{ steps.compiler.outputs.any_changed }}
optimizer: ${{ steps.optimizer.outputs.any_changed }}
concrete-cpu: ${{ steps.concrete-cpu.outputs.any_changed }}
concrete-cpu-api: ${{ steps.concrete-cpu-api.outputs.any_changed }}
concrete-cuda-api: ${{ steps.concrete-cuda-api.outputs.any_changed }}
concrete-python: ${{ steps.concrete-python.outputs.any_changed }}
concrete-compiler-cpu-workflow: ${{ steps.concrete-compiler-cpu-workflow.outputs.any_changed }}
concrete-compiler-gpu-workflow: ${{ steps.concrete-compiler-gpu-workflow.outputs.any_changed }}
concrete-compiler-format-and-linting-workflow: ${{ steps.concrete-compiler-format-and-linting-workflow.outputs.any_changed }}
concrete-compiler-macos-workflow: ${{ steps.concrete-compiler-macos-workflow.outputs.any_changed }}
concrete-compiler-docker-images-workflow: ${{ steps.concrete-compiler-docker-images-workflow.outputs.any_changed }}
concrete-cpu-workflow: ${{ steps.concrete-cpu-workflow.outputs.any_changed }}
concrete-python-workflow: ${{ steps.concrete-python-workflow.outputs.any_changed }}
concrete-optimizer-workflow: ${{ steps.concrete-optimizer-workflow.outputs.any_changed }}
push-main: ${{ steps.github.outputs.push-main }}
steps:
- name: Checkout the repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Get changed files in the concrete-compiler directory
id: compiler
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./compilers/concrete-compiler/**
- name: Get changed files for concrete-optimizer
id: optimizer
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: |
./tools/parameter-curves/concrete-security-curves-rust/**
./compilers/concrete-optimizer/**
./.github/workflows/optimizer.yml
- name: Get changed files in the concrete-cpu directory
id: concrete-cpu
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./backends/concrete-cpu/implementation/**
- name: Get changed files in the concrete-python directory
id: concrete-python
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./frontends/concrete-python/**
- name: Check if compiler_build_and_test_cpu workflow has changed
id: concrete-compiler-cpu-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/compiler_build_and_test_cpu.yml
- name: Check if compiler_build_and_test_gpu workflow has changed
id: concrete-compiler-gpu-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/compiler_build_and_test_gpu.yml
- name: Check if compiler_format_and_linting.yml workflow has changed
id: concrete-compiler-format-and-linting-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/compiler_format_and_linting.yml
- name: Check if compiler_macos_build_and_test workflow has changed
id: concrete-compiler-macos-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/compiler_macos_build_and_test.yml
- name: Check if compiler_publish_docker_images workflow has changed
id: concrete-compiler-docker-images-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: |
./.github/workflows/compiler_publish_docker_images.yml
./docker/**
- name: Check if concrete_cpu_test workflow has changed
id: concrete-cpu-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/concrete_cpu_test.yml
- name: Check if concrete_python_checks workflow has changed
id: concrete-python-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/concrete_python_checks.yml
- name: Check if optimizer workflow has changed
id: concrete-optimizer-workflow
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./.github/workflows/optimizer.yml
- name: Get changed files in the concrete-cpu directory
id: concrete-cpu-api
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./backends/concrete-cpu/implementation/include/**
- name: Get changed files in the concrete-cuda directory
id: concrete-cuda-api
uses: tj-actions/changed-files@e9772d140489982e0e3704fea5ee93d536f1e275
with:
files: ./backends/concrete-cuda/implementation/include/**
- name: Set some github event outputs
id: github
if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release'))
run: echo "push-main=true" >> "$GITHUB_OUTPUT"
#################################################
# Compiler jobs #################################
compiler-compliance:
needs: file-change
if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-format-and-linting-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/compiler_format_and_linting.yml
compiler-cpu-build:
needs: file-change
if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-cpu-build
compiler-cpu-build-distributed:
needs: file-change
if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cpu-api == 'true'|| needs.file-change.outputs.concrete-compiler-cpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-cpu-build-distributed
compiler-gpu-build:
needs: file-change
if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-cuda-api == 'true' || needs.file-change.outputs.concrete-compiler-gpu-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-gpu-build
compiler-macos-tests:
needs: file-change
if: needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-macos-workflow == 'true' || needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/compiler_macos_build_and_test.yml
secrets: inherit
compiler-publish-docker-images:
needs: file-change
if: (needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.concrete-compiler-docker-images-workflow == 'true') && (needs.file-change.outputs.push-main == 'true' || contains(github.ref, 'refs/heads/force-docker-images'))
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-publish-docker-images
compiler-cpu-benchmark:
needs: file-change
if: needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-cpu-benchmark
# compiler-gpu-benchmark:
# needs: file-change
# if: needs.file-change.outputs.push-main == 'true'
# uses: ./.github/workflows/start_slab.yml
# secrets: inherit
# with:
# command: compiler-gpu-benchmark
#################################################
# Optimizer jobs ################################
optimizer:
needs: file-change
if: |
needs.file-change.outputs.parameters-curves == 'true' ||
needs.file-change.outputs.concrete-cpu == 'true' ||
needs.file-change.outputs.optimizer == 'true'||
needs.file-change.outputs.push-main
uses: ./.github/workflows/optimizer.yml
secrets: inherit
#################################################
# ConcreteCPU jobs ##############################
concrete-cpu:
needs: file-change
if: needs.file-change.outputs.concrete-cpu == 'true' || needs.file-change.outputs.concrete-cpu-workflow == 'true' || needs.file-change.outputs.push-main
uses: ./.github/workflows/concrete_cpu_test.yml
secrets: inherit
#################################################
# Concrete Python jobs ##########################
concrete-python:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.concrete-python-workflow == 'true' || needs.file-change.outputs.push-main
uses: ./.github/workflows/concrete_python_checks.yml
secrets: inherit
concrete-python-tests-linux:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-tests-linux
concrete-python-tests-linux-gpu:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true' && needs.file-change.outputs.push-main
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-tests-linux-gpu
concrete-python-tests-macos:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.push-main
uses: ./.github/workflows/concrete_python_test_macos.yml
secrets: inherit
#################################################
# Concrete-ML tests #############################
concrete-ml-tests-linux:
needs: file-change
if: needs.file-change.outputs.concrete-python == 'true' || needs.file-change.outputs.compiler == 'true' || needs.file-change.outputs.push-main
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: ml-test
#################################################
# Release jobs #################################
concrete-python-nightly-release:
if: ${{ github.event_name == 'schedule' }}
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-release
user_inputs: 'nightly'
concrete-python-nightly-release-gpu:
if: ${{ github.event_name == 'schedule' }}
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-release-gpu
user_inputs: 'nightly'
concrete-python-public-release:
# needs: [compiler-cpu-build, compiler-macos-tests, compiler-publish-docker-images, concrete-python-tests-linux, concrete-python-tests-macos]
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-release
user_inputs: 'public'
concrete-python-public-release-gpu:
if: github.event_name == 'push' && contains(github.ref, 'refs/tags/')
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-release-gpu
user_inputs: 'public'
concrete-python-private-release:
if: github.event_name == 'push' && contains(github.ref, 'refs/heads/private_release/')
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: concrete-python-release
user_inputs: 'private'

20
.github/workflows/markdown_link_check.yml vendored
View File

@@ -1,20 +0,0 @@
name: Check Markdown links
on:
pull_request:
paths:
- '**.md'
- .github/workflows/markdown_link_check.yml
push:
branches:
- main
jobs:
markdown-link-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: gaurav-nelson/github-action-markdown-link-check@5c5dfc0ac2e225883c0e5f03a85311ec2830d368 # v1
with:
use-quiet-mode: 'yes'
use-verbose-mode: 'yes'

7
.github/workflows/optimizer_setup/action.yml vendored
View File

@@ -1,8 +1,3 @@
inputs:
ssh_private_key:
description: 'A ssh key to access private github repository'
required: true
runs:
using: "composite"
steps:
@@ -10,6 +5,8 @@ runs:
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1.0.7
with:
toolchain: stable
default: true
override: true
- name: Download cargo cache
uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3

35
.github/workflows/push_wheels_to_public_pypi.yml vendored
View File

@@ -1,35 +0,0 @@
name: Push Wheels to Public PyPI
on:
workflow_dispatch:
inputs:
version:
description: 'version of concrete-python to pull from Zama PyPI and push to public PyPI. Use the version as it appears in the wheel file (e.g. 2.7.0rc1)'
required: true
type: string
jobs:
pull_and_push:
runs-on: ubuntu-latest
steps:
- name: Pull wheels from S3
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_IAM_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_IAM_KEY }}
AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
S3_BUCKET_NAME: ${{ secrets.AWS_S3_PYPI_BUCKET_NAME }}
run: |
mkdir wheels
aws s3 cp s3://${S3_BUCKET_NAME}/cpu/concrete-python/ ./wheels/ --recursive --exclude "*" --include "concrete_python-${{ inputs.version }}-*"
echo "============== Downloaded wheels ==============="
ls -la ./wheels/
- name: Push wheels to public PyPI (public)
run: |
pip install twine==4.0.2
twine upload wheels/concrete_python-${{ inputs.version }}*.whl \
-u "${{ secrets.PUBLIC_PYPI_USER }}" \
-p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" \
-r pypi

10
.github/workflows/scripts/teardown-check.sh vendored Executable file
View File

@@ -0,0 +1,10 @@
#!/bin/bash -e
grep setup-instance -Rl .github/workflows/ | xargs grep -L teardown-instance &> missing-teardown.txt
if [ -s missing-teardown.txt ]; then
echo "There are missing teardown-instance jobs in following jobs:"
echo
cat missing-teardown.txt
exit 1
fi

62
.github/workflows/start_slab.yml vendored
View File

@@ -1,62 +0,0 @@
# Start job on Slab CI bot given by input command.
name: Start AWS job
on:
workflow_call:
inputs:
command:
required: true
type: string
user_inputs:
required: false
type: string
workflow_dispatch:
inputs:
command:
required: true
type: string
user_inputs:
description: 'user inputs to be forwarded to the called workflow'
required: false
type: string
env:
GIT_REF: ${{ github.head_ref }}
jobs:
sl:
runs-on: ubuntu-latest
steps:
- name: Checkout concrete
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
fetch-depth: 0
- name: Get git ref
# github.head_ref is only available from a Pull Request
if: env.GIT_REF == ''
run: |
echo "GIT_REF=${{ github.ref_name }}" >> $GITHUB_ENV
- name: Checkout Slab repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: zama-ai/slab
path: slab
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Start AWS job in Slab
shell: bash
run: |
GIT_SHA="$(git --no-pager show -s --format="%H" origin/${{ env.GIT_REF }})" || GIT_SHA=${{ github.sha }}
echo -n '{"command": "${{ inputs.command }}", "git_ref": "${{ env.GIT_REF }}", "sha":"'${GIT_SHA}'", "user_inputs": "${{ inputs.user_inputs }}"}' > command.json
cat command.json
SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')"
curl -v -k \
--fail-with-body \
-H "Content-Type: application/json" \
-H "X-Slab-Repository: ${{ github.repository }}" \
-H "X-Slab-Command: start_aws" \
-H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-d @command.json \
${{ secrets.SLAB_URL }}

1
ci/ec2_products_cost.json
View File

@@ -1,4 +1,5 @@
{
"hpc7a.96xlarge": 7.200,
"m7i.48xlarge": 9.677,
"m7i.metal-48xl": 9.677,
"m6i.metal": 7.168,

127
ci/slab.toml
View File

@@ -1,136 +1,29 @@
# This is the new version of Slab that handles multi backend providers.
[backend.aws.m7i-cpu-bench]
[backend.aws.cpu-test]
region = "eu-west-1"
image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4
instance_type = "m7i.48xlarge"
instance_type = "m7i.16xlarge"
security_group = ["sg-0e55cc31dfda0d8a7", ]
[profile.m7i-cpu-bench]
region = "eu-west-1"
image_id = "ami-002bdcd64b8472cf9" # Based on Ubuntu 22.4
instance_type = "m7i.48xlarge"
security_group= ["sg-0e55cc31dfda0d8a7", ]
[profile.m7i-cpu-test]
[backend.aws.cpu-bench]
region = "eu-west-1"
image_id = "ami-002bdcd64b8472cf9"
instance_type = "m7i.16xlarge"
security_group= ["sg-0e55cc31dfda0d8a7", ]
instance_type = "hpc7a.96xlarge"
[profile.m7i-metal]
region = "eu-west-1"
image_id = "ami-002bdcd64b8472cf9"
instance_type = "m7i.metal-24xl"
security_group= ["sg-0e55cc31dfda0d8a7", ]
[profile.gpu-bench]
region = "us-east-1"
image_id = "ami-08e27480d79e82238"
instance_type = "p3.2xlarge"
subnet_id = "subnet-8123c9e7"
security_group= ["sg-017afab1f328af917", ]
# Docker is well configured for test inside docker in this AMI
[profile.gpu-test]
[backend.aws.gpu-test]
region = "us-east-1"
image_id = "ami-0257c6ad39f902b5e"
instance_type = "p3.2xlarge"
subnet_id = "subnet-8123c9e7"
security_group= ["sg-017afab1f328af917", ]
# It has CUDA Driver (<=12.5) and Docker installed
[profile.gpu-test-ubuntu22]
region = "us-east-1"
image_id = "ami-05385e0c3c574621f"
instance_type = "p3.2xlarge"
subnet_id = "subnet-8123c9e7"
security_group= ["sg-017afab1f328af917", ]
[profile.slurm-cluster]
[backend.aws.slurm-cluster]
region = "eu-west-3"
image_id = "ami-0bb5bb9cb747b5ddd"
instance_id = "i-0e5ae2a14134d6275"
instance_type = "m6i.8xlarge"
security_group= ["sg-02dd8470fa845f31b", ]
#################################################
# Compiler commands
#################################################
[command.compiler-cpu-build]
workflow = "compiler_build_and_test_cpu.yml"
profile = "m7i-cpu-test"
check_run_name = "Compiler Build and Test (CPU)"
[command.compiler-cpu-build-distributed]
workflow = "compiler_build_and_test_cpu_distributed.yml"
profile = "slurm-cluster"
check_run_name = "Compiler Distributed Build and Test (CPU)"
runner_name = "distributed-ci"
[command.compiler-gpu-build]
workflow = "compiler_build_and_test_gpu.yml"
profile = "gpu-test"
check_run_name = "Compiler Build and Test (GPU)"
[command.compiler-cpu-benchmark]
workflow = "compiler_benchmark.yml"
profile = "m7i-cpu-bench"
check_run_name = "Compiler Performances Benchmarks (CPU)"
[command.compiler-gpu-benchmark]
workflow = "compiler_benchmark.yml"
profile = "gpu-bench"
check_run_name = "Compiler Performances Benchmarks (GPU)"
# Trigger Docker images build
[command.compiler-publish-docker-images]
workflow = "compiler_publish_docker_images.yml"
profile = "m7i-cpu-test"
check_run_name = "Compiler - Docker images build & publish"
# Trigger ML benchmarks by running each use cases subset in parallel.
[command.ml-bench]
workflow = "ml_benchmark_subset.yml"
profile = "m7i-cpu-bench"
matrix = [0,1,2,3,4,5,6,7,8,9,10]
max_parallel_jobs = 2
# Trigger ML tests with latest CP
[command.ml-test]
workflow = "concrete_ml_tests.yml"
profile = "m7i-cpu-test"
check_run_name = "Concrete ML Tests"
#################################################
# Concrete Python Commands
#################################################
[command.concrete-python-tests-linux]
workflow = "concrete_python_tests_linux.yml"
profile = "m7i-cpu-test"
check_run_name = "Concrete Python Tests (Linux)"
[command.concrete-python-tests-linux-gpu]
workflow = "concrete_python_tests_linux.yml"
profile = "gpu-test"
check_run_name = "Concrete Python Tests (Linux Gpu)"
#################################################
# Release Commands
#################################################
[command.concrete-python-release]
workflow = "concrete_python_release.yml"
profile = "m7i-cpu-test"
check_run_name = "Concrete Python Release"
[command.concrete-python-release-gpu]
workflow = "concrete_python_release_gpu.yml"
profile = "m7i-cpu-test"
check_run_name = "Concrete Python Release (GPU)"
[command.concrete-python-test-gpu-wheel]
workflow = "concrete_python_test_gpu_wheel.yml"
profile = "gpu-test"
check_run_name = "Concrete Python Test GPU Wheel"
[backend.aws.release]
region = "eu-west-1"
image_id = "ami-002bdcd64b8472cf9"
instance_type = "hpc7a.96xlarge"

27
compilers/concrete-compiler/compiler/Makefile
View File

@@ -479,33 +479,6 @@ else
detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown')
endif
PIP=$(Python3_EXECUTABLE) -m pip
PIP_WHEEL=$(PIP) wheel --no-deps -w $(BUILD_DIR)/wheels .
AUDIT_WHEEL_REPAIR=$(Python3_EXECUTABLE) -m auditwheel repair -w $(BUILD_DIR)/wheels
linux-python-package:
$(PIP) install wheel auditwheel
# We need to run it twice: the first will generate the directories, so that
# the second run can find the packages via find_namespace_packages
$(PIP_WHEEL)
$(PIP_WHEEL)
GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_'); \
for PLATFORM in manylinux_$${GLIBC_VER}_x86_64 linux_x86_64; do \
if $(AUDIT_WHEEL_REPAIR) $(BUILD_DIR)/wheels/*.whl --plat $$PLATFORM; then \
echo Success for $$PLATFORM; \
break; \
else \
echo No repair with $$PLATFORM; \
fi \
done
darwin-python-package:
$(PIP) install wheel delocate
$(PIP_WHEEL)
delocate-wheel -v $(BUILD_DIR)/wheels/*macosx*.whl
python-package: python-bindings $(OS)-python-package
@echo The python package is: $(BUILD_DIR)/wheels/*.whl
install: concretecompiler install-deps
$(info Install prefix set to $(INSTALL_PREFIX))

1
compilers/concrete-compiler/compiler/lib/Bindings/Python/requirements_dev.txt
View File

@@ -1,3 +1,4 @@
black==24.4.0
pylint==2.11.1
mypy==1.11.2
numpy>=1.23,<2.0

3
frontends/concrete-python/Makefile
View File

@@ -30,9 +30,6 @@ CONCRETE_VERSION?="" # empty mean latest
venv:
$(PYTHON) -m venv .venv
. .venv/bin/activate
ifeq (,$(wildcard ${RUNTIME_LIBRARY}))
$(PIP) install --extra-index-url https://pypi.zama.ai/cpu "concrete-python$(CONCRETE_VERSION)"
endif
$(PIP) install -r requirements.dev.txt
$(PIP) install -r requirements.extra-full.txt
$(PIP) install -r requirements.txt

8
frontends/concrete-python/scripts/checks/checks.sh
View File

@@ -1,8 +0,0 @@
#!/bin/bash
set -ex
cd frontends/concrete-python
make venv
source .venv/bin/activate
make pcc