chore(ci): Cleanup ci

This commit is contained in:
Quentin Bourgerie
2023-03-14 13:55:34 +01:00
parent 2a3a0907d8
commit 67a4ba3d1b
14 changed files with 52 additions and 477 deletions

View File

@@ -1,14 +0,0 @@
# https://github.com/styfle/cancel-workflow-action#user-content-advanced-canceling-other-workflows
name: Auto-cancel-outdated PR
on:
pull_request:
types: [opened, synchronize, reopened]
jobs:
cancel:
name: Auto-cancel-outdated PR
runs-on: ubuntu-latest
steps:
- uses: styfle/cancel-workflow-action@0.10.0
with:
workflow_id: continuous-integration.yml
access_token: ${{ github.token }}

View File

@@ -20,6 +20,10 @@ on:
description: 'Slab request ID'
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
CARGO_TERM_COLOR: always
RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json

View File

@@ -22,6 +22,10 @@ on:
description: 'Build matrix item'
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
@@ -29,9 +33,6 @@ env:
jobs:
BuildAndTest:
name: Build and test compiler in EC2
concurrency:
group: ${{ github.ref }}_${{ github.event.inputs.instance_image_id }}_${{ github.event.inputs.instance_type }}
cancel-in-progress: true
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
steps:

View File

@@ -22,6 +22,10 @@ on:
description: 'Build matrix item'
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
CUDA_PATH: /usr/local/cuda-11.7
@@ -30,9 +34,6 @@ env:
jobs:
BuildAndTest:
name: Build and test compiler in EC2 with CUDA support
concurrency:
group: ${{ github.ref }}_${{ github.event.inputs.instance_image_id }}_${{ github.event.inputs.instance_type }}
cancel-in-progress: true
runs-on: ${{ github.event.inputs.runner_name }}
if: ${{ !cancelled() }}
steps:

View File

@@ -10,6 +10,10 @@ on:
CONCRETE_ACTIONS_TOKEN:
required: true
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
jobs:
BuildAndTestMacOS:
runs-on: macos-11

View File

@@ -23,6 +23,10 @@ on:
description: 'Build matrix item'
type: string
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
BuildAndPushDockerImages:
needs: [BuildAndPublishHPXDockerImage, BuildAndPublishCUDADockerImage]

View File

@@ -1,36 +0,0 @@
# Trigger benchmarks jobs on Slab CI bot.
name: Compiler - Trigger AWS benchmarks
on:
push:
branches:
- 'main'
workflow_dispatch:
jobs:
trigger-benchmarks:
strategy:
matrix:
command: [compiler-cpu-benchmark, compiler-gpu-benchmark]
runs-on: ubuntu-latest
steps:
- name: Checkout Slab repo
uses: actions/checkout@v3
with:
repository: zama-ai/slab
path: slab
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Start AWS job in Slab
shell: bash
# TODO: step result must be correlated to HTTP return code.
run: |
echo -n '{"command": "${{ matrix.command }}", "git_ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}' > command.json
SIGNATURE="$(slab/scripts/hmac_calculator.sh command.json '${{ secrets.JOB_SECRET }}')"
curl -v -k \
-H "Content-Type: application/json" \
-H "X-Slab-Repository: ${{ github.repository }}" \
-H "X-Slab-Command: start_aws" \
-H "X-Hub-Signature-256: sha256=${SIGNATURE}" \
-d @command.json \
${{ secrets.SLAB_URL }}

View File

@@ -4,6 +4,10 @@ on:
workflow_call:
workflow_dispatch:
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
CARGO_TERM_COLOR: always
jobs:

View File

@@ -46,8 +46,8 @@ jobs:
run-cuda-tests-linux:
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
name: Test code in EC2
needs: start-runner
runs-on: ${{ needs.start-runner.outputs.label }}

View File

@@ -4,8 +4,8 @@ on:
workflow_dispatch:
concurrency:
group: concrete-python-${{ github.ref }}
cancel-in-progress: true
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}

View File

@@ -7,6 +7,10 @@ on:
branches:
- 'main'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
jobs:
# This jobs outputs for each modules of our mono-repo if it changed,
# in order to launch jobs only for the changed modules
@@ -108,6 +112,22 @@ jobs:
with:
command: compiler-publish-docker-images
compiler-cpu-benchmark:
needs: file-change
if: needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-cpu-benchmark
compiler-gpu-benchmark:
needs: file-change
if: needs.file-change.outputs.push-main == 'true'
uses: ./.github/workflows/start_slab.yml
secrets: inherit
with:
command: compiler-gpu-benchmark
#################################################
# Optimizer jobs ################################
optimizer:

View File

@@ -9,6 +9,10 @@ on:
CONCRETE_ACTIONS_TOKEN:
required: true
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }}
env:
CARGO_TERM_COLOR: always
jobs:

View File

@@ -1,371 +0,0 @@
# Prepare release packages for GNU/Linux and MacOS on private stores.
name: Prepare Release
on:
push:
tags:
- 'v[0-9]+.*'
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
jobs:
WaitOnAllWorkflows:
runs-on: ubuntu-latest
if: ${{ startsWith(github.ref_name, 'v') }}
steps:
- uses: actions/checkout@v2
- uses: ahmadnassri/action-workflow-run-wait@v1
CreateRelease:
runs-on: ubuntu-20.04
needs: WaitOnAllWorkflows
outputs:
upload_url: ${{ steps.release.outputs.upload_url }}
release_id: ${{ steps.release.outputs.id }}
steps:
- name: Release
id: release
uses: softprops/action-gh-release@v1
with:
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
draft: true
prerelease: true
generate_release_notes: true
BuildAndPushPythonPackagesLinux:
runs-on: ubuntu-20.04
strategy:
matrix:
include:
- python: 37
python_dir: "cp37-cp37m"
- python: 38
python_dir: "cp38-cp38"
- python: 39
python_dir: "cp39-cp39"
- python: 310
python_dir: "cp310-cp310"
outputs:
python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }}
python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }}
python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }}
python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }}
needs: CreateRelease
steps:
# A SSH private key is required as some dependencies are from private repos
- uses: webfactory/ssh-agent@v0.5.2
with:
ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
- uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Update Python Version
run: cd compiler && make update-python-version
- name: Login to Github Container Registry
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Set up ssh auth in docker
run: |
echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}"
- name: Build Wheel
uses: addnab/docker-run-action@v3
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}/llvm-project:/llvm-project
-v ${{ github.workspace }}/compiler:/compiler
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
shell: bash
run: |
set -e
cd /compiler
rm -rf /build
export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python
$PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt
# setup env variable for wheel building
export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC
export CONCRETE_COMPILER_BUILD_DIR=/build
export CONCRETE_COMPILER_CUDA_SUPPORT=ON
export PATH=/opt/python/${{ matrix.python_dir }}/bin:$PATH
make python-package
cp build/wheels/*manylinux_${{ env.GLIB_VER }}_x86_64.whl /wheels
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Set Outputs
id: set-output-wheel-linux
run: |
echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )"
# used later for python package test
echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )"
- name: Upload Python Package
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
with:
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
asset_content_type: application/zip
BuildAndPushTarballLinux:
runs-on: ubuntu-20.04
needs: CreateRelease
steps:
# A SSH private key is required as some dependencies are from private repos
- uses: webfactory/ssh-agent@v0.6.0
with:
ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
- uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Login to Github Container Registry
run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
- name: Build Tarball
uses: addnab/docker-run-action@v3
with:
registry: ghcr.io
image: ${{ env.DOCKER_IMAGE_TEST }}
username: ${{ secrets.GHCR_LOGIN }}
password: ${{ secrets.GHCR_PASSWORD }}
options: >-
-v ${{ github.workspace }}/llvm-project:/llvm-project
-v ${{ github.workspace }}/compiler:/compiler
-v ${{ github.workspace }}/tarballs:/tarballs
-v ${{ github.workspace }}/.github/workflows/assets/Installation.md:/Installation.md
shell: bash
run: |
set -e
cd /compiler
rm -rf /build
make BINDINGS_PYTHON_ENABLED=OFF BUILD_DIR=/build INSTALL_PREFIX=/tarballs/ install
echo "Debug: ccache statistics (after the build):"
ccache -s
# package installation file and make tarball
cp /Installation.md /tarballs/concretecompiler/
cd /tarballs && tar -czvf concretecompiler.tar.gz concretecompiler
- name: Tag Tarball
id: tag-tarball
run: |
TAG="$(git describe --tags --abbrev=0)"
sudo cp "${{ github.workspace }}/tarballs/concretecompiler.tar.gz" "${{ github.workspace }}/tarballs/concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz"
echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz"
- name: Upload Tarball
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
with:
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
asset_path: ${{ github.workspace }}/tarballs/${{ steps.tag-tarball.outputs.ASSET_NAME }}
asset_name: ${{ steps.tag-tarball.outputs.ASSET_NAME }}
asset_content_type: application/tar+gzip
BuildAndPushPackagesMacOS:
needs: CreateRelease
runs-on: macos-11
strategy:
matrix:
python: [ '3.8', '3.9', '3.10' ]
outputs:
python-package-name-macos-py38: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY38 }}
python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }}
python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }}
steps:
# A SSH private key is required as some dependencies are from private repos
- uses: webfactory/ssh-agent@v0.5.0
with:
ssh-private-key: ${{ secrets.CONCRETE_CI_SSH_PRIVATE }}
- uses: actions/checkout@v3
with:
submodules: recursive
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
- name: Install Rust
uses: actions-rs/toolchain@v1
with:
toolchain: stable
- name: Concrete-Optimizer
run: |
cd compiler
make concrete-optimizer-lib
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install Deps
run: |
brew install ninja ccache
pip install numpy pybind11==2.8 wheel delocate
- name: Update Python Version
run: cd compiler && make update-python-version
- name: Use Compilation Cache
uses: actions/cache@v3
with:
path: /Users/runner/Library/Caches/ccache
key: ${{ runner.os }}-compilation-cache-${{ github.sha }}
restore-keys: |
${{ runner.os }}-compilation-cache-
- name: Build
id: build-wheel-macos
run: |
cd compiler
make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
make python-package
cp -R build/wheels ${{ github.workspace }}/wheels
cd ${{ github.workspace }}/wheels/
WHEEL=$(ls *macosx*.whl)
echo "::set-output name=ASSET_NAME::$WHEEL"
# used later for python package test
echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$WHEEL"
- name: Upload Python Package
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
with:
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
asset_name: ${{ steps.build-wheel-macos.outputs.ASSET_NAME }}
asset_content_type: application/zip
- name: Build tarball
if: matrix.python == '3.8'
id: build-mac-tarball
run: |
cd compiler
make INSTALL_PREFIX=./tarballs/ install
cp ../.github/workflows/assets/Installation.md ./tarballs/concretecompiler/
TAG=$(git describe --tags --abbrev=0)
cd ./tarballs && tar -czvf "concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" concretecompiler
echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz"
- name: Upload Tarball
if: matrix.python == '3.8'
uses: actions/upload-release-asset@v1
env:
GITHUB_TOKEN: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
with:
upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
asset_path: ${{ github.workspace }}/compiler/tarballs/${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
asset_name: ${{ steps.build-mac-tarball.outputs.ASSET_NAME }}
asset_content_type: application/tar+gzip
TestPythonPackageLinux:
runs-on: ubuntu-20.04
needs: BuildAndPushPythonPackagesLinux
strategy:
matrix:
include:
- python: '3.7'
filename-index: 'python-package-name-linux-py37'
- python: '3.8'
filename-index: 'python-package-name-linux-py38'
- python: '3.9'
filename-index: 'python-package-name-linux-py39'
- python: '3.10'
filename-index: 'python-package-name-linux-py310'
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Extract Package Filename
id: extract-filename
run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPythonPackagesLinux.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )"
- name: Download and Install Package
run: |
FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.CONCRETE_ACTIONS_TOKEN }}" \
https://api.github.com/repos/${{ github.repository }}/releases | \
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
jq '.[0].assets' | \
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
jq '.[].id')
wget --auth-no-challenge --header='Accept:application/octet-stream' \
"https://${{ secrets.CONCRETE_ACTIONS_TOKEN }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \
-O ${{ steps.extract-filename.outputs.FILE_NAME }}
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
- name: Test
run: |
cd compiler
pip install pytest
pytest -vs tests/python
TestPythonPackageMacOS:
runs-on: macos-11
needs: BuildAndPushPackagesMacOS
env:
SYSTEM_VERSION_COMPAT: 0
strategy:
matrix:
include:
- python: '3.8'
filename-index: 'python-package-name-macos-py38'
- python: '3.9'
filename-index: 'python-package-name-macos-py39'
- python: '3.10'
filename-index: 'python-package-name-macos-py310'
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Extract Package Filename
id: extract-filename
run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPackagesMacOS.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )"
- name: Download and Install Package
run: |
FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.CONCRETE_ACTIONS_TOKEN }}" \
https://api.github.com/repos/${{ github.repository }}/releases | \
jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \
jq '.[0].assets' | \
jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \
jq '.[].id')
wget --auth-no-challenge --header='Accept:application/octet-stream' \
"https://${{ secrets.CONCRETE_ACTIONS_TOKEN }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \
-O ${{ steps.extract-filename.outputs.FILE_NAME }}
pip install ${{ steps.extract-filename.outputs.FILE_NAME }}
- name: Test
run: |
cd compiler
pip install pytest
pytest -vs -m "not parallel" tests/python

View File

@@ -1,46 +0,0 @@
name: Push Python Packages
on:
workflow_dispatch:
inputs:
internal_pypi:
description: 'Whether to push to internal pypi'
default: 'false'
required: true
type: boolean
public_pypi:
description: 'Whether to push to public pypi'
default: 'false'
required: true
type: boolean
tag:
description: 'Release tag to push'
required: true
jobs:
PushPackages:
runs-on: ubuntu-latest
steps:
- name: Download release assets
uses: duhow/download-github-release-assets@v1
with:
token: ${{ secrets.CONCRETE_ACTIONS_TOKEN }}
repository: ${{ github.repository }}
tag: ${{ github.event.inputs.tag }}
files: '*'
target: ${{ github.workspace }}/release/
- name: Install dependencies
if: ${{ github.event.inputs.internal_pypi || github.event.inputs.public_pypi }}
run: |
python -m pip install --upgrade pip
pip install twine
- name: Push packages to internal pypi
if: ${{ github.event.inputs.internal_pypi }}
run: twine upload -u ${{ secrets.INTERNAL_PYPI_USER }} -p "${{ secrets.INTERNAL_PYPI_PASSWORD }}" --repository-url ${{ secrets.INTERNAL_PYPI_URL }} ${{ github.workspace }}/release/*.whl
- name: Push packages to public pypi
if: ${{ github.event.inputs.public_pypi }}
run: twine upload -u ${{ secrets.PUBLIC_PYPI_USER }} -p "${{ secrets.PUBLIC_PYPI_PASSWORD }}" -r pypi ${{ github.workspace }}/release/*.whl