From af265206a96ac12e4ca08a510f8194a45caf6833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Test=C3=A9?= Date: Thu, 12 Jan 2023 12:30:28 +0100 Subject: [PATCH] chore(ci): move macos build and release jobs to their own workflow --- .github/workflows/continuous-integration.yml | 439 ------------------- .github/workflows/macos_build.yml | 85 ++++ .github/workflows/release.yml | 377 ++++++++++++++++ 3 files changed, 462 insertions(+), 439 deletions(-) create mode 100644 .github/workflows/macos_build.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 6925fea0e..cd7880c43 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -14,445 +14,6 @@ env: THIS_FILE: .github/workflows/continuous-integration.yml jobs: - ##################### - # Build and testing # - ##################### - - BuildAndTestMacOS: - runs-on: macos-11 - steps: - # A SSH private key is required as some dependencies are from private repos - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ secrets.GH_TOKEN }} - - - name: Install rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Install Deps - run: | - brew install ninja ccache - pip3.10 install numpy pybind11==2.8 wheel delocate - pip3.10 install pytest - - - name: Cache compilation (push) - if: github.event_name == 'push' - uses: actions/cache@v3 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-compilation-cache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-compilation-cache- - - - name: Cache compilation (pull_request) - if: github.event_name == 'pull_request' - uses: actions/cache@v3 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-compilation-cache-${{ github.event.pull_request.base.sha }} - restore-keys: | - ${{ runner.os }}-compilation-cache- - - - name: Get tmpdir path - if: github.event_name == 'push' - id: tmpdir-path - run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" - - # We do run run-check-tests as part of the build, as they aren't that costly - # and will at least give minimum confidence that the compiler works in PRs - - name: Build - run: | - set -e - cd compiler - echo "Debug: ccache statistics (prior to the build):" - ccache -s - make Python3_EXECUTABLE=$(which python3.10) all run-check-tests - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Test - if: github.event_name == 'push' - run: | - set -e - cd compiler - echo "Debug: ccache statistics (prior to the tests):" - ccache -s - export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip3.10 wheel --no-deps -w ${{ github.workspace }}/wheels . - delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') - pip3.10 install $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') - make Python3_EXECUTABLE=$(which python3.10) run-tests - echo "Debug: ccache statistics (after the tests):" - ccache -s - - ################################## - # Releasing and Testing Packages # - ################################## - - CreateRelease: - runs-on: ubuntu-20.04 - needs: [BuildAndTest, BuildAndTestMacOS] - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }} - outputs: - upload_url: ${{ steps.release.outputs.upload_url }} - release_id: ${{ steps.release.outputs.id }} - steps: - - name: Release - id: release - uses: softprops/action-gh-release@v1 - with: - token: ${{ secrets.GH_TOKEN_RELEASE }} - draft: true - prerelease: true - generate_release_notes: true - - BuildAndPushPythonPackagesLinux: - runs-on: ubuntu-20.04 - strategy: - matrix: - include: - - python: 37 - python_dir: "cp37-cp37m" - - python: 38 - python_dir: "cp38-cp38" - - python: 39 - python_dir: "cp39-cp39" - - python: 310 - python_dir: "cp310-cp310" - outputs: - python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }} - python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }} - python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }} - python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }} - needs: CreateRelease - steps: - # A SSH private key is required as some dependencies are from private repos - - uses: webfactory/ssh-agent@v0.5.2 - with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ secrets.GH_TOKEN }} - - - name: Update Python Version - run: cd compiler && make update-python-version - - - name: Login to Github Container Registry - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - - name: Set up ssh auth in docker - run: | - echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" - - - name: Build Wheel - uses: addnab/docker-run-action@v3 - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}/llvm-project:/llvm-project - -v ${{ github.workspace }}/compiler:/compiler - -v ${{ github.workspace }}/wheels:/wheels - -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket - -e SSH_AUTH_SOCK=/ssh.socket - shell: bash - run: | - set -e - cd /compiler - rm -rf /build - export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python - $PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt - # setup env variable for wheel building - export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC - export CONCRETE_COMPILER_BUILD_DIR=/build - /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . - # We need to run it twice: the first will generate the directories, so that - # the second run can find the packages via find_namespace_packages - /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . - auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels - echo "Debug: ccache statistics (after the build):" - ccache -s - - - name: Set Outputs - id: set-output-wheel-linux - run: | - echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )" - # used later for python package test - echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )" - - - name: Upload Python Package - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} - with: - upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} - asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} - asset_content_type: application/zip - - BuildAndPushTarballLinux: - runs-on: ubuntu-20.04 - needs: CreateRelease - steps: - # A SSH private key is required as some dependencies are from private repos - - uses: webfactory/ssh-agent@v0.6.0 - with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ secrets.GH_TOKEN }} - - - name: Login to Github Container Registry - run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io - - - name: Build Tarball - uses: addnab/docker-run-action@v3 - with: - registry: ghcr.io - image: ${{ env.DOCKER_IMAGE_TEST }} - username: ${{ secrets.GHCR_LOGIN }} - password: ${{ secrets.GHCR_PASSWORD }} - options: >- - -v ${{ github.workspace }}/llvm-project:/llvm-project - -v ${{ github.workspace }}/compiler:/compiler - -v ${{ github.workspace }}/tarballs:/tarballs - -v ${{ github.workspace }}/.github/workflows/assets/Installation.md:/Installation.md - shell: bash - run: | - set -e - cd /compiler - rm -rf /build - make BINDINGS_PYTHON_ENABLED=OFF BUILD_DIR=/build INSTALL_PREFIX=/tarballs/ install - echo "Debug: ccache statistics (after the build):" - ccache -s - # package installation file and make tarball - cp /Installation.md /tarballs/concretecompiler/ - cd /tarballs && tar -czvf concretecompiler.tar.gz concretecompiler - - - name: Tag Tarball - id: tag-tarball - run: | - TAG="$(git describe --tags --abbrev=0)" - - sudo cp "${{ github.workspace }}/tarballs/concretecompiler.tar.gz" "${{ github.workspace }}/tarballs/concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz" - echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz" - - - name: Upload Tarball - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} - with: - upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/tarballs/${{ steps.tag-tarball.outputs.ASSET_NAME }} - asset_name: ${{ steps.tag-tarball.outputs.ASSET_NAME }} - asset_content_type: application/tar+gzip - - BuildAndPushPackagesMacOS: - needs: CreateRelease - runs-on: macos-11 - strategy: - matrix: - python: ['3.8', '3.9', '3.10'] - outputs: - python-package-name-macos-py38: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY38 }} - python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }} - python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }} - steps: - # A SSH private key is required as some dependencies are from private repos - - uses: webfactory/ssh-agent@v0.5.0 - with: - ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} - - - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ secrets.GH_TOKEN }} - - - name: Install Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - - name: Concrete-Optimizer - run: | - cd compiler - make concrete-optimizer-lib - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Install Deps - run: | - brew install ninja ccache - pip install numpy pybind11==2.8 wheel delocate - - - name: Update Python Version - run: cd compiler && make update-python-version - - - name: Use Compilation Cache - uses: actions/cache@v3 - with: - path: /Users/runner/Library/Caches/ccache - key: ${{ runner.os }}-compilation-cache-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-compilation-cache- - - - name: Build - id: build-wheel-macos - run: | - cd compiler - make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings - export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip wheel --no-deps -w ${{ github.workspace }}/wheels . - delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') - echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" - # used later for python package test - echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" - - - name: Upload Python Package - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} - with: - upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-macos.outputs.ASSET_NAME }} - asset_name: ${{ steps.build-wheel-macos.outputs.ASSET_NAME }} - asset_content_type: application/zip - - - name: Build tarball - if: matrix.python == '3.8' - id: build-mac-tarball - run: | - cd compiler - make concretecompiler - mkdir -p tarballs/concretecompiler/lib tarballs/concretecompiler/bin - cp build/bin/concretecompiler tarballs/concretecompiler/bin - cp build/lib/libConcretelangRuntime.dylib tarballs/concretecompiler/lib - cp ../.github/workflows/assets/Installation.md tarballs/concretecompiler/ - - TAG=$(git describe --tags --abbrev=0) - - cd tarballs && tar -czvf "concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" concretecompiler - echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" - - - name: Upload Tarball - if: matrix.python == '3.8' - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} - with: - upload_url: ${{ needs.CreateRelease.outputs.upload_url }} - asset_path: ${{ github.workspace }}/compiler/tarballs/${{ steps.build-mac-tarball.outputs.ASSET_NAME }} - asset_name: ${{ steps.build-mac-tarball.outputs.ASSET_NAME }} - asset_content_type: application/tar+gzip - - TestPythonPackageLinux: - runs-on: ubuntu-20.04 - needs: [BuildAndPushPythonPackagesLinux, CreateRelease] - strategy: - matrix: - include: - - python: '3.7' - filename-index: 'python-package-name-linux-py37' - - python: '3.8' - filename-index: 'python-package-name-linux-py38' - - python: '3.9' - filename-index: 'python-package-name-linux-py39' - - python: '3.10' - filename-index: 'python-package-name-linux-py310' - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Extract Package Filename - id: extract-filename - run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPythonPackagesLinux.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )" - - - name: Download and Install Package - run: | - FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \ - https://api.github.com/repos/${{ github.repository }}/releases | \ - jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \ - jq '.[0].assets' | \ - jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \ - jq '.[].id') - - wget --auth-no-challenge --header='Accept:application/octet-stream' \ - "https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \ - -O ${{ steps.extract-filename.outputs.FILE_NAME }} - pip install ${{ steps.extract-filename.outputs.FILE_NAME }} - - - name: Test - run: | - cd compiler - pip install pytest - pytest -vs tests/python - - TestPythonPackageMacOS: - runs-on: macos-11 - needs: [BuildAndPushPackagesMacOS, CreateRelease] - env: - SYSTEM_VERSION_COMPAT: 0 - strategy: - matrix: - include: - - python: '3.8' - filename-index: 'python-package-name-macos-py38' - - python: '3.9' - filename-index: 'python-package-name-macos-py39' - - python: '3.10' - filename-index: 'python-package-name-macos-py310' - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - - - name: Extract Package Filename - id: extract-filename - run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPackagesMacOS.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )" - - - name: Download and Install Package - run: | - FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \ - https://api.github.com/repos/${{ github.repository }}/releases | \ - jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \ - jq '.[0].assets' | \ - jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \ - jq '.[].id') - - wget --auth-no-challenge --header='Accept:application/octet-stream' \ - "https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \ - -O ${{ steps.extract-filename.outputs.FILE_NAME }} - pip install ${{ steps.extract-filename.outputs.FILE_NAME }} - - - name: Test - run: | - cd compiler - pip install pytest - pytest -vs -m "not parallel" tests/python - ################# # Docker Images # ################# diff --git a/.github/workflows/macos_build.yml b/.github/workflows/macos_build.yml new file mode 100644 index 000000000..2446b5848 --- /dev/null +++ b/.github/workflows/macos_build.yml @@ -0,0 +1,85 @@ +# Perform a build on MacOS platform with M1 chip. +name: MacOsBuild + +on: + push: + branches: + - main + tags: + - 'v*' + pull_request: + +jobs: + BuildAndTestMacOS: + runs-on: macos-11 + steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Install rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Install Deps + run: | + brew install ninja ccache + pip3.10 install numpy pybind11==2.8 wheel delocate + pip3.10 install pytest + + - name: Cache compilation (push) + if: github.event_name == 'push' + uses: actions/cache@v3 + with: + path: /Users/runner/Library/Caches/ccache + key: ${{ runner.os }}-compilation-cache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-compilation-cache- + + - name: Cache compilation (pull_request) + if: github.event_name == 'pull_request' + uses: actions/cache@v3 + with: + path: /Users/runner/Library/Caches/ccache + key: ${{ runner.os }}-compilation-cache-${{ github.event.pull_request.base.sha }} + restore-keys: | + ${{ runner.os }}-compilation-cache- + + - name: Get tmpdir path + if: github.event_name == 'push' + id: tmpdir-path + run: echo "::set-output name=TMPDIR_PATH::$TMPDIR" + + # We do run run-check-tests as part of the build, as they aren't that costly + # and will at least give minimum confidence that the compiler works in PRs + - name: Build + run: | + set -e + cd compiler + echo "Debug: ccache statistics (prior to the build):" + ccache -s + make Python3_EXECUTABLE=$(which python3.10) all run-check-tests + echo "Debug: ccache statistics (after the build):" + ccache -s + + - name: Test + if: github.event_name == 'push' + run: | + set -e + cd compiler + echo "Debug: ccache statistics (prior to the tests):" + ccache -s + export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF + pip3.10 wheel --no-deps -w ${{ github.workspace }}/wheels . + delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') + pip3.10 install $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') + make Python3_EXECUTABLE=$(which python3.10) run-tests + echo "Debug: ccache statistics (after the tests):" + ccache -s diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 000000000..71f4651b8 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,377 @@ +# Release packages for GNU/Linux and MacOS. +name: Release + +on: + workflow_run: + workflows: [AwsEc2BuildCpu, AwsEc2BuildGpu, MacOsBuild] + branches: + - main + types: + - completed + +env: + DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + +jobs: + WaitOnAllWorkflows: + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref_name, 'v') }} + steps: + - uses: actions/checkout@v2 + - uses: ahmadnassri/action-workflow-run-wait@v1 + + CreateRelease: + runs-on: ubuntu-20.04 + needs: WaitOnAllWorkflows + outputs: + upload_url: ${{ steps.release.outputs.upload_url }} + release_id: ${{ steps.release.outputs.id }} + steps: + - name: Release + id: release + uses: softprops/action-gh-release@v1 + with: + token: ${{ secrets.GH_TOKEN_RELEASE }} + draft: true + prerelease: true + generate_release_notes: true + + BuildAndPushPythonPackagesLinux: + runs-on: ubuntu-20.04 + strategy: + matrix: + include: + - python: 37 + python_dir: "cp37-cp37m" + - python: 38 + python_dir: "cp38-cp38" + - python: 39 + python_dir: "cp39-cp39" + - python: 310 + python_dir: "cp310-cp310" + outputs: + python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }} + python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }} + python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }} + python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }} + needs: CreateRelease + steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.2 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Update Python Version + run: cd compiler && make update-python-version + + - name: Login to Github Container Registry + run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io + + - name: Set up ssh auth in docker + run: | + echo "SSH_AUTH_SOCK_DIR=$(dirname $SSH_AUTH_SOCK)" >> "${GITHUB_ENV}" + + - name: Build Wheel + uses: addnab/docker-run-action@v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}/llvm-project:/llvm-project + -v ${{ github.workspace }}/compiler:/compiler + -v ${{ github.workspace }}/wheels:/wheels + -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket + -e SSH_AUTH_SOCK=/ssh.socket + shell: bash + run: | + set -e + cd /compiler + rm -rf /build + export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python + $PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt + # setup env variable for wheel building + export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC + export CONCRETE_COMPILER_BUILD_DIR=/build + /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . + # We need to run it twice: the first will generate the directories, so that + # the second run can find the packages via find_namespace_packages + /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels . + auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels + echo "Debug: ccache statistics (after the build):" + ccache -s + + - name: Set Outputs + id: set-output-wheel-linux + run: | + echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )" + # used later for python package test + echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::$(find ${{ github.workspace }}/wheels/ -name '*manylinux*.whl' | rev |cut -d "/" -f 1 |rev )" + + - name: Upload Python Package + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} + with: + upload_url: ${{ needs.CreateRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} + asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }} + asset_content_type: application/zip + + BuildAndPushTarballLinux: + runs-on: ubuntu-20.04 + needs: CreateRelease + steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.6.0 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Login to Github Container Registry + run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io + + - name: Build Tarball + uses: addnab/docker-run-action@v3 + with: + registry: ghcr.io + image: ${{ env.DOCKER_IMAGE_TEST }} + username: ${{ secrets.GHCR_LOGIN }} + password: ${{ secrets.GHCR_PASSWORD }} + options: >- + -v ${{ github.workspace }}/llvm-project:/llvm-project + -v ${{ github.workspace }}/compiler:/compiler + -v ${{ github.workspace }}/tarballs:/tarballs + -v ${{ github.workspace }}/.github/workflows/assets/Installation.md:/Installation.md + shell: bash + run: | + set -e + cd /compiler + rm -rf /build + make BINDINGS_PYTHON_ENABLED=OFF BUILD_DIR=/build INSTALL_PREFIX=/tarballs/ install + echo "Debug: ccache statistics (after the build):" + ccache -s + # package installation file and make tarball + cp /Installation.md /tarballs/concretecompiler/ + cd /tarballs && tar -czvf concretecompiler.tar.gz concretecompiler + + - name: Tag Tarball + id: tag-tarball + run: | + TAG="$(git describe --tags --abbrev=0)" + + sudo cp "${{ github.workspace }}/tarballs/concretecompiler.tar.gz" "${{ github.workspace }}/tarballs/concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz" + echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-linux-gnu.tar.gz" + + - name: Upload Tarball + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} + with: + upload_url: ${{ needs.CreateRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/tarballs/${{ steps.tag-tarball.outputs.ASSET_NAME }} + asset_name: ${{ steps.tag-tarball.outputs.ASSET_NAME }} + asset_content_type: application/tar+gzip + + BuildAndPushPackagesMacOS: + needs: CreateRelease + runs-on: macos-11 + strategy: + matrix: + python: [ '3.8', '3.9', '3.10' ] + outputs: + python-package-name-macos-py38: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY38 }} + python-package-name-macos-py39: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY39 }} + python-package-name-macos-py310: ${{ steps.build-wheel-macos.outputs.ASSET_NAME_PY310 }} + steps: + # A SSH private key is required as some dependencies are from private repos + - uses: webfactory/ssh-agent@v0.5.0 + with: + ssh-private-key: ${{ secrets.CONCRETE_COMPILER_CI_SSH_PRIVATE }} + + - uses: actions/checkout@v3 + with: + submodules: recursive + token: ${{ secrets.GH_TOKEN }} + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + + - name: Concrete-Optimizer + run: | + cd compiler + make concrete-optimizer-lib + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Install Deps + run: | + brew install ninja ccache + pip install numpy pybind11==2.8 wheel delocate + + - name: Update Python Version + run: cd compiler && make update-python-version + + - name: Use Compilation Cache + uses: actions/cache@v3 + with: + path: /Users/runner/Library/Caches/ccache + key: ${{ runner.os }}-compilation-cache-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-compilation-cache- + + - name: Build + id: build-wheel-macos + run: | + cd compiler + make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings + export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF + pip wheel --no-deps -w ${{ github.workspace }}/wheels . + delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') + echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" + # used later for python package test + echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" + + - name: Upload Python Package + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} + with: + upload_url: ${{ needs.CreateRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-macos.outputs.ASSET_NAME }} + asset_name: ${{ steps.build-wheel-macos.outputs.ASSET_NAME }} + asset_content_type: application/zip + + - name: Build tarball + if: matrix.python == '3.8' + id: build-mac-tarball + run: | + cd compiler + make concretecompiler + mkdir -p tarballs/concretecompiler/lib tarballs/concretecompiler/bin + cp build/bin/concretecompiler tarballs/concretecompiler/bin + cp build/lib/libConcretelangRuntime.dylib tarballs/concretecompiler/lib + cp ../.github/workflows/assets/Installation.md tarballs/concretecompiler/ + + TAG=$(git describe --tags --abbrev=0) + + cd tarballs && tar -czvf "concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" concretecompiler + echo "::set-output name=ASSET_NAME::concretecompiler-${TAG}-x86_64-macos-catalina.tar.gz" + + - name: Upload Tarball + if: matrix.python == '3.8' + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }} + with: + upload_url: ${{ needs.CreateRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/compiler/tarballs/${{ steps.build-mac-tarball.outputs.ASSET_NAME }} + asset_name: ${{ steps.build-mac-tarball.outputs.ASSET_NAME }} + asset_content_type: application/tar+gzip + + TestPythonPackageLinux: + runs-on: ubuntu-20.04 + needs: BuildAndPushPythonPackagesLinux + strategy: + matrix: + include: + - python: '3.7' + filename-index: 'python-package-name-linux-py37' + - python: '3.8' + filename-index: 'python-package-name-linux-py38' + - python: '3.9' + filename-index: 'python-package-name-linux-py39' + - python: '3.10' + filename-index: 'python-package-name-linux-py310' + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Extract Package Filename + id: extract-filename + run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPythonPackagesLinux.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )" + + - name: Download and Install Package + run: | + FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \ + https://api.github.com/repos/${{ github.repository }}/releases | \ + jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \ + jq '.[0].assets' | \ + jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \ + jq '.[].id') + + wget --auth-no-challenge --header='Accept:application/octet-stream' \ + "https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \ + -O ${{ steps.extract-filename.outputs.FILE_NAME }} + pip install ${{ steps.extract-filename.outputs.FILE_NAME }} + + - name: Test + run: | + cd compiler + pip install pytest + pytest -vs tests/python + + TestPythonPackageMacOS: + runs-on: macos-11 + needs: BuildAndPushPackagesMacOS + env: + SYSTEM_VERSION_COMPAT: 0 + strategy: + matrix: + include: + - python: '3.8' + filename-index: 'python-package-name-macos-py38' + - python: '3.9' + filename-index: 'python-package-name-macos-py39' + - python: '3.10' + filename-index: 'python-package-name-macos-py310' + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + + - name: Extract Package Filename + id: extract-filename + run: echo "::set-output name=FILE_NAME::$(echo '${{ toJson(needs.BuildAndPushPackagesMacOS.outputs) }}' | jq '.["${{ matrix.filename-index }}"]' | tr -d '\"' )" + + - name: Download and Install Package + run: | + FILE_NAME=$(curl -s -u "zama-bot:${{ secrets.GH_TOKEN_RELEASE }}" \ + https://api.github.com/repos/${{ github.repository }}/releases | \ + jq 'map(select(.tag_name == "${{ github.ref_name }}"))' | \ + jq '.[0].assets' | \ + jq 'map(select(.name == "${{ steps.extract-filename.outputs.FILE_NAME }}" ))' | \ + jq '.[].id') + + wget --auth-no-challenge --header='Accept:application/octet-stream' \ + "https://${{ secrets.GH_TOKEN_RELEASE }}:@api.github.com/repos/${{ github.repository }}/releases/assets/${FILE_NAME}" \ + -O ${{ steps.extract-filename.outputs.FILE_NAME }} + pip install ${{ steps.extract-filename.outputs.FILE_NAME }} + + - name: Test + run: | + cd compiler + pip install pytest + pytest -vs -m "not parallel" tests/python