From 117e15cc058c470bab40204fb775caad04532e7d Mon Sep 17 00:00:00 2001 From: rudy Date: Mon, 30 Jan 2023 09:29:41 +0100 Subject: [PATCH] fix: build python package during build --- .github/workflows/aws_build_cpu.yml | 14 ++++++++-- .github/workflows/aws_build_gpu.yml | 12 ++++++-- .github/workflows/macos_build.yml | 4 +-- .github/workflows/prepare_release.yml | 6 ++-- builders/Dockerfile.concrete-compiler-env | 2 +- compiler/Makefile | 34 +++++++++++++++++------ 6 files changed, 51 insertions(+), 21 deletions(-) diff --git a/.github/workflows/aws_build_cpu.yml b/.github/workflows/aws_build_cpu.yml index 955af36b6..f399a0fdf 100644 --- a/.github/workflows/aws_build_cpu.yml +++ b/.github/workflows/aws_build_cpu.yml @@ -25,6 +25,7 @@ on: env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + GLIB_VER: 2_28 jobs: BuildAndTest: @@ -83,6 +84,7 @@ jobs: -v ${{ github.workspace }}/llvm-project:/llvm-project -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket -e SSH_AUTH_SOCK=/ssh.socket ${{ env.DOCKER_GPU_OPTION }} @@ -91,9 +93,7 @@ jobs: set -e cd /compiler rm -rf /build/* - pip install pytest - sed "s/pytest/python -m pytest/g" -i Makefile - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all build-end-to-end-dataflow-tests + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all python-package build-end-to-end-dataflow-tests echo "Debug: ccache statistics (after the build):" ccache -s @@ -114,10 +114,18 @@ jobs: set -e cd /compiler pip install pytest + sed "s/pytest/python -m pytest/g" -i Makefile mkdir -p /tmp/concrete_compiler/gpu_tests/ make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests chmod -R ugo+rwx /tmp/KeySetCache + - name: Archive python package + uses: actions/upload-artifact@v3 + with: + name: concrete-compiler.whl + path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl + retention-days: 14 + - name: Build the documentation id: build-doc if: ${{ steps.build-compiler.outcome == 'success' && !cancelled() }} diff --git a/.github/workflows/aws_build_gpu.yml b/.github/workflows/aws_build_gpu.yml index 70a8a7752..fc8c69f89 100644 --- a/.github/workflows/aws_build_gpu.yml +++ b/.github/workflows/aws_build_gpu.yml @@ -85,6 +85,7 @@ jobs: -v ${{ github.workspace }}/llvm-project:/llvm-project -v ${{ github.workspace }}/compiler:/compiler -v ${{ github.workspace }}/build:/build + -v ${{ github.workspace }}/wheels:/wheels -v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket -e SSH_AUTH_SOCK=/ssh.socket --gpus all @@ -93,12 +94,16 @@ jobs: set -e cd /compiler rm -rf /build/* - pip install pytest - sed "s/pytest/python -m pytest/g" -i Makefile - make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} all build-end-to-end-dataflow-tests + make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} all python-package build-end-to-end-dataflow-tests echo "Debug: ccache statistics (after the build):" ccache -s + - name: Archive python package + uses: actions/upload-artifact@v3 + with: + name: concrete-compiler-gpu + path: build/wheels/*.whl + - name: Test compiler uses: addnab/docker-run-action@v3 with: @@ -116,6 +121,7 @@ jobs: set -e cd /compiler pip install pytest + sed "s/pytest/python -m pytest/g" -i Makefile mkdir -p /tmp/concrete_compiler/gpu_tests/ make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-end-to-end-tests-gpu chmod -R ugo+rwx /tmp/KeySetCache diff --git a/.github/workflows/macos_build.yml b/.github/workflows/macos_build.yml index 32b308a53..ea765f77e 100644 --- a/.github/workflows/macos_build.yml +++ b/.github/workflows/macos_build.yml @@ -63,7 +63,7 @@ jobs: cd compiler echo "Debug: ccache statistics (prior to the build):" ccache -s - make Python3_EXECUTABLE=$(which python3.10) all run-check-tests + make Python3_EXECUTABLE=$(which python3.10) all run-check-tests python-package echo "Debug: ccache statistics (after the build):" ccache -s @@ -75,8 +75,6 @@ jobs: echo "Debug: ccache statistics (prior to the tests):" ccache -s export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip3.10 wheel --no-deps -w ${{ github.workspace }}/wheels . - delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') pip3.10 install $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') make Python3_EXECUTABLE=$(which python3.10) run-tests echo "Debug: ccache statistics (after the tests):" diff --git a/.github/workflows/prepare_release.yml b/.github/workflows/prepare_release.yml index b1758d426..f02748471 100644 --- a/.github/workflows/prepare_release.yml +++ b/.github/workflows/prepare_release.yml @@ -8,6 +8,7 @@ on: env: DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler + GLIB_VER: 2_28 jobs: WaitOnAllWorkflows: @@ -99,7 +100,7 @@ jobs: export CONCRETE_COMPILER_CUDA_SUPPORT=ON export PATH=/opt/python/${{ matrix.python_dir }}/bin:PATH$ make python-package - cp build/wheels/*.whl /wheels + cp build/wheels/*manylinux_${{ env.GLIB_VER }}_x86_64.whl /wheels echo "Debug: ccache statistics (after the build):" ccache -s @@ -237,8 +238,7 @@ jobs: cd compiler make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF - pip wheel --no-deps -w ${{ github.workspace }}/wheels . - delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl') + make python-package echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" # used later for python package test echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )" diff --git a/builders/Dockerfile.concrete-compiler-env b/builders/Dockerfile.concrete-compiler-env index 06562b373..130f3511b 100644 --- a/builders/Dockerfile.concrete-compiler-env +++ b/builders/Dockerfile.concrete-compiler-env @@ -34,7 +34,7 @@ RUN ln -s /opt/python/${python_tag}/bin/pip /bin/pip RUN ln -s /opt/python/${python_tag}/bin/python /bin/python ENV PYTHON_EXEC=/opt/python/${python_tag}/bin/python # Install python deps -RUN pip install numpy pybind11==2.8 PyYAML +RUN pip install numpy pybind11==2.8 PyYAML pytest wheel auditwheel # Setup LLVM COPY /llvm-project /llvm-project # Setup and build compiler diff --git a/compiler/Makefile b/compiler/Makefile index ee8616f4d..c9d59ce12 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -443,16 +443,34 @@ else HIERARCHY_PRESERVING_COPY=cp --parents endif -python-package: - export GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_') - $(Python3_EXECUTABLE) -m pip install wheel auditwheel - $(Python3_EXECUTABLE) -m pip wheel -vvv --no-deps -w $(BUILD_DIR)/wheels . +ifeq ($(OS),Windows_NT) + detected_OS := Windows +else + detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown') +endif + +PIP=$(Python3_EXECUTABLE) -m pip +PIP_WHEEL=$(PIP) wheel --no-deps -w $(BUILD_DIR)/wheels . +AUDIT_WHEEL_REPAIR=$(Python3_EXECUTABLE) -m auditwheel repair -w $(BUILD_DIR)/wheels + +linux-python-package: + $(PIP) install wheel auditwheel # We need to run it twice: the first will generate the directories, so that # the second run can find the packages via find_namespace_packages - $(Python3_EXECUTABLE) -m pip wheel -vvv --no-deps -w $(BUILD_DIR)/wheels . - $(Python3_EXECUTABLE) -m auditwheel repair $(BUILD_DIR)/wheels/*.whl --plat manylinux_$(GLIBC_VER)_x86_64 -w $(BUILD_DIR)/wheels || echo No repair to manylinux - $(Python3_EXECUTABLE) -m auditwheel repair $(BUILD_DIR)/wheels/*.whl --plat linux_x86_64 -w $(BUILD_DIR)/wheels || echo No repair - @echo The wheels: $(BUILD_DIR)/wheels/*.whl + $(PIP_WHEEL) + $(PIP_WHEEL) + export GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_') + for PLATFORM in manylinux_$(GLIBC_VER)_x86_64 linux_x86_64; do \ + $(AUDIT_WHEEL_REPAIR) $(BUILD_DIR)/wheels/*.whl --plat $$PLATFORM || echo No repair with $(PLATFORM) ;\ + done + +darwin-python-package: + $(PIP) install wheel delocate + $(PIP_WHEEL) + delocate-wheel -v $(BUILD_DIR)/wheels/*macosx*.whl + +python-package: python-bindings $(OS)-python-package + @echo The python package is: $(BUILD_DIR)/wheels/*.whl install: concretecompiler concrete-optimizer-lib CAPI install-deps $(info Install prefix set to $(INSTALL_PREFIX))