github/concrete
1
1
Fork 0
mirror of https://github.com/zama-ai/concrete.git synced 2026-02-08 19:44:57 -05:00
Code Issues Packages Projects Releases Wiki Activity

fix: build python package during build

Browse Source
This commit is contained in:
rudy
2023-01-30 09:29:41 +01:00
committed by rudy-6-4
parent a01fab0a90
commit 117e15cc05
6 changed files with 51 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.github/workflows/aws_build_cpu.yml vendored
View File

@@ -25,6 +25,7 @@ on:
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
jobs:
BuildAndTest:
@@ -83,6 +84,7 @@ jobs:
-v ${{ github.workspace }}/llvm-project:/llvm-project
-v ${{ github.workspace }}/compiler:/compiler
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
${{ env.DOCKER_GPU_OPTION }}
@@ -91,9 +93,7 @@ jobs:
set -e
cd /compiler
rm -rf /build/*
pip install pytest
sed "s/pytest/python -m pytest/g" -i Makefile
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all build-end-to-end-dataflow-tests
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build all python-package build-end-to-end-dataflow-tests
echo "Debug: ccache statistics (after the build):"
ccache -s
@@ -114,10 +114,18 @@ jobs:
set -e
cd /compiler
pip install pytest
sed "s/pytest/python -m pytest/g" -i Makefile
mkdir -p /tmp/concrete_compiler/gpu_tests/
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-tests run-end-to-end-dataflow-tests
chmod -R ugo+rwx /tmp/KeySetCache
- name: Archive python package
uses: actions/upload-artifact@v3
with:
name: concrete-compiler.whl
path: build/wheels/concrete_compiler-*-manylinux_{{ env.GLIB_VER }}_x86_64.whl
retention-days: 14
- name: Build the documentation
id: build-doc
if: ${{ steps.build-compiler.outcome == 'success' && !cancelled() }}

12
.github/workflows/aws_build_gpu.yml vendored
View File

@@ -85,6 +85,7 @@ jobs:
-v ${{ github.workspace }}/llvm-project:/llvm-project
-v ${{ github.workspace }}/compiler:/compiler
-v ${{ github.workspace }}/build:/build
-v ${{ github.workspace }}/wheels:/wheels
-v ${{ env.SSH_AUTH_SOCK }}:/ssh.socket
-e SSH_AUTH_SOCK=/ssh.socket
--gpus all
@@ -93,12 +94,16 @@ jobs:
set -e
cd /compiler
rm -rf /build/*
pip install pytest
sed "s/pytest/python -m pytest/g" -i Makefile
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} all build-end-to-end-dataflow-tests
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build CUDA_SUPPORT=ON CUDA_PATH=${{ env.CUDA_PATH }} all python-package build-end-to-end-dataflow-tests
echo "Debug: ccache statistics (after the build):"
ccache -s
- name: Archive python package
uses: actions/upload-artifact@v3
with:
name: concrete-compiler-gpu
path: build/wheels/*.whl
- name: Test compiler
uses: addnab/docker-run-action@v3
with:
@@ -116,6 +121,7 @@ jobs:
set -e
cd /compiler
pip install pytest
sed "s/pytest/python -m pytest/g" -i Makefile
mkdir -p /tmp/concrete_compiler/gpu_tests/
make DATAFLOW_EXECUTION_ENABLED=ON CCACHE=ON Python3_EXECUTABLE=$PYTHON_EXEC BUILD_DIR=/build run-end-to-end-tests-gpu
chmod -R ugo+rwx /tmp/KeySetCache

4
.github/workflows/macos_build.yml vendored
View File

@@ -63,7 +63,7 @@ jobs:
cd compiler
echo "Debug: ccache statistics (prior to the build):"
ccache -s
make Python3_EXECUTABLE=$(which python3.10) all run-check-tests
make Python3_EXECUTABLE=$(which python3.10) all run-check-tests python-package
echo "Debug: ccache statistics (after the build):"
ccache -s
@@ -75,8 +75,6 @@ jobs:
echo "Debug: ccache statistics (prior to the tests):"
ccache -s
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
pip3.10 wheel --no-deps -w ${{ github.workspace }}/wheels .
delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
pip3.10 install $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
make Python3_EXECUTABLE=$(which python3.10) run-tests
echo "Debug: ccache statistics (after the tests):"

6
.github/workflows/prepare_release.yml vendored
View File

@@ -8,6 +8,7 @@ on:
env:
DOCKER_IMAGE_TEST: ghcr.io/zama-ai/concrete-compiler
GLIB_VER: 2_28
jobs:
WaitOnAllWorkflows:
@@ -99,7 +100,7 @@ jobs:
export CONCRETE_COMPILER_CUDA_SUPPORT=ON
export PATH=/opt/python/${{ matrix.python_dir }}/bin:PATH$
make python-package
cp build/wheels/*.whl /wheels
cp build/wheels/*manylinux_${{ env.GLIB_VER }}_x86_64.whl /wheels
echo "Debug: ccache statistics (after the build):"
ccache -s
@@ -237,8 +238,7 @@ jobs:
cd compiler
make Python3_EXECUTABLE=$(which python) DATAFLOW_EXECUTION_ENABLED=OFF python-bindings
export CONCRETE_COMPILER_DATAFLOW_EXECUTION_ENABLED=OFF
pip wheel --no-deps -w ${{ github.workspace }}/wheels .
delocate-wheel -v $(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl')
make python-package
echo "::set-output name=ASSET_NAME::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )"
# used later for python package test
echo "::set-output name=ASSET_NAME_PY$(echo ${{ matrix.python }} |tr -d '.')::$(find ${{ github.workspace }}/wheels/ -name '*macosx*.whl' | rev |cut -d "/" -f 1 |rev )"

2
builders/Dockerfile.concrete-compiler-env
View File

@@ -34,7 +34,7 @@ RUN ln -s /opt/python/${python_tag}/bin/pip /bin/pip
RUN ln -s /opt/python/${python_tag}/bin/python /bin/python
ENV PYTHON_EXEC=/opt/python/${python_tag}/bin/python
# Install python deps
RUN pip install numpy pybind11==2.8 PyYAML
RUN pip install numpy pybind11==2.8 PyYAML pytest wheel auditwheel
# Setup LLVM
COPY /llvm-project /llvm-project
# Setup and build compiler

34
compiler/Makefile
View File

@@ -443,16 +443,34 @@ else
HIERARCHY_PRESERVING_COPY=cp --parents
endif
python-package:
export GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_')
$(Python3_EXECUTABLE) -m pip install wheel auditwheel
$(Python3_EXECUTABLE) -m pip wheel -vvv --no-deps -w $(BUILD_DIR)/wheels .
ifeq ($(OS),Windows_NT)
detected_OS := Windows
else
detected_OS := $(shell sh -c 'uname 2>/dev/null || echo Unknown')
endif
PIP=$(Python3_EXECUTABLE) -m pip
PIP_WHEEL=$(PIP) wheel --no-deps -w $(BUILD_DIR)/wheels .
AUDIT_WHEEL_REPAIR=$(Python3_EXECUTABLE) -m auditwheel repair -w $(BUILD_DIR)/wheels
linux-python-package:
$(PIP) install wheel auditwheel
# We need to run it twice: the first will generate the directories, so that
# the second run can find the packages via find_namespace_packages
$(Python3_EXECUTABLE) -m pip wheel -vvv --no-deps -w $(BUILD_DIR)/wheels .
$(Python3_EXECUTABLE) -m auditwheel repair $(BUILD_DIR)/wheels/*.whl --plat manylinux_$(GLIBC_VER)_x86_64 -w $(BUILD_DIR)/wheels || echo No repair to manylinux
$(Python3_EXECUTABLE) -m auditwheel repair $(BUILD_DIR)/wheels/*.whl --plat linux_x86_64 -w $(BUILD_DIR)/wheels || echo No repair
@echo The wheels: $(BUILD_DIR)/wheels/*.whl
$(PIP_WHEEL)
$(PIP_WHEEL)
export GLIBC_VER=$(shell ldd --version | head -n 1 | grep -o '[^ ]*$$'|head|tr '.' '_')
for PLATFORM in manylinux_$(GLIBC_VER)_x86_64 linux_x86_64; do \
$(AUDIT_WHEEL_REPAIR) $(BUILD_DIR)/wheels/*.whl --plat $$PLATFORM || echo No repair with $(PLATFORM) ;\
done
darwin-python-package:
$(PIP) install wheel delocate
$(PIP_WHEEL)
delocate-wheel -v $(BUILD_DIR)/wheels/*macosx*.whl
python-package: python-bindings $(OS)-python-package
@echo The python package is: $(BUILD_DIR)/wheels/*.whl
install: concretecompiler concrete-optimizer-lib CAPI install-deps
$(info Install prefix set to $(INSTALL_PREFIX))