diff --git a/README.md b/README.md index af3f44889..d0d8547e5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,49 @@ # Homomorphizer -The homomorphizer is a compiler that takes a high level computation model and produces a programs that evaluate the model in an homomorphic way. \ No newline at end of file +The homomorphizer is a compiler that takes a high level computation model and produces a programs that evaluate the model in an homomorphic way. + + + +## Build the Python Package + +Currently supported platforms: +- Linux x86_64 for python 3.8, 3.9, and 3.10 + +### Linux + +We use the [manylinux](https://github.com/pypa/manylinux) docker images for building python packages for Linux. Those packages should work on distributions that have GLIBC >= 2.24. + +You can use Make to build the python wheels using these docker images: + +```bash +$ cd compiler +$ make package_py38 # package_py39 package_py310 +``` + +This will build the image for the appropriate python version then copy the wheels out under `/wheels` + +### Build wheels in your environment + +#### Temporary MLIR issue + +Due to an issue with MLIR, you will need to manually add `__init__.py` files to the `mlir` python package after the build. + +```bash +$ make python-bindings +$ touch build/tools/zamalang/python_packages/zamalang_core/mlir/__init__.py +$ touch build/tools/zamalang/python_packages/zamalang_core/mlir/dialects/__init__.py +``` + +#### Build wheel + +Building the wheels is actually simple. + +```bash +$ pip wheel --no-deps -w ../wheels . +``` + +Depending on the platform you are using (specially Linux), you might need to use `auditwheel` to specify the platform this wheel is targeting. For example, in our build of the package for Linux x86_64 and GLIBC 2.24, we also run: + +```bash +$ auditwheel repair ../wheels/*.whl --plat manylinux_2_24_x86_64 -w ../wheels +``` diff --git a/builders/Dockerfile.release_manylinux_2_24_x86_64 b/builders/Dockerfile.release_manylinux_2_24_x86_64 new file mode 100644 index 000000000..efd00a0fd --- /dev/null +++ b/builders/Dockerfile.release_manylinux_2_24_x86_64 @@ -0,0 +1,24 @@ +FROM quay.io/pypa/manylinux_2_24_x86_64 + +RUN apt-get update +RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build +# Set the python path. Options: [cp38-cp38, cp39-cp39, cp310-cp310] +ARG python_tag=cp38-cp38 +# Install python deps +RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.6.2 PyYAML +# Setup LLVM +COPY /llvm-project /llvm-project +# Setup Concrete +COPY --from=ghcr.io/zama-ai/concrete-api-env:latest /target/release /concrete/target/release +ENV CONCRETE_PROJECT=/concrete +# Setup and build compiler +COPY /compiler /compiler +WORKDIR /compiler +RUN make Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python build +RUN make python-bindings +# Fix MLIR package +RUN touch build/tools/zamalang/python_packages/zamalang_core/mlir/__init__.py +RUN touch build/tools/zamalang/python_packages/zamalang_core/mlir/dialects/__init__.py +# Build wheel +RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels . +RUN auditwheel repair /wheels/*.whl --plat manylinux_2_24_x86_64 -w /wheels \ No newline at end of file diff --git a/compiler/CMakeLists.txt b/compiler/CMakeLists.txt index 7c8adbdda..1de40aa21 100644 --- a/compiler/CMakeLists.txt +++ b/compiler/CMakeLists.txt @@ -56,7 +56,18 @@ if(ZAMALANG_BINDINGS_PYTHON_ENABLED) message(STATUS "ZamaLang Python bindings are enabled.") include(MLIRDetectPythonEnv) - find_package(Python3 COMPONENTS Interpreter Development REQUIRED) + # After CMake 3.18, we are able to limit the scope of the search to just + # Development.Module. Searching for Development will fail in situations where + # the Python libraries are not available. When possible, limit to just + # Development.Module. + # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode + if(CMAKE_VERSION VERSION_LESS "3.18.0") + set(_python_development_component Development) + else() + set(_python_development_component Development.Module) + endif() + find_package(Python3 COMPONENTS Interpreter ${_python_development_component} REQUIRED) + unset(_python_development_component) message(STATUS "Found Python include dirs: ${Python3_INCLUDE_DIRS}") message(STATUS "Found Python libraries: ${Python3_LIBRARIES}") message(STATUS "Found Python executable: ${Python3_EXECUTABLE}") diff --git a/compiler/Makefile b/compiler/Makefile index dda452d31..325753c13 100644 --- a/compiler/Makefile +++ b/compiler/Makefile @@ -1,4 +1,5 @@ BUILD_DIR=./build +Python3_EXECUTABLE= build: @@ -12,7 +13,8 @@ build: -DZAMALANG_BINDINGS_PYTHON_ENABLED=ON \ -DCONCRETE_FFI_RELEASE=${CONCRETE_PROJECT}/target/release \ -DLLVM_EXTERNAL_PROJECTS=zamalang \ - -DLLVM_EXTERNAL_ZAMALANG_SOURCE_DIR=. + -DLLVM_EXTERNAL_ZAMALANG_SOURCE_DIR=. \ + -DPython3_EXECUTABLE=${Python3_EXECUTABLE} build-end-to-end-jit: build cmake --build $(BUILD_DIR) --target end_to_end_jit_test @@ -30,7 +32,7 @@ test-end-to-end-jit: build-end-to-end-jit $(BUILD_DIR)/bin/end_to_end_jit_test test-python: python-bindings - PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core/mlir/_mlir_libs/ LD_PRELOAD=$(BUILD_DIR)/lib/libZamalangRuntime.so pytest -vs tests/python + PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core LD_PRELOAD=$(BUILD_DIR)/lib/libZamalangRuntime.so pytest -vs tests/python test: test-check test-end-to-end-jit test-python @@ -42,3 +44,19 @@ file-check: cmake --build $(BUILD_DIR) --target FileCheck not: cmake --build $(BUILD_DIR) --target not + +# Python packages + +define build_image_and_copy_wheels + docker image build -t concretefhe-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_24_x86_64 .. + docker container run --rm -v ${PWD}/../wheels:/wheels_volume concretefhe-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/. +endef + +package_py38: + $(call build_image_and_copy_wheels,cp38-cp38) + +package_py39: + $(call build_image_and_copy_wheels,cp39-cp39) + +package_py310: + $(call build_image_and_copy_wheels,cp310-cp310) diff --git a/compiler/lib/Bindings/Python/CMakeLists.txt b/compiler/lib/Bindings/Python/CMakeLists.txt index c80c9c889..4dcf5f79b 100644 --- a/compiler/lib/Bindings/Python/CMakeLists.txt +++ b/compiler/lib/Bindings/Python/CMakeLists.txt @@ -27,6 +27,7 @@ declare_mlir_python_sources(ZamalangBindingsPythonSources SOURCES zamalang/__init__.py zamalang/compiler.py + zamalang/dialects/__init__.py zamalang/dialects/_ods_common.py) ################################################################################ diff --git a/compiler/lib/Bindings/Python/zamalang/__init__.py b/compiler/lib/Bindings/Python/zamalang/__init__.py index 647b40273..05a481546 100644 --- a/compiler/lib/Bindings/Python/zamalang/__init__.py +++ b/compiler/lib/Bindings/Python/zamalang/__init__.py @@ -1,3 +1,3 @@ """Zamalang python module""" -from _zamalang import * +from mlir._mlir_libs._zamalang import * from .compiler import CompilerEngine diff --git a/compiler/lib/Bindings/Python/zamalang/compiler.py b/compiler/lib/Bindings/Python/zamalang/compiler.py index 185e4a169..76e372463 100644 --- a/compiler/lib/Bindings/Python/zamalang/compiler.py +++ b/compiler/lib/Bindings/Python/zamalang/compiler.py @@ -1,8 +1,8 @@ """Compiler submodule""" from typing import List, Union -from _zamalang._compiler import CompilerEngine as _CompilerEngine -from _zamalang._compiler import ExecutionArgument as _ExecutionArgument -from _zamalang._compiler import round_trip as _round_trip +from mlir._mlir_libs._zamalang._compiler import CompilerEngine as _CompilerEngine +from mlir._mlir_libs._zamalang._compiler import ExecutionArgument as _ExecutionArgument +from mlir._mlir_libs._zamalang._compiler import round_trip as _round_trip def round_trip(mlir_str: str) -> str: diff --git a/compiler/lib/Bindings/Python/zamalang/dialects/__init__.py b/compiler/lib/Bindings/Python/zamalang/dialects/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/compiler/lib/Bindings/Python/zamalang/dialects/hlfhe.py b/compiler/lib/Bindings/Python/zamalang/dialects/hlfhe.py index 3d996e829..b87ed8953 100644 --- a/compiler/lib/Bindings/Python/zamalang/dialects/hlfhe.py +++ b/compiler/lib/Bindings/Python/zamalang/dialects/hlfhe.py @@ -1,3 +1,3 @@ """HLFHE dialect module""" from ._HLFHE_ops_gen import * -from _zamalang._hlfhe import * +from mlir._mlir_libs._zamalang._hlfhe import * diff --git a/compiler/setup.py b/compiler/setup.py new file mode 100644 index 000000000..48eccdc9a --- /dev/null +++ b/compiler/setup.py @@ -0,0 +1,55 @@ +import os +import subprocess +import setuptools + +from setuptools import Extension +from setuptools.command.build_ext import build_ext + + +def read(fname): + return open(os.path.join(os.path.dirname(__file__), fname)).read() + + +class MakeExtension(Extension): + def __init__(self, name, sourcedir=""): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class MakeBuild(build_ext): + def run(self): + for ext in self.extensions: + self.build_extension(ext) + + def build_extension(self, ext): + subprocess.check_call(["make", "python-bindings"]) + + +setuptools.setup( + name="concretefhe-compiler", + version="0.1.0", + author="Zama Team", + author_email="hello@zama.ai", + description="Concrete Compiler", + license="", + keywords="homomorphic encryption compiler", + long_description=read("README.md"), + long_description_content_type="text/markdown", + url="https://github.com/zama-ai/homomorphizer", + packages=setuptools.find_packages( + where="build/tools/zamalang/python_packages/zamalang_core", + include=["zamalang", "zamalang.*", "mlir", "mlir.*"], + ), + package_dir={"": "build/tools/zamalang/python_packages/zamalang_core"}, + include_package_data=True, + package_data={"": ["*.so"]}, + classifiers=[ + "Programming Language :: C++", + "Programming Language :: Python :: 3", + "Topic :: Software Development :: Compilers", + "Topic :: Security :: Cryptography", + ], + ext_modules=[MakeExtension("python-bindings")], + cmdclass=dict(build_ext=MakeBuild), + zip_safe=False, +)