build: setup build tools for python package

- Docker image to build wheels for linux_x86_64 CPython 3.[8,9,10] with
  GLIBC >= 2.24
- Specify which Python to use in Makefile
- Fix cmake build to handle when libpython isn't available (cmake>3.18)

README.md

@@ -1,3 +1,49 @@
 # Homomorphizer
 
-The homomorphizer is a compiler that takes a high level computation model and produces a programs that evaluate the model in an homomorphic way.
+The homomorphizer is a compiler that takes a high level computation model and produces a programs that evaluate the model in an homomorphic way.
+
+
+
+## Build the Python Package
+
+Currently supported platforms:
+- Linux x86_64 for python 3.8, 3.9, and 3.10
+
+### Linux
+
+We use the [manylinux](https://github.com/pypa/manylinux) docker images for building python packages for Linux. Those packages should work on distributions that have GLIBC >= 2.24.
+
+You can use Make to build the python wheels using these docker images:
+
+```bash
+$ cd compiler
+$ make package_py38  # package_py39 package_py310
+```
+
+This will build the image for the appropriate python version then copy the wheels out under `/wheels`
+
+### Build wheels in your environment
+
+#### Temporary MLIR issue
+
+Due to an issue with MLIR, you will need to manually add `__init__.py` files to the `mlir` python package after the build.
+
+```bash
+$ make python-bindings
+$ touch build/tools/zamalang/python_packages/zamalang_core/mlir/__init__.py
+$ touch build/tools/zamalang/python_packages/zamalang_core/mlir/dialects/__init__.py
+```
+
+#### Build wheel
+
+Building the wheels is actually simple.
+
+```bash
+$ pip wheel --no-deps -w ../wheels .
+```
+
+Depending on the platform you are using (specially Linux), you might need to use `auditwheel` to specify the platform this wheel is targeting. For example, in our build of the package for Linux x86_64 and GLIBC 2.24, we also run:
+
+```bash
+$ auditwheel repair ../wheels/*.whl --plat manylinux_2_24_x86_64 -w ../wheels
+```

builders/Dockerfile.release_manylinux_2_24_x86_64

@@ -0,0 +1,24 @@
+FROM quay.io/pypa/manylinux_2_24_x86_64
+
+RUN apt-get update
+RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build 
+# Set the python path. Options: [cp38-cp38, cp39-cp39, cp310-cp310]
+ARG python_tag=cp38-cp38
+# Install python deps
+RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.6.2 PyYAML
+# Setup LLVM
+COPY /llvm-project /llvm-project
+# Setup Concrete
+COPY --from=ghcr.io/zama-ai/concrete-api-env:latest /target/release /concrete/target/release
+ENV CONCRETE_PROJECT=/concrete
+# Setup and build compiler
+COPY /compiler /compiler
+WORKDIR /compiler
+RUN make Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python build
+RUN make python-bindings
+# Fix MLIR package
+RUN touch build/tools/zamalang/python_packages/zamalang_core/mlir/__init__.py
+RUN touch build/tools/zamalang/python_packages/zamalang_core/mlir/dialects/__init__.py
+# Build wheel
+RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels .
+RUN auditwheel repair /wheels/*.whl --plat manylinux_2_24_x86_64 -w /wheels

compiler/CMakeLists.txt

@@ -56,7 +56,18 @@ if(ZAMALANG_BINDINGS_PYTHON_ENABLED)
   message(STATUS "ZamaLang Python bindings are enabled.")
 
   include(MLIRDetectPythonEnv)
-  find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+  # After CMake 3.18, we are able to limit the scope of the search to just
+  # Development.Module. Searching for Development will fail in situations where
+  # the Python libraries are not available. When possible, limit to just
+  # Development.Module.
+  # See https://pybind11.readthedocs.io/en/stable/compiling.html#findpython-mode
+  if(CMAKE_VERSION VERSION_LESS "3.18.0")
+    set(_python_development_component Development)
+  else()
+    set(_python_development_component Development.Module)
+  endif()
+  find_package(Python3 COMPONENTS Interpreter ${_python_development_component} REQUIRED)
+  unset(_python_development_component)
   message(STATUS "Found Python include dirs: ${Python3_INCLUDE_DIRS}")
   message(STATUS "Found Python libraries: ${Python3_LIBRARIES}")
   message(STATUS "Found Python executable: ${Python3_EXECUTABLE}")

compiler/Makefile

@@ -1,4 +1,5 @@
 BUILD_DIR=./build
+Python3_EXECUTABLE=
 
 
 build:
@@ -12,7 +13,8 @@ build:
 	-DZAMALANG_BINDINGS_PYTHON_ENABLED=ON \
 	-DCONCRETE_FFI_RELEASE=${CONCRETE_PROJECT}/target/release \
 	-DLLVM_EXTERNAL_PROJECTS=zamalang \
-	-DLLVM_EXTERNAL_ZAMALANG_SOURCE_DIR=.
+	-DLLVM_EXTERNAL_ZAMALANG_SOURCE_DIR=. \
+	-DPython3_EXECUTABLE=${Python3_EXECUTABLE}
 
 build-end-to-end-jit: build
 	cmake --build $(BUILD_DIR) --target end_to_end_jit_test
@@ -30,7 +32,7 @@ test-end-to-end-jit: build-end-to-end-jit
 	$(BUILD_DIR)/bin/end_to_end_jit_test
 
 test-python: python-bindings
-	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core/mlir/_mlir_libs/ LD_PRELOAD=$(BUILD_DIR)/lib/libZamalangRuntime.so pytest -vs tests/python
+	PYTHONPATH=${PYTHONPATH}:$(BUILD_DIR)/tools/zamalang/python_packages/zamalang_core LD_PRELOAD=$(BUILD_DIR)/lib/libZamalangRuntime.so pytest -vs tests/python
 
 test: test-check test-end-to-end-jit test-python
 
@@ -42,3 +44,19 @@ file-check:
 	cmake --build $(BUILD_DIR) --target FileCheck
 not:
 	cmake --build $(BUILD_DIR) --target not
+
+# Python packages
+
+define build_image_and_copy_wheels
+	docker image build -t concretefhe-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_24_x86_64 ..
+	docker container run --rm -v ${PWD}/../wheels:/wheels_volume concretefhe-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/.
+endef
+
+package_py38:
+	$(call build_image_and_copy_wheels,cp38-cp38)
+
+package_py39:
+	$(call build_image_and_copy_wheels,cp39-cp39)
+
+package_py310:
+	$(call build_image_and_copy_wheels,cp310-cp310)

compiler/lib/Bindings/Python/CMakeLists.txt

@@ -27,6 +27,7 @@ declare_mlir_python_sources(ZamalangBindingsPythonSources
   SOURCES
     zamalang/__init__.py
     zamalang/compiler.py
+    zamalang/dialects/__init__.py
     zamalang/dialects/_ods_common.py)
 
 ################################################################################

compiler/lib/Bindings/Python/zamalang/__init__.py

@@ -1,3 +1,3 @@
 """Zamalang python module"""
-from _zamalang import *
+from mlir._mlir_libs._zamalang import *
 from .compiler import CompilerEngine

compiler/lib/Bindings/Python/zamalang/compiler.py

@@ -1,8 +1,8 @@
 """Compiler submodule"""
 from typing import List, Union
-from _zamalang._compiler import CompilerEngine as _CompilerEngine
-from _zamalang._compiler import ExecutionArgument as _ExecutionArgument
-from _zamalang._compiler import round_trip as _round_trip
+from mlir._mlir_libs._zamalang._compiler import CompilerEngine as _CompilerEngine
+from mlir._mlir_libs._zamalang._compiler import ExecutionArgument as _ExecutionArgument
+from mlir._mlir_libs._zamalang._compiler import round_trip as _round_trip
 
 
 def round_trip(mlir_str: str) -> str:

compiler/lib/Bindings/Python/zamalang/dialects/hlfhe.py

@@ -1,3 +1,3 @@
 """HLFHE dialect module"""
 from ._HLFHE_ops_gen import *
-from _zamalang._hlfhe import *
+from mlir._mlir_libs._zamalang._hlfhe import *

compiler/setup.py

@@ -0,0 +1,55 @@
+import os
+import subprocess
+import setuptools
+
+from setuptools import Extension
+from setuptools.command.build_ext import build_ext
+
+
+def read(fname):
+    return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+class MakeExtension(Extension):
+    def __init__(self, name, sourcedir=""):
+        Extension.__init__(self, name, sources=[])
+        self.sourcedir = os.path.abspath(sourcedir)
+
+
+class MakeBuild(build_ext):
+    def run(self):
+        for ext in self.extensions:
+            self.build_extension(ext)
+
+    def build_extension(self, ext):
+        subprocess.check_call(["make", "python-bindings"])
+
+
+setuptools.setup(
+    name="concretefhe-compiler",
+    version="0.1.0",
+    author="Zama Team",
+    author_email="hello@zama.ai",
+    description="Concrete Compiler",
+    license="",
+    keywords="homomorphic encryption compiler",
+    long_description=read("README.md"),
+    long_description_content_type="text/markdown",
+    url="https://github.com/zama-ai/homomorphizer",
+    packages=setuptools.find_packages(
+        where="build/tools/zamalang/python_packages/zamalang_core",
+        include=["zamalang", "zamalang.*", "mlir", "mlir.*"],
+    ),
+    package_dir={"": "build/tools/zamalang/python_packages/zamalang_core"},
+    include_package_data=True,
+    package_data={"": ["*.so"]},
+    classifiers=[
+        "Programming Language :: C++",
+        "Programming Language :: Python :: 3",
+        "Topic :: Software Development :: Compilers",
+        "Topic :: Security :: Cryptography",
+    ],
+    ext_modules=[MakeExtension("python-bindings")],
+    cmdclass=dict(build_ext=MakeBuild),
+    zip_safe=False,
+)