feat: support dataflow execution in python packages

use latest image manylinux_2_28_x86_64 using libc 2.28

.github/workflows/continuous-integration.yml

@@ -807,8 +807,8 @@ jobs:
       - name: Build Tag and Publish
         if: ${{ steps.login.conclusion != 'skipped' }}
         run: |
-          docker build -t $IMAGE -f builders/Dockerfile.hpx-env .
-          docker push $IMAGE:latest
+          docker build -t $IMAGE:libc_2_28 -f builders/Dockerfile.hpx-env .
+          docker push $IMAGE:libc_2_28
 
       - name: Is Image Built
         id: is-built

builders/Dockerfile.hpx-env

@@ -1,22 +1,22 @@
-FROM ubuntu:latest
+FROM quay.io/pypa/manylinux_2_28_x86_64
 
-RUN apt-get update --fix-missing
-RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y curl cmake g++ build-essential python3 python3-pip python3-setuptools ninja-build git libboost-filesystem-dev libhwloc-dev
-RUN pip install numpy pybind11==2.8 PyYAML
-RUN mkdir /cmake-build
-ADD https://github.com/Kitware/CMake/releases/download/v3.22.0/cmake-3.22.0-linux-x86_64.tar.gz /cmake-build/cmake.tar.gz
-RUN cd /cmake-build && tar xzf cmake.tar.gz
-ENV PATH=/cmake-build/cmake-3.22.0-linux-x86_64/bin:${PATH}
-RUN git clone https://github.com/STEllAR-GROUP/hpx.git
-ENV HPX=$PWD/hpx
-RUN cd ${HPX} && git checkout 1.7.1
-RUN mkdir ${HPX}/build
-RUN cd ${HPX}/build && cmake \
-       		    -DHPX_WITH_FETCH_ASIO=on \
-		    -DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \
-		    -DHPX_WITH_MALLOC=system ..
-RUN cd ${HPX}/build && make -j2
+RUN dnf update -y
+RUN dnf install -y ninja-build hwloc-devel
+# Install boost
+ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz
+RUN tar -xzvf /boost_1_71_0.tar.gz
+WORKDIR /boost_1_71_0
+RUN ./bootstrap.sh && ./b2 --with-filesystem install
+# Build HPX
+RUN git clone https://github.com/STEllAR-GROUP/hpx.git /hpx
+WORKDIR /hpx
+RUN git checkout 1.7.1
+RUN mkdir build
+RUN cd build && cmake \
+		-DHPX_WITH_FETCH_ASIO=on \
+		-DHPX_FILESYSTEM_WITH_BOOST_FILESYSTEM_COMPATIBILITY=ON \
+		-DHPX_WITH_MALLOC=system ..
+RUN cd build && make -j2
 
-FROM ubuntu:latest
+FROM quay.io/pypa/manylinux_2_28_x86_64
 COPY --from=0 /hpx/ /hpx/
-COPY --from=0 /cmake-build/ /cmake/

builders/Dockerfile.release_manylinux_2_24_x86_64

@@ -1,30 +0,0 @@
-FROM quay.io/pypa/manylinux_2_24_x86_64
-
-RUN apt-get update
-RUN DEBIAN_FRONTEND="noninteractive" apt-get install -y build-essential ninja-build
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
-# Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
-ARG python_tag=cp38-cp38
-# Install python deps
-RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.8 PyYAML
-# Setup gcc7
-COPY --from=ghcr.io/zama-ai/gcc7:latest /gcc7 /gcc7
-ENV PATH=/gcc7/bin:$PATH
-ENV LD_LIBRARY_PATH=/gcc7/lib/:/gcc7/lib64/:$LD_LIBRARY_PATH
-ENV CC=/gcc7/bin/gcc-7.5.0
-ENV CXX=/gcc7/bin/g++-7.5.0
-# Setup LLVM
-COPY /llvm-project /llvm-project
-# Setup Concrete
-COPY --from=ghcr.io/zama-ai/concrete-compiler-api-env:latest /target/release /concrete/target/release
-ENV CONCRETE_PROJECT=/concrete
-# Setup and build compiler
-COPY /compiler /compiler
-WORKDIR /compiler
-RUN make -e CXX_COMPILER=$CXX CC_COMPILER=$CC Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python python-bindings
-# Fix MLIR package
-RUN touch build/tools/concretelang/python_packages/concretelang_core/mlir/__init__.py
-RUN touch build/tools/concretelang/python_packages/concretelang_core/mlir/dialects/__init__.py
-# Build wheel
-RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels .
-RUN auditwheel repair /wheels/*.whl --plat manylinux_2_24_x86_64 -w /wheels

builders/Dockerfile.release_manylinux_2_28_x86_64

@@ -0,0 +1,25 @@
+FROM quay.io/pypa/manylinux_2_28_x86_64
+
+RUN dnf update -y
+RUN dnf install -y ninja-build hwloc-devel
+# Install boost
+ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz
+RUN tar -xzvf /boost_1_71_0.tar.gz
+WORKDIR /boost_1_71_0
+RUN ./bootstrap.sh && ./b2 --with-filesystem install
+# Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
+ARG python_tag=cp38-cp38
+# Install python deps
+RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.8 PyYAML
+# Setup HPX
+COPY --from=ghcr.io/zama-ai/hpx:libc_2_28 /hpx /hpx
+ENV HPX_INSTALL_DIR=/hpx/build
+# Setup LLVM
+COPY /llvm-project /llvm-project
+# Setup and build compiler
+COPY /compiler /compiler
+WORKDIR /compiler
+RUN make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python python-bindings
+# Build wheel
+RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels .
+RUN auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels

compiler/Makefile

@@ -262,7 +262,7 @@ not: build-initialized
 # Python packages
 
 define build_image_and_copy_wheels
-	docker image build -t concrete-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_24_x86_64 ..
+	docker image build -t concrete-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_28_x86_64 ..
 	docker container run --rm -v ${PWD}/../wheels:/wheels_volume concrete-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/.
 endef