ci: change package release process

it should be now faster using the same compilation cache as the test
build.

.github/workflows/continuous-integration.yml

@@ -360,12 +360,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python: [37, 38, 39, 310]
+        include:
+          - python: 37
+            python_dir: "cp37-cp37m"
+          - python: 38
+            python_dir: "cp38-cp38"
+          - python: 39
+            python_dir: "cp39-cp39"
+          - python: 310
+            python_dir: "cp310-cp310"
     outputs:
-      python-package-name-linux-py37: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY37 }}
-      python-package-name-linux-py38: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY38 }}
-      python-package-name-linux-py39: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY39 }}
-      python-package-name-linux-py310: ${{ steps.build-wheel-linux.outputs.ASSET_NAME_PY310 }}
+      python-package-name-linux-py37: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY37 }}
+      python-package-name-linux-py38: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY38 }}
+      python-package-name-linux-py39: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY39 }}
+      python-package-name-linux-py310: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME_PY310 }}
     needs: CreateRelease
     steps:
       # A SSH private key is required as some dependencies are from private repos
@@ -388,14 +396,44 @@ jobs:
           cd compiler
           make concrete-optimizer-lib
 
+      - name: Update Python Version
+        run: cd compiler && make update-python-version
+
       - name: Login to Github Container Registry
         run: echo "${{ secrets.GHCR_PASSWORD }}" | docker login -u ${{ secrets.GHCR_LOGIN }} --password-stdin ghcr.io
 
-      - name: Build
-        id: build-wheel-linux
+      - name: Build Wheel
+        uses: addnab/docker-run-action@v3
+        with:
+          registry: ghcr.io
+          image: ${{ env.DOCKER_IMAGE_TEST }}
+          username: ${{ secrets.GHCR_LOGIN }}
+          password: ${{ secrets.GHCR_PASSWORD }}
+          options: >-
+            -v ${{ github.workspace }}/llvm-project:/llvm-project
+            -v ${{ github.workspace }}/compiler:/compiler
+            -v ${{ github.workspace }}/wheels:/wheels
+          shell: bash
+          run: |
+            set -e
+            cd /compiler
+            rm -rf /build
+            export PYTHON_EXEC=/opt/python/${{ matrix.python_dir }}/bin/python
+            $PYTHON_EXEC -m pip install -r /llvm-project/mlir/python/requirements.txt
+            # setup env variable for wheel building
+            export CONCRETE_COMPILER_Python3_EXECUTABLE=$PYTHON_EXEC
+            export CONCRETE_COMPILER_BUILD_DIR=/build
+            /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels .
+            # We need to run it twice: the first will generate the directories, so that
+            # the second run can find the packages via find_namespace_packages
+            /opt/python/${{ matrix.python_dir }}/bin/pip wheel -vvv --no-deps -w /wheels .
+            auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels
+            echo "Debug: ccache statistics (after the build):"
+            ccache -s
+
+      - name: Set Outputs
+        id: set-output-wheel-linux
         run: |
-          cd compiler
-          make package_py${{ matrix.python }}
           echo "::set-output name=ASSET_NAME::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `"
           # used later for python package test
           echo "::set-output name=ASSET_NAME_PY${{ matrix.python }}::`find ${{ github.workspace }}/wheels/ -name *manylinux*.whl | rev |cut -d "/" -f 1 |rev `"
@@ -406,8 +444,8 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GH_TOKEN_RELEASE }}
         with:
           upload_url: ${{ needs.CreateRelease.outputs.upload_url }}
-          asset_path: ${{ github.workspace }}/wheels/${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
-          asset_name: ${{ steps.build-wheel-linux.outputs.ASSET_NAME }}
+          asset_path: ${{ github.workspace }}/wheels/${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
+          asset_name: ${{ steps.set-output-wheel-linux.outputs.ASSET_NAME }}
           asset_content_type: application/zip
 
   BuildAndPushTarballLinux:
@@ -597,7 +635,7 @@ jobs:
         run: |
           cd compiler
           pip install pytest
-          pytest -vs -m "not parallel" tests/python
+          pytest -vs tests/python
 
   TestPythonPackageMacOS:
     runs-on: macos-11

builders/Dockerfile.release_manylinux_2_28_x86_64

@@ -1,25 +0,0 @@
-FROM quay.io/pypa/manylinux_2_28_x86_64
-
-RUN dnf update -y
-RUN dnf install -y ninja-build hwloc-devel
-# Install boost
-ADD https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz /boost_1_71_0.tar.gz
-RUN tar -xzvf /boost_1_71_0.tar.gz
-WORKDIR /boost_1_71_0
-RUN ./bootstrap.sh && ./b2 --with-filesystem install
-# Set the python path. Options: [cp37-cp37m, cp38-cp38, cp39-cp39, cp310-cp310]
-ARG python_tag=cp38-cp38
-# Install python deps
-RUN /opt/python/${python_tag}/bin/pip install numpy pybind11==2.8 PyYAML
-# Setup HPX
-COPY --from=ghcr.io/zama-ai/hpx:libc_2_28 /hpx /hpx
-ENV HPX_INSTALL_DIR=/hpx/build
-# Setup LLVM
-COPY /llvm-project /llvm-project
-# Setup and build compiler
-COPY /compiler /compiler
-WORKDIR /compiler
-RUN make DATAFLOW_EXECUTION_ENABLED=ON Python3_EXECUTABLE=/opt/python/${python_tag}/bin/python python-bindings
-# Build wheel
-RUN /opt/python/${python_tag}/bin/pip wheel --no-deps -w /wheels .
-RUN auditwheel repair /wheels/*.whl --plat manylinux_2_28_x86_64 -w /wheels

compiler/Makefile

@@ -273,26 +273,6 @@ file-check: build-initialized
 not: build-initialized
 	cmake --build $(BUILD_DIR) --target not
 
-# Python packages
-
-define build_image_and_copy_wheels
-	docker image build -t concrete-compiler-manylinux:$(1) --build-arg python_tag=$(1) -f ../builders/Dockerfile.release_manylinux_2_28_x86_64 ..
-	docker container run --rm -v ${PWD}/../wheels:/wheels_volume concrete-compiler-manylinux:$(1) cp -r /wheels/. /wheels_volume/.
-endef
-
-
-package_py37: update_python_version
-	$(call build_image_and_copy_wheels,cp37-cp37m)
-
-package_py38: update_python_version
-	$(call build_image_and_copy_wheels,cp38-cp38)
-
-package_py39: update_python_version
-	$(call build_image_and_copy_wheels,cp39-cp39)
-
-package_py310: update_python_version
-	$(call build_image_and_copy_wheels,cp310-cp310)
-
 release_tarballs:
 	docker image build -t concrete-compiler-manylinux:linux_x86_64_tarball -f ../builders/Dockerfile.release_tarball_linux_x86_64 ..
 	docker container run --rm -v ${PWD}/../tarballs:/tarballs_volume concrete-compiler-manylinux:linux_x86_64_tarball cp -r /tarballs/. /tarballs_volume/.
@@ -316,10 +296,6 @@ python_lint:
 	add-deps \
 	file-check \
 	not \
-	package_py37 \
-	package_py38 \
-	package_py39 \
-	package_py310 \
 	release_tarballs \
 	update_python_version \
 	python_lint \

compiler/setup.py

@@ -35,13 +35,11 @@ class MakeBuild(build_ext):
             self.build_extension(ext)
 
     def build_extension(self, ext):
-        cmd = ["make", f"BUILD_DIR={build_dir()}"]
+        cmd = ["make", "DATAFLOW_EXECUTION_ENABLED=ON", "CCACHE=ON"]
         py_exec = os.environ.get("CONCRETE_COMPILER_Python3_EXECUTABLE")
         if py_exec:
             cmd.append(f"Python3_EXECUTABLE={py_exec}")
-        ccache = os.environ.get("CONCRETE_COMPILER_CCACHE")
-        if ccache:
-            cmd.append(f"CCACHE={ccache}")
+        cmd.append(f"BUILD_DIR={build_dir()}")
         cmd.append("python-bindings")
         subprocess.check_call(cmd)